@alexkroman1/aai-cli 7.0.0 → 8.1.0

package/dist/templates/pipeline-simple/agent.ts

@@ -1,5 +1,5 @@
 import { agent } from "@alexkroman1/aai";
-import { anthropic } from "@alexkroman1/aai/llm";
+import { anthropicLlm } from "@alexkroman1/aai/llm";
 
 export default agent({
   name: "pipeline-simple",
@@ -8,5 +8,5 @@ export default agent({
   // left unset (STT and TTS here) runs on the AssemblyAI default, billed to
   // the one key a published agent is guaranteed to have, so swapping a stage
   // never means restating the other two.
-  llm: anthropic({ model: "claude-haiku-4-5" }),
+  llm: anthropicLlm({ model: "claude-haiku-4-5" }),
 });

package/dist/templates/pizza-ordering/agent.eval.test.ts

@@ -0,0 +1,171 @@
+/// <reference types="vite/client" />
+
+import type { SessionEvent } from "@alexkroman1/aai/protocol";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+// An EVAL: does the order-taker actually take the order?
+//
+// `agent.test.ts` drives the six tools directly, which settles what each one
+// does with the arguments it is given. What no test in it can settle is whether
+// the MODEL reaches for the right one, with the arguments the caller actually
+// said — and for a cart that lives in a `sessionSlot`, whether what turn 1
+// wrote is still there on turn 2. That is what this file is for.
+//
+// Run it with `aai eval`. Without a provider key every case runs against a
+// SCRIPTED model (its `stubReply`), which still boots this agent, still
+// resolves `tools/`, and still executes the tool a script names — so a stub run
+// proves the wiring and proves nothing about what the agent chose.
+import { lastStateIn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import { MENU } from "./shared.ts";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * The glob is written HERE rather than reached for from a shared helper because
+ * this file SHIPS — a scaffolded project has no repo helper to import. Without
+ * it the eval would drive an agent with no tools at all, which is the one
+ * failure mode a green eval must not be able to hide.
+ */
+const agentDef = withDiscoveredTools(
+  authoredAgent,
+  import.meta.glob("./tools/*.ts", { eager: true }),
+);
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, so a projection
+ * that stopped matching FAILS naming the field, where the cast this replaced
+ * handed the assertions `undefined` and failed a line later on something else.
+ * It names only the fields asserted below, so `orderView` may grow without
+ * touching this.
+ */
+const ProjectedOrder = z.object({
+  pizzas: z.array(z.object({ id: z.number(), size: z.string(), toppings: z.array(z.string()) })),
+  orderPlaced: z.boolean(),
+});
+
+/**
+ * The last cart `syncState` pushed to the browser, i.e. what the page shows.
+ *
+ * This agent declares ONE projection, so the frame IS `orderView`'s result —
+ * not a record keyed by the slot. That is the same value `useAgentState(
+ * orderProjection)` reads in `client.tsx`, which is why an eval can assert on
+ * it: it is the cart the customer is looking at. `lastStateIn` is the SDK's
+ * reader for exactly this.
+ */
+const lastPushedView = (events: readonly SessionEvent[]) => lastStateIn(events, ProjectedOrder);
+
+/** Every cart the session pushed, in stream order. */
+const pushedViews = (events: readonly SessionEvent[]) =>
+  events.flatMap((e) => (e.type === "state.updated" ? [ProjectedOrder.parse(e.state)] : []));
+
+describeEval(agentDef, (test) => {
+  test(
+    "adds the pizza the caller described, at the price the menu quotes",
+    async ({ session }) => {
+      const turn = await session.say(
+        "Hi, I'd like a large pepperoni pizza with extra cheese please.",
+      );
+
+      // One tool, and the right one: quoting a price without adding the pizza,
+      // or adding it twice, are both real findings.
+      expect(turn.toolCalls.map((c) => c.name)).toEqual(["add_pizza"]);
+      const call = turn.toolCalls[0]!;
+      const args = call.args as { size: string; crust: string; toppings: string[] };
+      expect(args.size).toBe("large");
+      // The two toppings the caller named, however the model spelled them.
+      const toppings = args.toppings.map((t) => t.toLowerCase().replaceAll(" ", "_"));
+      expect(toppings).toContain("pepperoni");
+      expect(toppings).toContain("extra_cheese");
+
+      // The claim the template's own comment makes: the menu is generated from
+      // MENU, so the agent can never quote a price the ordering code doesn't
+      // charge. Computed from the menu here rather than from the tool's own
+      // output, which is what gives it teeth — a topping name the pricing table
+      // doesn't recognise silently falls back to $1.00.
+      const expected =
+        MENU.sizes[args.size as keyof typeof MENU.sizes] +
+        MENU.crusts[args.crust as keyof typeof MENU.crusts] +
+        MENU.toppings.pepperoni +
+        MENU.toppings.extra_cheese;
+      expect(
+        toolResultIn(turn.toolCalls, "add_pizza", z.object({ orderTotal: z.string() })).orderTotal,
+      ).toBe(`$${expected.toFixed(2)}`);
+    },
+    {
+      stubReply: [
+        {
+          tool: "add_pizza",
+          args: {
+            size: "large",
+            crust: "regular",
+            toppings: ["pepperoni", "extra cheese"],
+            quantity: 1,
+          },
+        },
+        "Added a large pepperoni with extra cheese.",
+      ],
+    },
+  );
+
+  test(
+    "changes the pizza it added a turn ago, by the id that turn returned",
+    async ({ session }) => {
+      await session.say("Can I get a medium thin crust pizza with mushrooms?");
+      // The id only exists because turn 1 wrote it into the session's cart.
+      // Nothing in this utterance names it, so a slot that did not survive the
+      // turn leaves the model with nothing to address.
+      const turn = await session.say("Actually, make that a large.");
+
+      expect(turn.toolCalls.map((c) => c.name)).toEqual(["update_pizza"]);
+      const call = turn.toolCalls[0]!;
+      expect(call.args).toMatchObject({ pizza_id: 1, size: "large" });
+
+      // And the cart the page renders is the same one: still one pizza, resized
+      // rather than replaced, with the mushrooms from the first turn intact.
+      const view = lastPushedView(turn.events);
+      expect(view?.pizzas).toHaveLength(1);
+      expect(view?.pizzas[0]).toMatchObject({ id: 1, size: "large", toppings: ["mushrooms"] });
+    },
+    {
+      stubReply: [
+        {
+          tool: "add_pizza",
+          args: { size: "medium", crust: "thin", toppings: ["mushrooms"], quantity: 1 },
+        },
+        "A medium thin crust with mushrooms, coming up.",
+        { tool: "update_pizza", args: { pizza_id: 1, size: "large" } },
+        "Made it a large.",
+      ],
+    },
+  );
+
+  test(
+    "never places an empty order",
+    async ({ session }) => {
+      const turn = await session.say("That's everything, go ahead and place my order.");
+
+      // The model may check the cart first, or answer from the fact that
+      // nothing was ordered. What it may NOT do is get an order number: every
+      // place_order call on an empty cart has to come back refused, which is
+      // what stops the agent reading out a confirmation for nothing.
+      for (const call of turn.toolCalls.filter((c) => c.name === "place_order")) {
+        // `toolResultIn` over a ONE-CALL list: the name is this call's own, so
+        // the reader's "no such call" and "two calls" throws are unreachable and
+        // what is left is the parse plus the schema. The schema requires
+        // `error`, which is the claim — a call that SUCCEEDED fails here.
+        expect(toolResultIn([call], call.name, z.object({ error: z.string() })).error).toContain(
+          "Cannot place an empty order",
+        );
+      }
+      expect(turn.completed).toBe(true);
+      // Nothing the session pushed to the page may claim an order was placed.
+      expect(pushedViews(session.events()).filter((view) => view.orderPlaced)).toEqual([]);
+    },
+    { stubReply: [{ tool: "place_order" }, "There's nothing in your order yet."] },
+  );
+});

package/dist/templates/pizza-ordering/agent.test.ts

@@ -26,6 +26,7 @@ const agentDef = withDiscoveredTools(
 
 import {
   calculateTotal,
+  MENU,
   orderProjection,
   orderSlot,
   orderView,
@@ -72,6 +73,20 @@ describe("pricing (shared.ts)", () => {
     expect(pizzaPrice({ id: 1, ...margherita })).toBeCloseTo(11.99, 5);
   });
 
+  test("a topping named the way the MENU PROSE spells it is charged menu price", () => {
+    // `menuText()` writes `extra_cheese` as "extra cheese", so that is what a
+    // model reading the prompt asks for. Before `toppingKey`, the table missed
+    // and charged the $1.00 unknown-topping default for a $1.50 topping —
+    // found live by `agent.eval.test.ts`, which priced the same pizza off MENU
+    // and got $17.99 against the tool's $17.49.
+    const keyed = pizzaPrice({ id: 1, ...margherita, toppings: ["extra_cheese"] });
+    expect(pizzaPrice({ id: 1, ...margherita, toppings: ["extra cheese"] })).toBeCloseTo(keyed, 5);
+    expect(pizzaPrice({ id: 1, ...margherita, toppings: ["Green Peppers"] })).toBeCloseTo(
+      11.99 + MENU.toppings.green_peppers,
+      5,
+    );
+  });
+
   test("unknown topping falls back to the $1.00 default", () => {
     const known = pizzaPrice({ id: 1, ...margherita, toppings: ["onions"] }); // $1.00 topping
     const unknown = pizzaPrice({ id: 1, ...margherita, toppings: ["dragonfruit"] });

package/dist/templates/pizza-ordering/shared.ts

@@ -62,11 +62,30 @@ export function menuText(): string {
   ].join("\n");
 }
 
+/**
+ * A topping name as the price table keys it.
+ *
+ * `menuText()` renders `extra_cheese` as "extra cheese", so the menu the model
+ * reads and the table that charges for it are spelled differently — and the raw
+ * lookup below then missed and charged the $1.00 unknown-topping default for a
+ * $1.50 topping. `agent.eval.test.ts` found it against a live model: a large
+ * pepperoni with extra cheese was quoted at the menu's $17.99 and rung up at
+ * $17.49. Normalizing here keeps ONE spelling authoritative for pricing while
+ * the cart still stores what the caller actually said, which is what the
+ * sidebar and the read-back description show.
+ */
+export function toppingKey(name: string): string {
+  return name
+    .trim()
+    .toLowerCase()
+    .replaceAll(/[\s-]+/g, "_");
+}
+
 export function pizzaPrice(p: ReadonlyPizza): number {
   const base = MENU.sizes[p.size];
   const crust = MENU.crusts[p.crust];
   const toppings = p.toppings.reduce(
-    (sum, t) => sum + (MENU.toppings[t as keyof typeof MENU.toppings] ?? 1.0),
+    (sum, t) => sum + (MENU.toppings[toppingKey(t) as keyof typeof MENU.toppings] ?? 1.0),
     0,
   );
   return (base + crust + toppings) * p.quantity;

package/dist/templates/pizza-ordering/system-prompt.md

@@ -8,6 +8,12 @@ Behavior:
 - When a customer wants a pizza, collect size, crust, and toppings, then use add_pizza to add it.
 - If they just say something like "pepperoni pizza", assume medium, regular crust, and confirm before adding.
 - Always confirm what you added after using add_pizza.
+- NEVER say a pizza was added, changed, or removed unless the matching tool has
+  returned it. Saying "I've added that" with no tool call leaves the customer's
+  cart empty and their order unplaced.
+- NEVER work out a price or a total yourself. Every price you say out loud comes
+  from a tool result — add_pizza, update_pizza, remove_pizza and view_order all
+  return the running total. If you have not called one, you do not have a total.
 - Use view_order when the customer asks to review their order.
 - Use update_pizza if they want to change an existing pizza.
 - Use remove_pizza if they want to remove one.

package/dist/templates/plan-and-execute/agent.eval.test.ts

@@ -0,0 +1,209 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the desk really plan, work one step, and replan? Run it with
+// `aai eval`.
+//
+// `agent.test.ts` drives the three nodes and the four tools directly. What it
+// cannot see is the LOOP as a caller drives it: whether the flow gate stops a
+// step being worked before there is a plan, and whether one turn does exactly
+// one step rather than running the plan to completion down a silent line. Both
+// of those are read off the event stream here.
+//
+// **One thing a SCRIPTED run cannot reach: `ctx.generate`.** The stub model
+// registers as an LLM provider, so `ctx.generate` resolves it — and the fake
+// answers `doStream` only, so any tool that calls a model comes back
+// `{"error":"The planner failed: fake LLM: doGenerate not implemented"}`. That
+// is three of this template's four tools. So the two cases that run without a
+// key are the ones about the GATE, which refuses before the body runs, and
+// everything about the plan itself is `{ live: true }`.
+//
+// **`system-prompt.md` is applied HERE, not by `agent.ts`.** The build discovers
+// the file, so an eval driving the raw default export would run this agent with
+// the FRAMEWORK DEFAULT prompt — and the discipline that prompt imposes is the
+// entire subject of this file. Measured against the default: the model answers
+// product questions from its own knowledge and skips the tools the prompt exists
+// to route it through, so a case run that way measures nothing it claims to.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { type EvalSession, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, so a projection
+ * that stopped matching FAILS naming the field, where the cast this replaced
+ * handed the assertions `undefined` and failed a line later on something else.
+ * It names only the fields asserted below, so `planView` may grow without
+ * touching this.
+ */
+const ProjectedPlan = z.object({
+  objective: z.string().nullable(),
+  plan: z.array(z.string()),
+  done: z.array(z.object({ step: z.string(), result: z.string() })),
+  response: z.string().nullable(),
+  revisions: z.array(z.string()),
+  progress: z.number(),
+});
+
+/**
+ * The latest `syncState` frame — what the browser would be rendering.
+ *
+ * `planProjection` rides out on `state.updated` after every tool call, so this
+ * is how an eval sees the plan at all: the session hands back events, and the
+ * projection is the one of them that carries state. `lastStateIn` is the SDK's
+ * reader for exactly this.
+ */
+const planState = (session: EvalSession) => lastStateIn(session.events(), ProjectedPlan);
+
+const named = (calls: readonly { name: string }[]): string[] => calls.map((call) => call.name);
+
+describeEval(agentDef, (test) => {
+  test(
+    "the stage the desk reports is the flow's, not a guess at the plan",
+    async ({ session }) => {
+      const turn = await session.say("Where do we stand with all this?");
+
+      // `plan_status` is the one tool that reads the machine's own position, and
+      // it is legal in every state. Whatever the model did with the answer, the
+      // desk must not have invented a plan to report.
+      const view = planState(session);
+      expect(view?.objective ?? null).toBeNull();
+      expect(view?.progress ?? 0).toBe(0);
+
+      for (const call of turn.toolCalls.filter((one) => one.name === "plan_status")) {
+        // `stage` comes off `planFlow.position`, and `reads` off `stageLabel` —
+        // deriving either from `!plan.objective` a second time is the drift this
+        // template removed, and it would show up right here.
+        expect(call.result).toMatch(/"stage":"idle"/);
+        expect(call.result).toMatch(/no plan yet/);
+        expect(call.result).toMatch(/start_plan/);
+      }
+    },
+    { stubReply: [{ tool: "plan_status" }, "Nothing on the go yet — what are you trying to do?"] },
+  );
+
+  test(
+    "no step can be worked before a plan exists",
+    async ({ session, mode }) => {
+      const turn = await session.say("Skip the planning — just go and do the next step.");
+
+      const attempts = turn.toolCalls.filter((call) => call.name === "work_next_step");
+      // The script FORCES the call in stub mode, so the gate really runs; a live
+      // model that declines to call it has honoured the same rule a level
+      // earlier, which is why the count is asserted only where it is decided.
+      if (mode === "stub") expect(attempts).toHaveLength(1);
+      for (const attempt of attempts) {
+        // Refused BEFORE the body: `when: "working"` is what replaced the two
+        // hand-rolled guards, and its refusal names the position and quotes the
+        // state's instruction so the model can recover on its own turn.
+        expect(attempt.result).toMatch(/Not available yet/);
+        expect(attempt.result).toMatch(/idle/);
+        expect(attempt.result).toMatch(/start_plan/);
+      }
+      // And whatever else happened, nothing was ANSWERED: a desk that produced
+      // a response without working a step is the failure the flow exists to
+      // prevent.
+      expect(planState(session)?.response ?? null).toBeNull();
+    },
+    {
+      stubReply: [
+        { tool: "work_next_step" },
+        "There's no plan yet — tell me what you're trying to get done.",
+      ],
+    },
+  );
+
+  test(
+    "a step is worked once per call, and a worked step leaves the plan",
+    async ({ session }) => {
+      await session.say(
+        "I want to work out whether it is cheaper to take the train or fly from London to Edinburgh next month.",
+      );
+      const started = session.toolCalls().find((call) => call.name === "start_plan");
+      expect(started, `tools called: ${named(session.toolCalls()).join(", ")}`).toBeDefined();
+      const planned = planState(session);
+      // The tool's own result rides in the message: a planner that FAILED (a
+      // gateway error, a schema the provider would not honour) writes nothing,
+      // and "objective is null" on its own reads as a projection bug.
+      expect(planned?.objective, `start_plan answered: ${started?.result}`).toBeTruthy();
+      expect(planned?.plan.length ?? 0).toBeGreaterThan(0);
+
+      const worked = await session.say("Yes, go ahead and start on it.");
+      const calls = named(worked.toolCalls).filter((name) => name === "work_next_step").length;
+      expect(calls, `tools called: [${named(worked.toolCalls).join(", ")}]`).toBeGreaterThan(0);
+
+      const after = planState(session);
+      const done = after?.done ?? [];
+      expect(done.length).toBeGreaterThan(0);
+      // ONE STEP PER CALL: `work_next_step` is one execute-then-replan turn, not
+      // a loop. More steps recorded than calls made would mean the tool had run
+      // the plan to completion, leaving the caller on a silent line with no gap
+      // to change their mind in — which is the pause the whole design is for.
+      //
+      // Note this is the guarantee the template ENFORCES, and it is weaker than
+      // the one it asks for: "call this once per step, never in a loop" is in the
+      // system prompt AND the tool description, and a live model was measured
+      // calling it twice in one turn anyway. One session has one tool list and
+      // no per-turn hook, so the pause is asked for rather than enforced; what IS
+      // enforced is everything below.
+      expect(done.length).toBeLessThanOrEqual(calls);
+      for (const step of done) expect(step.result).toBeTruthy();
+      expect(after?.progress ?? 0).toBeGreaterThan(0);
+      // The step is CLAIMED off the head of the plan inside the synchronous
+      // update window, so no step can be done twice and a completed step is gone
+      // from what is left — that claim is what makes two concurrent calls safe.
+      const labels = done.map((step) => step.step);
+      expect(new Set(labels).size, `a step was worked twice: ${labels.join(" | ")}`).toBe(
+        labels.length,
+      );
+      for (const label of labels) expect(after?.plan ?? []).not.toContain(label);
+    },
+    // Live only: this step is a real model call and a real web search, which is
+    // the point — a scripted planner is a template that "plans" by imagining
+    // having looked something up.
+    { live: true },
+  );
+
+  test(
+    "when the caller changes their mind it is the replanner that rewrites the plan",
+    async ({ session }) => {
+      await session.say(
+        "I need to sort out getting from London to Edinburgh next month — train or flight, whichever works.",
+      );
+      const before = planState(session);
+      expect(before?.plan.length ?? 0).toBeGreaterThan(0);
+
+      const turn = await session.say("Actually, forget the train entirely — I only want to fly.");
+
+      const revised = turn.toolCalls.find((call) => call.name === "revise_plan");
+      expect(
+        revised,
+        `tools called: ${named(turn.toolCalls).join(", ")} — the caller changed the objective, ` +
+          "so this is `revise_plan`, not a plan rewritten by hand",
+      ).toBeDefined();
+      expect(revised?.args.instruction).toBeTruthy();
+
+      const after = planState(session);
+      // The trail is what the sidebar renders and what a caller cannot hold by
+      // ear: a revision the caller asked for is recorded as theirs.
+      expect(after?.revisions.some((entry) => entry.startsWith("Caller:"))).toBe(true);
+      // A revision reopens the plan rather than answering it, and completed
+      // steps are never redone — there are none here, so the whole plan is new.
+      expect(after?.plan).not.toEqual(before?.plan);
+      expect(after?.done).toHaveLength(0);
+    },
+    { live: true },
+  );
+});