@alexkroman1/aai-cli 6.11.0 → 8.0.0

package/dist/templates/health-assistant/agent.test.ts

@@ -1,7 +1,7 @@
 /// <reference types="vite/client" />
 
-import { isToolFailure, type ToolContext } from "@alexkroman1/aai";
-import { createToolContext, runTool, withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { isToolFailure } from "@alexkroman1/aai";
+import { toolInputIssues, toolRunner, withDiscoveredTools } from "@alexkroman1/aai/testing";
 import { beforeEach, describe, expect, test, vi } from "vitest";
 import authoredAgent from "./agent.ts";
 import { excerptAround, type FdaLabel, toDrugInfo } from "./fda.ts";
@@ -35,8 +35,13 @@ const agentDef = withDiscoveredTools(
   import.meta.glob("./tools/*.ts", { eager: true }),
 );
 
-const run = (name: string, args: Record<string, unknown>, ctx: ToolContext = createToolContext()) =>
-  runTool(agentDef, name, args, ctx);
+/**
+ * Every tool here takes arguments and none of them touches session state, so no
+ * call passes a context: `runTool` builds a fresh one per call, which is a
+ * distinct session — right for a stateless tool, and never what two calls
+ * sharing state want.
+ */
+const run = toolRunner(agentDef);
 
 const IBUPROFEN: FdaLabel = {
   openfda: { generic_name: ["IBUPROFEN"], brand_name: ["Advil"], manufacturer_name: ["Acme"] },
@@ -150,6 +155,19 @@ describe("check_drug_interaction", () => {
     expect(isToolFailure(result) && result.error).toContain("at least two");
     expect(label).not.toHaveBeenCalled();
   });
+
+  test("the schema itself accepts those names, which is why the body re-checks", async () => {
+    // The other half of the claim above, asked of the schema directly rather
+    // than through `~standard`: `min(2)` counts ENTRIES and `min(1)` counts
+    // CHARACTERS, so `"   "` is a valid entry and the refusal is the body's.
+    expect(
+      await toolInputIssues(agentDef, "check_drug_interaction", { drugs: ["ibuprofen", "   "] }),
+    ).toBeUndefined();
+    // And the schema is still doing its own half — one drug is not a check.
+    expect(
+      await toolInputIssues(agentDef, "check_drug_interaction", { drugs: ["ibuprofen"] }),
+    ).toBeDefined();
+  });
 });
 
 describe("fda.ts helpers", () => {

package/dist/templates/infocom-adventure/agent.eval.test.ts

@@ -0,0 +1,158 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the game engine actually keep its world? Run it with `aai eval`.
+//
+// `agent.test.ts` drives the tools directly, one call at a time, against a
+// context it made itself. This drives the AGENT: a real session, the real tool
+// executor, the real event stream, and — the part no unit test can reach — MORE
+// THAN ONE TURN through the same session slot. Everything worth asserting here
+// is a claim about state that has to survive a turn boundary, or about the world
+// really being replaced when the player asks to start over.
+//
+// Two modes, and `describeEval` announces which it picked:
+//
+//   * with ASSEMBLYAI_API_KEY — a LIVE model. It really has to map "I pick up
+//     the rusted lantern" onto `game_state_take`, which is the behaviour the
+//     system prompt spends a whole section on.
+//   * without one — a SCRIPTED model, whose tool calls REALLY EXECUTE. The
+//     state changes for real, so a scripted take followed by a scripted read is
+//     a genuine two-turn state test with no model involved.
+//
+// What no eval here can see: anything below the audio boundary — endpointing,
+// barge-in, two commands merging into one turn. Those need real paced audio.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { type EvalTurn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import { DEFAULT_GAME_STATE } from "./shared.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs, assembled the way the build assembles it: the
+ * authored export, plus what `tools/` declares, plus `system-prompt.md`.
+ *
+ * Both wrappers matter here in a way they do not in `agent.test.ts`. A tool
+ * missing from the registry is a tool the model cannot call, and the prompt is
+ * the only thing that tells a live model that "grab the rope" means
+ * `game_state_take` — an eval run against the framework default prompt measures
+ * an agent nobody deployed.
+ */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * What each of the three tools this file drives answers, off the wire.
+ *
+ * `tool.completed` carries a tool result as a JSON STRING, so a case either
+ * casts it or validates it — and a cast is silent exactly when a tool's result
+ * changed shape underneath the eval, which is the regression an eval exists to
+ * catch. `toolResultIn` takes a schema for that reason, and these name only the
+ * fields asserted below.
+ */
+const Carried = z.object({ inventory: z.array(z.string()) });
+const Restarted = z.object({ restarted: z.boolean() });
+const Status = z.object({
+  inventory: z.array(z.string()),
+  score: z.number(),
+  moves: z.number(),
+  currentRoom: z.string(),
+});
+
+/**
+ * What the ONE call to `name` answered on this turn.
+ *
+ * `toolResultIn` is the SDK's reader: it throws rather than returning
+ * undefined, and names what the agent called instead — "it called a different
+ * tool" is the finding, and a case that read `undefined` off a missing call
+ * would assert against nothing. A turn that called it TWICE is refused too,
+ * rather than silently answered with the first.
+ */
+const answerOf = <T>(turn: EvalTurn, name: string, schema: z.ZodType<T>): T =>
+  toolResultIn(turn.toolCalls, name, schema);
+
+describeEval(agentDef, (test) => {
+  test(
+    "an item taken on one turn is still carried on the next",
+    async ({ session }) => {
+      const taken = await session.say("I pick up the rusted lantern.");
+      // The write really wrote. `game_state_take` is a `gameSlot.updateTool`,
+      // and it shipped once as the READING half — pushing to a deep-frozen
+      // array, which throws on every call. A tool that threw answers with an
+      // error here, not an inventory.
+      expect(answerOf(taken, "game_state_take", Carried).inventory).toContain("rusted lantern");
+
+      // A SECOND turn, which is the whole point: the slot is keyed per session,
+      // so this is the only place the take can be shown to have outlived the
+      // turn that made it.
+      const status = await session.say("Check the game state. What am I carrying?");
+      expect(answerOf(status, "game_state_get", Status).inventory).toContain("rusted lantern");
+      expect(status.completed).toBe(true);
+    },
+    {
+      stubReply: [
+        { tool: "game_state_take", args: { value: "rusted lantern" } },
+        "You lift the rusted lantern from its iron hook. It is heavier than it looks.",
+        { tool: "game_state_get" },
+        "You are carrying the rusted lantern, and nothing else.",
+      ],
+    },
+  );
+
+  test(
+    "starting over really empties the world",
+    async ({ session }) => {
+      const taken = await session.say("I pick up the rusted lantern.");
+      expect(answerOf(taken, "game_state_take", Carried).inventory).toContain("rusted lantern");
+
+      const again = await session.say("Forget all that. Start a new game from the beginning.");
+      expect(answerOf(again, "game_state_restart", Restarted).restarted).toBe(true);
+
+      // `game_state_restart` is the one tool that REPLACES the slot's value
+      // (`gameSlot.reset`), so what has to be checked is the state a LATER turn
+      // reads — a reset that only rebuilt the value it returned would satisfy
+      // the assertion above and leave the lantern in the player's hands.
+      const status = await session.say("Check the game state. What am I carrying, and where am I?");
+      const fresh = answerOf(status, "game_state_get", Status);
+      expect(fresh.inventory).toEqual([]);
+      expect(fresh.score).toBe(0);
+      expect(fresh.currentRoom).toBe(DEFAULT_GAME_STATE.currentRoom);
+      // ONE, not zero — and this is the assertion that proves the turn counter
+      // is the framework's. The reset emptied it, then the player said the line
+      // above, and the `user-transcript.committed` hook counted it before the
+      // narrator took its turn. No tool call is involved anywhere in that.
+      expect(fresh.moves).toBe(1);
+    },
+    {
+      stubReply: [
+        { tool: "game_state_take", args: { value: "rusted lantern" } },
+        "You lift the rusted lantern from its iron hook.",
+        { tool: "game_state_restart" },
+        "Very well. We begin again at the mouth of the cave.",
+        { tool: "game_state_get" },
+        "You carry nothing. Your score is zero, and you stand at the cave mouth.",
+      ],
+    },
+  );
+
+  test(
+    "a restart is narrated, not merely recorded",
+    async ({ session }) => {
+      // LIVE only: the claim is about what the narrator SAYS after the reset,
+      // and a scripted reply is a line this file wrote. The prompt asks for the
+      // opening scene again — a restart that answers "done" leaves a voice
+      // player with no idea where they are.
+      const again = await session.say("Start over. New game, please.");
+
+      expect(answerOf(again, "game_state_restart", Restarted).restarted).toBe(true);
+      expect(again.text).toMatch(/cave|cavern|forest|entrance|lantern/i);
+      expect(again.completed).toBe(true);
+    },
+    { live: true },
+  );
+});

package/dist/templates/infocom-adventure/agent.test.ts

@@ -1,7 +1,6 @@
 /// <reference types="vite/client" />
 
-import type { ToolContext } from "@alexkroman1/aai";
-import { createToolContext, runTool, withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { createToolContext, toolRunner, withDiscoveredTools } from "@alexkroman1/aai/testing";
 import { describe, expect, test } from "vitest";
 import authoredAgent from "./agent.ts";
 
@@ -21,8 +20,22 @@ const agentDef = withDiscoveredTools(
 import { DEFAULT_GAME_STATE, gameSlot, MAX_HISTORY, REPORTED_HISTORY } from "./shared.ts";
 
 /** A tool by the name the model calls it by, bound to this agent. */
-const run = (name: string, args: Record<string, unknown>, ctx: ToolContext) =>
-  runTool(agentDef, name, args, ctx);
+const run = toolRunner(agentDef);
+
+/**
+ * What the player said, delivered the way the RUNTIME delivers it.
+ *
+ * A session event handler is a plain function on the def, so a template can
+ * drive one with no harness — which is the point of asserting on it here rather
+ * than trusting the wiring: `moves` and `history` are now maintained by
+ * something the model never calls, so nothing else in this file would notice if
+ * the hook stopped running.
+ */
+const say = (text: string, ctx: ReturnType<typeof makeCtx>) =>
+  agentDef.events?.["user-transcript.committed"]?.(
+    { type: "user-transcript.committed", text, meta: { id: "evt_1", at: 0 } },
+    ctx,
+  );
 
 /** Each context owns its OWN slot store, which is what makes two playthroughs
  *  independent by construction. */
@@ -99,13 +112,15 @@ describe("the adventure's tools", () => {
     expect(nothing.inventory).toEqual(["rope"]);
   });
 
-  test("move sets the room and counts the move", async () => {
+  test("move sets the room and reports the turn count without touching it", async () => {
     const ctx = makeCtx();
     const moved = (await run("game_state_move", { value: "Echo Chamber" }, ctx)) as {
       currentRoom: string;
       moves: number;
     };
-    expect(moved).toEqual({ currentRoom: "Echo Chamber", moves: 1 });
+    // `moves` is 0 because nobody has SAID anything — see `recordTurn`. It is
+    // still reported, because it is what the narrator wants back.
+    expect(moved).toEqual({ currentRoom: "Echo Chamber", moves: 0 });
     expect(gameSlot.get(ctx).currentRoom).toBe("Echo Chamber");
   });
 
@@ -116,26 +131,42 @@ describe("the adventure's tools", () => {
     expect(total.score).toBe(15);
   });
 
-  test("history logs the command, counts the move, and reports only the recent ones", async () => {
+  test("what the player SAYS logs the command and counts the turn", async () => {
     const ctx = makeCtx();
-    for (let i = 1; i <= REPORTED_HISTORY + 2; i++) {
-      await run("game_state_history", { value: `command ${i}` }, ctx);
-    }
+    for (let i = 1; i <= REPORTED_HISTORY + 2; i++) say(`command ${i}`, ctx);
+    say("look", ctx);
 
-    const last = (await run("game_state_history", { value: "look" }, ctx)) as {
+    const game = gameSlot.get(ctx);
+    expect(game.moves).toBe(REPORTED_HISTORY + 3);
+    expect(game.history.at(-1)).toBe("look");
+
+    // And the narrator reads it back through the ordinary state tool — the hook
+    // writes, the model reads, and the two never have to agree about who counts.
+    const read = (await run("game_state_get", {}, ctx)) as {
       moves: number;
       recentHistory: string[];
     };
-    expect(last.moves).toBe(REPORTED_HISTORY + 3);
-    expect(last.recentHistory).toHaveLength(REPORTED_HISTORY);
-    expect(last.recentHistory.at(-1)).toBe("look");
+    expect(read.moves).toBe(REPORTED_HISTORY + 3);
+    expect(read.recentHistory).toHaveLength(REPORTED_HISTORY);
   });
 
-  test("the history is capped, so a long playthrough does not grow without bound", async () => {
+  test("a turn is counted once, even when the narrator also moves the player", async () => {
     const ctx = makeCtx();
-    for (let i = 0; i < MAX_HISTORY + 10; i++) {
-      await run("game_state_history", { value: `command ${i}` }, ctx);
-    }
+    say("go north", ctx);
+    await run("game_state_move", { value: "Echo Chamber" }, ctx);
+
+    // Both used to bump `moves`, so this turn scored 2 — and a turn where the
+    // narrator called neither scored 0. A move is a room change; a turn is the
+    // player saying something.
+    const game = gameSlot.get(ctx);
+    expect(game.moves).toBe(1);
+    expect(game.currentRoom).toBe("Echo Chamber");
+  });
+
+  test("the history is capped, so a long playthrough does not grow without bound", () => {
+    const ctx = makeCtx();
+    for (let i = 0; i < MAX_HISTORY + 10; i++) say(`command ${i}`, ctx);
+
     const game = gameSlot.get(ctx);
     expect(game.history).toHaveLength(MAX_HISTORY);
     // The cap drops the OLDEST — the newest command is the one a narrator needs.
@@ -149,15 +180,13 @@ describe("the adventure's tools", () => {
     await run("game_state_flag", { value: "gate_opened" }, ctx);
     await run("game_state_move", { value: "Echo Chamber" }, ctx);
     await run("game_state_score", { value: 7 }, ctx);
-    for (let i = 0; i < REPORTED_HISTORY + 3; i++) {
-      await run("game_state_history", { value: `command ${i}` }, ctx);
-    }
+    for (let i = 0; i < REPORTED_HISTORY + 3; i++) say(`command ${i}`, ctx);
 
-    expect(await run("game_state_get", {}, ctx)).toEqual({
+    expect(await run("game_state_get", ctx)).toEqual({
       currentRoom: "Echo Chamber",
       inventory: ["lantern"],
       score: 7,
-      moves: REPORTED_HISTORY + 4,
+      moves: REPORTED_HISTORY + 3,
       flags: { gate_opened: true },
       recentHistory: Array.from({ length: REPORTED_HISTORY }, (_, i) => `command ${i + 3}`),
     });
@@ -169,7 +198,7 @@ describe("the adventure's tools", () => {
     await run("game_state_score", { value: 30 }, ctx);
     await run("game_state_move", { value: "Echo Chamber" }, ctx);
 
-    const restarted = (await run("game_state_restart", {}, ctx)) as {
+    const restarted = (await run("game_state_restart", ctx)) as {
       restarted: boolean;
       currentRoom: string;
     };

package/dist/templates/infocom-adventure/agent.ts

@@ -1,4 +1,5 @@
 import { agent } from "@alexkroman1/aai";
+import { gameSlot, recordTurn } from "./shared.ts";
 
 export default agent({
   name: "Cavern Adventure",
@@ -11,4 +12,24 @@ export default agent({
   // (shared.ts) and the world map in system-prompt.md.
   greeting:
     "Welcome, adventurer. You are standing at the mouth of a weathered cave at the edge of a pine forest. A cold wind carries the smell of damp stone up from the darkness below. A rusted lantern hangs from an iron hook beside the entrance. What would you like to do?",
+  /**
+   * The turn counter and the command log are the FRAMEWORK's, not the model's.
+   *
+   * Both used to be a `game_state_history` tool the system prompt told the
+   * narrator to call on every turn, handing back the player's own words — which
+   * the runtime already had. A hook is strictly better on all three counts a
+   * template is meant to teach: it costs no model call, it cannot be forgotten,
+   * and it needs no prose in the prompt to enforce it.
+   *
+   * `.committed` rather than `.updated`: partials arrive several times per
+   * utterance and would count one sentence as a dozen turns.
+   *
+   * It writes and does not speak, which is the whole line a session event hook
+   * draws — nothing here can decide what the narrator says next. The narrator
+   * reads the result on its next `game_state_get`.
+   */
+  events: {
+    "user-transcript.committed": (event, ctx) =>
+      gameSlot.update(ctx, (game) => recordTurn(game, event.text)),
+  },
 });