@alexkroman1/aai-cli 6.11.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/README.md +1 -0
  2. package/dist/{_agent-DpH2pBJd.mjs → _agent-BTwZJu4P.mjs} +2 -2
  3. package/dist/{_bundler-C2BDwU7r.mjs → _bundler-CDuYl5Gb.mjs} +2 -2
  4. package/dist/_dev-env.d.ts +22 -2
  5. package/dist/{_dev-server-LLLWnFBs.mjs → _dev-server-Me06wgeF.mjs} +38 -47
  6. package/dist/_dev-server.d.ts +0 -17
  7. package/dist/{_init-C57bWAQ8.mjs → _init-BhEfQ6Yi.mjs} +3 -2
  8. package/dist/{_preflight-BtfaYtbE.mjs → _preflight-Co1G9Jww.mjs} +1 -1
  9. package/dist/_preflight.d.ts +1 -1
  10. package/dist/_resource-commands.d.ts +23 -0
  11. package/dist/{_slug-api-BxrsJXVI.mjs → _slug-api-z-USxPtL.mjs} +16 -12
  12. package/dist/_slug-api.d.ts +5 -2
  13. package/dist/{_studio-U3jhKlA3.mjs → _studio-DR5yzZ1T.mjs} +2 -1
  14. package/dist/{_templates-Dxb_P2Wz.mjs → _templates-4WcKOjS5.mjs} +5 -6
  15. package/dist/_templates.d.ts +3 -4
  16. package/dist/_workflow-scan.d.ts +108 -0
  17. package/dist/{build-Dfza2pRI.mjs → build-BBxf8vom.mjs} +6 -3
  18. package/dist/cli.mjs +346 -349
  19. package/dist/{client-bundler-BEIqgOtd.mjs → client-bundler-CWnG42cU.mjs} +2 -1
  20. package/dist/client-bundler.mjs +1 -1
  21. package/dist/{delete-vbpjGzqs.mjs → delete-Dl_LSfvI.mjs} +3 -3
  22. package/dist/{deploy-B8lubiRT.mjs → deploy-DRC8jWr8.mjs} +5 -5
  23. package/dist/{dev-Uq5ujP8a.mjs → dev-BvcctCn2.mjs} +1 -1
  24. package/dist/eval-DfQ8cm5p.mjs +56 -0
  25. package/dist/eval.d.ts +43 -0
  26. package/dist/{init-l6xfU4xX.mjs → init-CUo87gcI.mjs} +3 -3
  27. package/dist/{login-C71-qz8F.mjs → login-BBuM1sxH.mjs} +3 -4
  28. package/dist/{logs-BWVGSjLR.mjs → logs-CQ2T46dr.mjs} +2 -2
  29. package/dist/project-config.mjs +1 -1
  30. package/dist/scaffold/CLAUDE.md +442 -80
  31. package/dist/scaffold/package.json +6 -3
  32. package/dist/scaffold/server.mjs +11 -4
  33. package/dist/scaffold/tsconfig.json +0 -1
  34. package/dist/{secret-BuMuFR4B.mjs → secret-Ct34kdgU.mjs} +2 -2
  35. package/dist/{storage-DzRZ-eCw.mjs → storage-i_79ZwAR.mjs} +2 -2
  36. package/dist/{studio-BCNUpDgP.mjs → studio-CzTzPqe1.mjs} +4 -4
  37. package/dist/templates/briefing-desk/agent.test.ts +350 -0
  38. package/dist/templates/briefing-desk/agent.ts +30 -0
  39. package/dist/templates/briefing-desk/shared.ts +200 -0
  40. package/dist/templates/briefing-desk/system-prompt.md +39 -0
  41. package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
  42. package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
  43. package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
  44. package/dist/templates/call-audit/agent.eval.test.ts +474 -0
  45. package/dist/templates/call-audit/agent.test.ts +57 -204
  46. package/dist/templates/call-audit/agent.ts +32 -19
  47. package/dist/templates/call-audit/client.tsx +17 -54
  48. package/dist/templates/call-audit/workflows/audit.ts +11 -18
  49. package/dist/templates/call-audit/workflows/ingest.ts +106 -114
  50. package/dist/templates/call-audit/workflows/media.ts +2 -12
  51. package/dist/templates/call-audit/workflows/summarize.ts +47 -52
  52. package/dist/templates/call-audit/workflows/sync-api.ts +7 -7
  53. package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
  54. package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
  55. package/dist/templates/dispatch-center/agent.test.ts +149 -25
  56. package/dist/templates/dispatch-center/client.tsx +239 -129
  57. package/dist/templates/dispatch-center/shared.ts +99 -1
  58. package/dist/templates/dispatch-center/system-prompt.md +3 -1
  59. package/dist/templates/dispatch-center/tools/incident_add_note.ts +16 -13
  60. package/dist/templates/dispatch-center/tools/incident_create.ts +19 -1
  61. package/dist/templates/dispatch-center/tools/incident_escalate.ts +68 -59
  62. package/dist/templates/dispatch-center/tools/incident_triage.ts +57 -43
  63. package/dist/templates/dispatch-center/tools/incident_update_status.ts +49 -40
  64. package/dist/templates/dispatch-center/tools/ops_run_scenario.ts +8 -2
  65. package/dist/templates/dispatch-center/tools/resources_dispatch.ts +69 -52
  66. package/dist/templates/dispatch-center/tools/resources_update_status.ts +45 -41
  67. package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
  68. package/dist/templates/embedded-assets/agent.test.ts +17 -14
  69. package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
  70. package/dist/templates/health-assistant/agent.test.ts +22 -4
  71. package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
  72. package/dist/templates/infocom-adventure/agent.test.ts +53 -24
  73. package/dist/templates/infocom-adventure/agent.ts +21 -0
  74. package/dist/templates/infocom-adventure/client.tsx +246 -164
  75. package/dist/templates/infocom-adventure/shared.ts +20 -9
  76. package/dist/templates/infocom-adventure/system-prompt.md +1 -1
  77. package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
  78. package/dist/templates/link-digest/agent.eval.test.ts +195 -0
  79. package/dist/templates/link-digest/agent.test.ts +24 -19
  80. package/dist/templates/link-digest/client.tsx +47 -61
  81. package/dist/templates/link-digest/workflows/digest.ts +19 -31
  82. package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
  83. package/dist/templates/night-owl/agent.eval.test.ts +272 -0
  84. package/dist/templates/night-owl/agent.test.ts +70 -19
  85. package/dist/templates/night-owl/agent.ts +5 -0
  86. package/dist/templates/night-owl/client.tsx +56 -56
  87. package/dist/templates/night-owl/shared.ts +24 -0
  88. package/dist/templates/night-owl/tools/recommend.ts +18 -6
  89. package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
  90. package/dist/templates/personal-finance/system-prompt.md +4 -0
  91. package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
  92. package/dist/templates/pipeline-simple/agent.ts +2 -2
  93. package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
  94. package/dist/templates/pizza-ordering/agent.test.ts +42 -27
  95. package/dist/templates/pizza-ordering/client.tsx +9 -26
  96. package/dist/templates/pizza-ordering/shared.ts +20 -1
  97. package/dist/templates/pizza-ordering/system-prompt.md +6 -0
  98. package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
  99. package/dist/templates/plan-and-execute/agent.test.ts +99 -58
  100. package/dist/templates/plan-and-execute/agent.ts +1 -1
  101. package/dist/templates/plan-and-execute/client.tsx +12 -15
  102. package/dist/templates/plan-and-execute/shared.ts +71 -2
  103. package/dist/templates/plan-and-execute/tools/plan_status.ts +18 -6
  104. package/dist/templates/plan-and-execute/tools/revise_plan.ts +18 -10
  105. package/dist/templates/plan-and-execute/tools/start_plan.ts +13 -2
  106. package/dist/templates/plan-and-execute/tools/work_next_step.ts +70 -22
  107. package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
  108. package/dist/templates/podcast-digest/agent.test.ts +717 -0
  109. package/dist/templates/podcast-digest/agent.ts +139 -0
  110. package/dist/templates/podcast-digest/client.tsx +154 -0
  111. package/dist/templates/podcast-digest/workflows/digest.ts +444 -0
  112. package/dist/templates/podcast-digest/workflows/feeds.ts +507 -0
  113. package/dist/templates/podcast-digest/workflows/slack.ts +88 -0
  114. package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
  115. package/dist/templates/recap-workflow/agent.test.ts +97 -94
  116. package/dist/templates/recap-workflow/tools/recap_progress.ts +9 -12
  117. package/dist/templates/recap-workflow/tools/recap_status.ts +8 -9
  118. package/dist/templates/recap-workflow/tools/request_recap.ts +2 -1
  119. package/dist/templates/recap-workflow/workflows/recap.ts +36 -40
  120. package/dist/templates/redline/agent.eval.test.ts +209 -0
  121. package/dist/templates/redline/agent.test.ts +17 -15
  122. package/dist/templates/redline/client.tsx +12 -12
  123. package/dist/templates/redline/workflows/redline.ts +19 -31
  124. package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
  125. package/dist/templates/research-workflow/agent.test.ts +60 -59
  126. package/dist/templates/research-workflow/tools/research_progress.ts +9 -12
  127. package/dist/templates/research-workflow/tools/research_status.ts +9 -11
  128. package/dist/templates/research-workflow/workflows/research.ts +44 -61
  129. package/dist/templates/retail/address.ts +87 -1
  130. package/dist/templates/retail/agent.eval.test.ts +314 -0
  131. package/dist/templates/retail/agent.test.ts +195 -112
  132. package/dist/templates/retail/agent.ts +35 -1
  133. package/dist/templates/retail/cancel.ts +116 -0
  134. package/dist/templates/retail/client.tsx +265 -117
  135. package/dist/templates/retail/payment.ts +107 -0
  136. package/dist/templates/retail/pending.test.ts +196 -0
  137. package/dist/templates/retail/pending.ts +137 -0
  138. package/dist/templates/retail/registry.test.ts +150 -7
  139. package/dist/templates/retail/returns.ts +102 -0
  140. package/dist/templates/retail/shared.test.ts +42 -0
  141. package/dist/templates/retail/shared.ts +35 -0
  142. package/dist/templates/retail/store.test.ts +84 -15
  143. package/dist/templates/retail/store.ts +231 -48
  144. package/dist/templates/retail/swap.test.ts +4 -4
  145. package/dist/templates/retail/swap.ts +248 -15
  146. package/dist/templates/retail/system-prompt.md +69 -28
  147. package/dist/templates/retail/tools/cancel_change.ts +33 -0
  148. package/dist/templates/retail/tools/cancel_pending_order.ts +16 -60
  149. package/dist/templates/retail/tools/confirm_change.ts +42 -0
  150. package/dist/templates/retail/tools/exchange_delivered_order_items.ts +22 -67
  151. package/dist/templates/retail/tools/find_user_id_by_email.ts +8 -5
  152. package/dist/templates/retail/tools/find_user_id_by_name_zip.ts +4 -5
  153. package/dist/templates/retail/tools/get_item_details.ts +3 -4
  154. package/dist/templates/retail/tools/get_order_details.ts +3 -4
  155. package/dist/templates/retail/tools/get_product_details.ts +3 -4
  156. package/dist/templates/retail/tools/get_user_details.ts +2 -2
  157. package/dist/templates/retail/tools/list_all_product_types.ts +5 -5
  158. package/dist/templates/retail/tools/modify_pending_order_address.ts +13 -25
  159. package/dist/templates/retail/tools/modify_pending_order_items.ts +23 -62
  160. package/dist/templates/retail/tools/modify_pending_order_payment.ts +16 -78
  161. package/dist/templates/retail/tools/modify_user_address.ts +13 -16
  162. package/dist/templates/retail/tools/return_delivered_order_items.ts +17 -72
  163. package/dist/templates/retail/tools/transfer_to_human_agents.ts +12 -6
  164. package/dist/templates/simple/agent.eval.test.ts +54 -0
  165. package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
  166. package/dist/templates/solo-rpg/agent.test.ts +227 -48
  167. package/dist/templates/solo-rpg/client.tsx +38 -37
  168. package/dist/templates/solo-rpg/shared.ts +145 -19
  169. package/dist/templates/solo-rpg/system-prompt.md +3 -2
  170. package/dist/templates/solo-rpg/tools/action_roll.ts +82 -63
  171. package/dist/templates/solo-rpg/tools/burn_momentum.ts +85 -50
  172. package/dist/templates/solo-rpg/tools/check_state.ts +24 -7
  173. package/dist/templates/solo-rpg/tools/load_game.ts +13 -1
  174. package/dist/templates/solo-rpg/tools/save_game.ts +16 -7
  175. package/dist/templates/solo-rpg/tools/setup_character.ts +22 -2
  176. package/dist/templates/solo-rpg/tools/update_state.ts +113 -100
  177. package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
  178. package/dist/templates/spoken-summary/agent.test.ts +68 -97
  179. package/dist/templates/spoken-summary/agent.ts +30 -17
  180. package/dist/templates/spoken-summary/client.tsx +10 -50
  181. package/dist/templates/spoken-summary/workflows/summarize.ts +17 -22
  182. package/dist/templates/spoken-summary/workflows/transcribe.ts +19 -26
  183. package/dist/templates/support-line/agent.eval.test.ts +273 -0
  184. package/dist/templates/support-line/agent.test.ts +11 -16
  185. package/dist/templates/support-line/agent.ts +1 -1
  186. package/dist/templates/support-line/client.tsx +9 -9
  187. package/dist/templates/support-line/nodes.ts +100 -0
  188. package/dist/templates/support-line/procedure.ts +407 -0
  189. package/dist/templates/support-line/system-prompt.md +2 -1
  190. package/dist/templates/support-line/tools/answer_question.ts +66 -4
  191. package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
  192. package/dist/templates/transcription-workflow/agent.test.ts +94 -194
  193. package/dist/templates/transcription-workflow/agent.ts +1 -1
  194. package/dist/templates/transcription-workflow/client.tsx +17 -42
  195. package/dist/templates/transcription-workflow/workflows/batch.ts +19 -22
  196. package/dist/templates/transcription-workflow/workflows/normalize.ts +90 -186
  197. package/dist/templates/transcription-workflow/workflows/stitch.ts +0 -11
  198. package/dist/templates/transcription-workflow/workflows/stream.ts +4 -5
  199. package/dist/templates/transcription-workflow/workflows/sync-api.ts +6 -6
  200. package/dist/templates/transcription-workflow/workflows/transcribe.ts +32 -26
  201. package/dist/templates/transcription-workflow/workflows/wav.ts +0 -31
  202. package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
  203. package/dist/templates/travel-concierge/agent.test.ts +174 -43
  204. package/dist/templates/travel-concierge/agent.ts +18 -9
  205. package/dist/templates/travel-concierge/client.tsx +11 -23
  206. package/dist/templates/travel-concierge/routing.ts +34 -15
  207. package/dist/templates/travel-concierge/shared.ts +180 -12
  208. package/dist/templates/travel-concierge/system-prompt.md +7 -1
  209. package/dist/templates/travel-concierge/tools/book_car_rental.ts +8 -5
  210. package/dist/templates/travel-concierge/tools/book_excursion.ts +8 -5
  211. package/dist/templates/travel-concierge/tools/book_hotel.ts +8 -5
  212. package/dist/templates/travel-concierge/tools/cancel_ticket.ts +8 -5
  213. package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
  214. package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
  215. package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
  216. package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
  217. package/dist/templates/travel-concierge/tools/update_ticket.ts +8 -5
  218. package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
  219. package/dist/templates/web-researcher/system-prompt.md +15 -4
  220. package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
  221. package/dist/test.d.ts +29 -5
  222. package/dist/{worker-bundler-CGD4r8Kc.mjs → worker-bundler-CCVEDjm1.mjs} +241 -3
  223. package/dist/worker-bundler.mjs +1 -1
  224. package/dist/{workflow-CFpxOFfQ.mjs → workflow-BflATMmU.mjs} +47 -12
  225. package/dist/workflow-bundler.d.ts +6 -1
  226. package/dist/workflow.d.ts +1 -1
  227. package/package.json +5 -4
  228. package/dist/eject-31gjtaHF.mjs +0 -116
  229. package/dist/eject.d.ts +0 -60
  230. package/dist/templates/call-audit/workflows/temp-media.ts +0 -138
  231. package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
  232. package/dist/templates/support-line/graph.ts +0 -224
  233. package/dist/{_config-D_s09e7g.mjs → _config-CmJOFsAP.mjs} +1 -1
  234. /package/dist/templates/plan-and-execute/{graph.ts → procedure.ts} +0 -0
@@ -0,0 +1,146 @@
1
+ // An EVAL: does Math Buddy delegate every calculation to code? Run it with
2
+ // `aai eval`.
3
+ //
4
+ // This tutor's whole design is "the model does the talking, run_code does the
5
+ // arithmetic" — which is also why it runs on Flash-Lite. So the claim worth
6
+ // pinning is not that the answer is right, it is that the answer came from
7
+ // CODE, and that the code is the recipe the prompt gave.
8
+ //
9
+ // Two things this file has to work around, both explained at length in
10
+ // `../code-interpreter/agent.eval.test.ts`: `system-prompt.md` is discovered by
11
+ // the build rather than imported, so an eval has to apply it or it measures a
12
+ // different agent; and `run_code` refuses unless the EVAL supplies an executor,
13
+ // which this suite does — so the cases below assert the answer the code came
14
+ // back with as well as the code the tutor wrote.
15
+ import { runInNewContext } from "node:vm";
16
+ import { withSystemPrompt } from "@alexkroman1/aai/manifest";
17
+ import { errorMessage } from "@alexkroman1/aai/utils";
18
+ import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
19
+ import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
20
+ import { expect } from "vitest";
21
+ import authored from "./agent.ts";
22
+ import systemPrompt from "./system-prompt.md?raw";
23
+
24
+ const agentDef = withSystemPrompt(authored, systemPrompt);
25
+
26
+ /** The code every `run_code` call in this turn carried, joined. */
27
+ const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
28
+ turn.toolCalls
29
+ .filter((c) => c.name === "run_code")
30
+ .map((c) => String(c.args.code ?? ""))
31
+ .join("\n");
32
+
33
+ /**
34
+ * A `run_code` executor, so these cases can assert the ANSWER.
35
+ *
36
+ * The builtin refuses without one — the Modal container is the security
37
+ * boundary, and off-platform there is none — so a case could assert the CALL and
38
+ * the code it carried, and never what the code came back with. A `node:vm`
39
+ * context with a capturing `console.log` is what a developer would reach for on
40
+ * their own machine, and it is enough here: what runs is arithmetic, not a
41
+ * program. It is NOT a sandbox and does not pretend to be one; a deployed agent
42
+ * still gets the refusal.
43
+ *
44
+ * A template eval imports from `@alexkroman1/aai-runtime/eval` and
45
+ * `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
46
+ * node-reaching module graph into this project's TypeScript program, which is
47
+ * three errors in runtime files no eval ever calls. `RunCodeExecutor` is
48
+ * re-exported from `/eval` for exactly that reason.
49
+ */
50
+ const runCode: RunCodeExecutor = async (code) => {
51
+ const lines: string[] = [];
52
+ const log = (...args: unknown[]): void => {
53
+ lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
54
+ };
55
+ try {
56
+ runInNewContext(code, { console: { log } }, { timeout: 1000 });
57
+ } catch (err) {
58
+ return { error: errorMessage(err) };
59
+ }
60
+ return lines.join("\n");
61
+ };
62
+
63
+ /** What every `run_code` call in this turn PRINTED, joined. */
64
+ const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
65
+ turn.toolCalls
66
+ .filter((c) => c.name === "run_code")
67
+ .map((c) => c.result ?? "")
68
+ .join("\n");
69
+
70
+ describeEval(
71
+ agentDef,
72
+ (test) => {
73
+ test(
74
+ "converts units in code, with a real conversion factor",
75
+ async ({ session }) => {
76
+ const turn = await session.say("Convert 5 miles to kilometres.");
77
+
78
+ // The prompt hands the tutor the factors; the finding it guards against
79
+ // is a tutor that recites a remembered figure instead. A factor in the
80
+ // code is the evidence that the conversion was computed, not recalled.
81
+ expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
82
+ const code = codeIn(turn);
83
+ expect(code).toContain("5");
84
+ expect(code).toMatch(/1\.60|1\.61|0\.621|8\.04/);
85
+ // And the factor was applied rather than merely mentioned: five miles is
86
+ // 8.0467 km, so whatever rounding the tutor chose the answer starts 8.0.
87
+ const output = outputIn(turn);
88
+ expect(output, `run_code printed: ${output}`).toMatch(/8\.0/);
89
+ },
90
+ { live: true },
91
+ );
92
+
93
+ test(
94
+ "rolls dice with a random draw rather than inventing numbers",
95
+ async ({ session }) => {
96
+ const turn = await session.say("Roll 3 twenty-sided dice for me.");
97
+
98
+ // A model asked for dice will happily make three numbers up, and the
99
+ // reply is indistinguishable from a real roll. `Math.random` in the code
100
+ // is the only thing that tells them apart.
101
+ expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
102
+ const code = codeIn(turn);
103
+ expect(code).toMatch(/Math\.random/);
104
+ expect(code).toContain("20");
105
+ expect(code).toContain("3");
106
+
107
+ // And the draw really happened: three integers, every one of them a legal
108
+ // face of a twenty-sided die. `Math.random` in the code says the tutor
109
+ // asked for a roll; this says it GOT one — a `run_code` that refused
110
+ // prints a sentence with no dice in it at all.
111
+ const output = outputIn(turn);
112
+ const rolled = [...output.matchAll(/\d+/g)].map((m) => Number(m[0]));
113
+ expect(rolled.length, `run_code printed: ${output}`).toBeGreaterThanOrEqual(3);
114
+ for (const face of rolled) {
115
+ expect(face, `run_code printed: ${output}`).toBeGreaterThanOrEqual(1);
116
+ expect(face, `run_code printed: ${output}`).toBeLessThanOrEqual(20);
117
+ }
118
+ },
119
+ { live: true },
120
+ );
121
+
122
+ test(
123
+ "the run_code builtin is wired to the agent's tool executor",
124
+ async ({ session }) => {
125
+ const turn = await session.say("What is 127 times 849?");
126
+
127
+ // A tool the agent does not declare produces a `tool.called` with no
128
+ // result, so the paired result is what says `builtinTools: ["run_code"]`
129
+ // still resolves to something executable. The ANSWER rather than
130
+ // `toBeDefined()`, which the refusal string satisfied too.
131
+ const [call] = turn.toolCalls;
132
+ expect(call?.name).toBe("run_code");
133
+ expect(call?.result).toBe("107823");
134
+ expect(turn.completed).toBe(true);
135
+ },
136
+ {
137
+ stubReply: [
138
+ { tool: "run_code", args: { code: "console.log(127 * 849)" } },
139
+ "That's 107,823.",
140
+ ],
141
+ },
142
+ );
143
+ },
144
+ // `runCode` is what makes these cases about the ANSWER and not just the call.
145
+ { runCode },
146
+ );
@@ -0,0 +1,272 @@
1
+ /// <reference types="vite/client" />
2
+
3
+ // An EVAL: does the companion actually pick from its own shelf?
4
+ //
5
+ // `agent.test.ts` calls `recommend` directly, which settles what the tool does
6
+ // with a category and a mood it is handed. What it cannot settle is whether the
7
+ // MODEL turns "something cozy to watch" into `{ category: "movie", mood:
8
+ // "cozy" }` rather than reciting three films it likes — and whether the night's
9
+ // log, which lives in a `sessionSlot`, is still there two turns later.
10
+ //
11
+ // Run it with `aai eval`. Without a provider key every case runs against a
12
+ // SCRIPTED model (its `stubReply`): the real session, the real slot, the real
13
+ // tool, a fake reply. That proves the wiring and nothing about the choice.
14
+
15
+ import { runInNewContext } from "node:vm";
16
+ import { withSystemPrompt } from "@alexkroman1/aai/manifest";
17
+ import type { SessionEvent } from "@alexkroman1/aai/protocol";
18
+ import { withDiscoveredTools } from "@alexkroman1/aai/testing";
19
+ import { errorMessage } from "@alexkroman1/aai/utils";
20
+ import {
21
+ customEventsIn,
22
+ lastStateIn,
23
+ type RunCodeExecutor,
24
+ toolResultIn,
25
+ } from "@alexkroman1/aai-runtime/eval";
26
+ import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
27
+ import { expect } from "vitest";
28
+ import { z } from "zod";
29
+ import authoredAgent from "./agent.ts";
30
+ import { CATEGORIES, MOODS } from "./shared.ts";
31
+ import systemPrompt from "./system-prompt.md?raw";
32
+
33
+ /**
34
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
35
+ *
36
+ * The glob is written HERE rather than reached for from a shared helper because
37
+ * this file SHIPS — a scaffolded project has no repo helper to import. Without
38
+ * it the eval would drive an agent with no `recommend` at all, which is the one
39
+ * failure a green eval must never be able to hide.
40
+ *
41
+ * And plus its PROMPT. `agent.ts` does not declare one — `system-prompt.md` is
42
+ * resolved by the BUILD (`aai build`/`aai deploy`), so the raw default export
43
+ * carries the FRAMEWORK DEFAULT prompt. An eval that drives it measures a
44
+ * different agent than the one that deploys, and every tool-choice claim below
45
+ * then passes or fails for the wrong reason.
46
+ */
47
+ const agentDef = withSystemPrompt(
48
+ withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
49
+ systemPrompt,
50
+ );
51
+
52
+ /**
53
+ * One `recommend` answer, and the whole projection, as the wire carries them.
54
+ *
55
+ * Schemas rather than casts, which is what `toolResultIn` and `lastStateIn`
56
+ * take one for: a shelf or a projection that stopped matching FAILS here naming
57
+ * the field, where a cast hands the assertions `undefined` and fails a line
58
+ * later on something unrelated.
59
+ */
60
+ const RecSchema = z.object({
61
+ category: z.enum(CATEGORIES),
62
+ mood: z.enum(MOODS),
63
+ picks: z.array(z.string()),
64
+ });
65
+ const ProjectedNight = z.object({ recs: z.array(RecSchema) });
66
+
67
+ /**
68
+ * The night's log as the PAGE has it: the last `syncState` frame pushed.
69
+ *
70
+ * This agent declares one projection, so the frame IS `nightProjection`'s
71
+ * result — the same `{ recs }` value `useAgentState(nightProjection)` reads in
72
+ * `client.tsx`. Asserting on it is asserting on what the sidebar shows.
73
+ */
74
+ const pushedRecs = (events: readonly SessionEvent[]) =>
75
+ lastStateIn(events, ProjectedNight)?.recs ?? [];
76
+
77
+ /** The `wind_down` nudges in `events` — `customEventsIn` filters by name. */
78
+ const nudges = (events: readonly SessionEvent[]) => customEventsIn(events, "wind_down");
79
+
80
+ /**
81
+ * A `run_code` executor, so the sleep-cycle case can assert the ANSWER.
82
+ *
83
+ * The builtin refuses without one — the Modal container is the security
84
+ * boundary, and off-platform there is none — which left this template's
85
+ * headline feature assertable as a CALL and never as a number. A `node:vm`
86
+ * context with a capturing `console.log` is what a developer would reach for on
87
+ * their own machine, and it is enough: the code under test is arithmetic the
88
+ * model wrote, not a program.
89
+ *
90
+ * A template eval imports from `@alexkroman1/aai-runtime/eval` and
91
+ * `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
92
+ * node-reaching module graph into this project's TypeScript program, which is
93
+ * three errors in runtime files no eval ever calls. `RunCodeExecutor` is
94
+ * re-exported from `/eval` for exactly that reason.
95
+ */
96
+ const runCode: RunCodeExecutor = async (code) => {
97
+ const lines: string[] = [];
98
+ const log = (...args: unknown[]): void => {
99
+ lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
100
+ };
101
+ try {
102
+ runInNewContext(code, { console: { log } }, { timeout: 1000 });
103
+ } catch (err) {
104
+ return { error: errorMessage(err) };
105
+ }
106
+ return lines.join("\n");
107
+ };
108
+
109
+ /** Two-digit, for the clock arithmetic below. */
110
+ const pad = (n: number): string => String(n).padStart(2, "0");
111
+
112
+ /**
113
+ * Every bedtime `system-prompt.md`'s recipe admits for a given wake-up hour,
114
+ * in each of the spellings a tutor might print it in.
115
+ *
116
+ * Derived from the recipe rather than typed out — 90 minutes a cycle plus 15 to
117
+ * fall asleep, wrapped into the previous day — so this and the prompt cannot
118
+ * disagree about the arithmetic the case is checking.
119
+ */
120
+ const bedtimesFor = (wakeHour: number): string[] =>
121
+ [3, 4, 5, 6].flatMap((cycles) => {
122
+ const at = (wakeHour * 60 - (cycles * 90 + 15) + 1440) % 1440;
123
+ const hour = Math.floor(at / 60);
124
+ const minute = pad(at % 60);
125
+ // Three spellings per time, because the prompt asks for HH:MM and a tutor
126
+ // reasonably prints any of them: padded, unpadded (measured — a live run
127
+ // printed "0:45" for four cycles), and the 12-hour clock.
128
+ return [
129
+ `${pad(hour)}:${minute}`,
130
+ `${hour}:${minute}`,
131
+ `${hour % 12 === 0 ? 12 : hour % 12}:${minute}`,
132
+ ];
133
+ });
134
+
135
+ describeEval(
136
+ agentDef,
137
+ (test) => {
138
+ test(
139
+ "turns a mood into the category and mood the tool takes",
140
+ async ({ session }) => {
141
+ const turn = await session.say("I want something cozy to watch tonight.");
142
+
143
+ // "to watch" is the category and "cozy" is the mood; the shelf is the
144
+ // tool's, so answering from the model's own taste is the regression.
145
+ expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
146
+ const call = turn.toolCalls[0]!;
147
+ expect(call.args).toEqual({ category: "movie", mood: "cozy" });
148
+
149
+ // And what it read out came back from the shelf: the tool answers with
150
+ // the picks for exactly the pair it was asked for.
151
+ const rec = toolResultIn(turn.toolCalls, "recommend", RecSchema);
152
+ expect(rec).toMatchObject({ category: "movie", mood: "cozy" });
153
+ expect(rec.picks.length).toBeGreaterThan(0);
154
+ // It read out what the shelf handed back rather than a title of its own.
155
+ expect(rec.picks.some((pick) => turn.text.includes(pick))).toBe(true);
156
+ },
157
+ {
158
+ stubReply: [
159
+ { tool: "recommend", args: { category: "movie", mood: "cozy" } },
160
+ "Paddington 2 is the coziest thing I own.",
161
+ ],
162
+ },
163
+ );
164
+
165
+ test(
166
+ "the night's log keeps what an earlier turn picked, newest first",
167
+ async ({ session }) => {
168
+ await session.say("I want something cozy to watch tonight.");
169
+ const turn = await session.say("Now give me something spooky to read.");
170
+
171
+ expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
172
+ expect(turn.toolCalls[0]!.args).toEqual({ category: "book", mood: "spooky" });
173
+
174
+ // The slot survived the turn boundary: the frame the page renders after
175
+ // the second answer still carries the first, and the newest is first —
176
+ // which is the order the sidebar lists them in.
177
+ const recs = pushedRecs(session.events());
178
+ expect(recs.map((r) => `${r.category}/${r.mood}`)).toEqual(["book/spooky", "movie/cozy"]);
179
+ },
180
+ {
181
+ stubReply: [
182
+ { tool: "recommend", args: { category: "movie", mood: "cozy" } },
183
+ "Paddington 2 it is.",
184
+ { tool: "recommend", args: { category: "book", mood: "spooky" } },
185
+ "Mexican Gothic, then.",
186
+ ],
187
+ },
188
+ );
189
+
190
+ test(
191
+ "the wind-down nudge arrives once, on the third pick",
192
+ async ({ session }) => {
193
+ const first = await session.say("I want something cozy to watch tonight.");
194
+ expect(nudges(first.events)).toEqual([]);
195
+ const second = await session.say("Now give me something spooky to read.");
196
+ expect(nudges(second.events)).toEqual([]);
197
+
198
+ // Counted in the SLOT, so the third pick is only the third if the two
199
+ // before it were still there — and it is a `ctx.send`, not a field on the
200
+ // projection, so it must arrive exactly once and never be replayed.
201
+ const third = await session.say("And some chill music too.");
202
+ expect(nudges(third.events)).toHaveLength(1);
203
+
204
+ const fourth = await session.say("One funny book as well, please.");
205
+ expect(nudges(fourth.events)).toEqual([]);
206
+ expect(nudges(session.events())).toHaveLength(1);
207
+ expect(pushedRecs(session.events())).toHaveLength(4);
208
+ },
209
+ {
210
+ stubReply: [
211
+ { tool: "recommend", args: { category: "movie", mood: "cozy" } },
212
+ "Paddington 2 it is.",
213
+ { tool: "recommend", args: { category: "book", mood: "spooky" } },
214
+ "Mexican Gothic, then.",
215
+ { tool: "recommend", args: { category: "music", mood: "chill" } },
216
+ "Tycho, Dive.",
217
+ { tool: "recommend", args: { category: "book", mood: "funny" } },
218
+ "Good Omens.",
219
+ ],
220
+ },
221
+ );
222
+ test(
223
+ "works the bedtime out in CODE, and the number is right",
224
+ async ({ session }) => {
225
+ const turn = await session.say(
226
+ "I need to be up at 7 in the morning. When should I fall asleep?",
227
+ );
228
+
229
+ const ran = turn.toolCalls.filter((c) => c.name === "run_code");
230
+ expect(
231
+ ran,
232
+ `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
233
+ ).not.toEqual([]);
234
+ // The recipe is the prompt's, and it is two constants: a 90-minute cycle
235
+ // plus the 15 minutes it takes to fall asleep. Arithmetic done in the
236
+ // model's head has neither of them anywhere in the code.
237
+ const code = ran.map((c) => String(c.args.code ?? "")).join("\n");
238
+ expect(code).toContain("90");
239
+ expect(code).toContain("15");
240
+
241
+ const output = ran.map((c) => c.result ?? "").join("\n");
242
+ // The builtin really EXECUTED. With no `runCode` executor this string is
243
+ // "run_code is only available in the sandboxed runtime", which every
244
+ // assertion about a CALL sails past — so this template's headline
245
+ // feature could be checked as a call and never as an answer.
246
+ expect(output).not.toMatch(/only available in the sandboxed runtime/);
247
+ // And the answer is a whole number of cycles back from 07:00 with the
248
+ // quarter hour added. A tutor that dropped the 15 lands on :00 and a
249
+ // tutor that guessed lands anywhere; both fail here.
250
+ expect(output, `run_code printed: ${output}`).toMatch(new RegExp(bedtimesFor(7).join("|")));
251
+ },
252
+ {
253
+ stubReply: [
254
+ {
255
+ tool: "run_code",
256
+ args: {
257
+ code: [
258
+ "const wake = 7 * 60;",
259
+ "const at = (wake - (6 * 90 + 15) + 1440) % 1440;",
260
+ "const two = (n) => String(n).padStart(2, '0');",
261
+ "console.log(two(Math.floor(at / 60)) + ':' + two(at % 60));",
262
+ ].join("\n"),
263
+ },
264
+ },
265
+ "Aim for nine forty-five tonight — that's six full cycles before seven.",
266
+ ],
267
+ },
268
+ );
269
+ },
270
+ // `runCode` is what makes the case above about an ANSWER rather than a call.
271
+ { runCode },
272
+ );
@@ -1,10 +1,16 @@
1
1
  /// <reference types="vite/client" />
2
2
 
3
3
  import { toAgentConfig } from "@alexkroman1/aai/manifest";
4
- import { createToolContext, runTool, withDiscoveredTools } from "@alexkroman1/aai/testing";
4
+ import {
5
+ createToolContext,
6
+ parseToolInput,
7
+ toolInputIssues,
8
+ toolRunner,
9
+ withDiscoveredTools,
10
+ } from "@alexkroman1/aai/testing";
5
11
  import { describe, expect, test } from "vitest";
6
12
  import authoredAgent from "./agent.ts";
7
- import { CATEGORIES, MOODS } from "./shared.ts";
13
+ import { CATEGORIES, MOODS, nightProjection, nightSlot } from "./shared.ts";
8
14
 
9
15
  /**
10
16
  * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
@@ -17,6 +23,14 @@ const agentDef = withDiscoveredTools(
17
23
  import.meta.glob("./tools/*.ts", { eager: true }),
18
24
  );
19
25
 
26
+ /**
27
+ * `runTool` takes the context in the ARGUMENTS' place when a tool needs none,
28
+ * so `toolRunner`'s second parameter takes either — which is why it is one
29
+ * signature rather than an overload pair. An omitted context is a fresh one,
30
+ * i.e. a distinct session with an empty slot.
31
+ */
32
+ const run = toolRunner(agentDef);
33
+
20
34
  describe("night-owl template", () => {
21
35
  test("config passes manifest validation", () => {
22
36
  // Same conversion `aai build`/`aai deploy` run.
@@ -33,22 +47,53 @@ describe("night-owl template", () => {
33
47
  test("recommend is discovered from tools/", () => {
34
48
  expect(Object.keys(agentDef.tools ?? {})).toEqual(["recommend"]);
35
49
  });
50
+
51
+ test("the projection an untouched session pushes is an empty log", () => {
52
+ // What `useAgentState(nightProjection)` reads before the first tool call —
53
+ // derived from the slot's own default rather than guessed at in the page.
54
+ expect(nightProjection()).toEqual({ recs: [] });
55
+ });
36
56
  });
37
57
 
38
58
  describe("recommend", () => {
39
59
  test("answers with picks for the category and mood asked for", async () => {
40
- const ctx = createToolContext();
41
- const result = await runTool(agentDef, "recommend", { category: "movie", mood: "cozy" }, ctx);
60
+ const result = await run("recommend", { category: "movie", mood: "cozy" });
42
61
  expect(result).toMatchObject({ category: "movie", mood: "cozy" });
43
62
  expect((result as { picks: string[] }).picks.length).toBeGreaterThan(0);
44
63
  });
45
64
 
46
- test("pushes the same picks to the client, which is what the page renders", async () => {
47
- // `ctx.send` is the only reason this tool takes a context at all: the
48
- // client renders the picks rather than waiting to hear them read aloud.
65
+ test("the picks land in the session's own log, newest first", async () => {
66
+ // The log is STATE, not an event stream: `syncState` pushes this projection
67
+ // after every tool call, so a page that reloads mid-session resumes with it
68
+ // rather than starting empty.
69
+ const ctx = createToolContext();
70
+ const first = await run("recommend", { category: "book", mood: "spooky" }, ctx);
71
+ const second = await run("recommend", { category: "music", mood: "chill" }, ctx);
72
+ expect(nightProjection(nightSlot.get(ctx))).toEqual({ recs: [second, first] });
73
+ });
74
+
75
+ test("two calls with no shared context are two sessions", async () => {
76
+ // The other half of the same rule, and the one that bites: an omitted
77
+ // context is a FRESH session, so nothing accumulates across these calls.
49
78
  const ctx = createToolContext();
50
- const result = await runTool(agentDef, "recommend", { category: "book", mood: "spooky" }, ctx);
51
- expect(ctx.sent).toEqual([{ event: "recommendations", data: result }]);
79
+ await run("recommend", { category: "book", mood: "cozy" });
80
+ expect(nightSlot.get(ctx).recs).toHaveLength(0);
81
+ });
82
+
83
+ test("the wind-down nudge is sent once, on the third pick", async () => {
84
+ // A moment rather than state, which is why it is a `ctx.send` the page
85
+ // consumes with `useEvent` and not a field on the projection: re-delivering
86
+ // it on every reconnect would be nagging.
87
+ const ctx = createToolContext();
88
+ await run("recommend", { category: "movie", mood: "cozy" }, ctx);
89
+ await run("recommend", { category: "music", mood: "cozy" }, ctx);
90
+ expect(ctx.sent).toEqual([]);
91
+ await run("recommend", { category: "book", mood: "cozy" }, ctx);
92
+ expect(ctx.sent).toEqual([
93
+ { event: "wind_down", data: "Three picks in. Want me to work out your bedtime?" },
94
+ ]);
95
+ await run("recommend", { category: "movie", mood: "chill" }, ctx);
96
+ expect(ctx.sent).toHaveLength(1);
52
97
  });
53
98
 
54
99
  test("every category/mood pair the schema admits has picks behind it", async () => {
@@ -58,23 +103,29 @@ describe("recommend", () => {
58
103
  // package's guide records three shipped tools having.
59
104
  for (const category of CATEGORIES) {
60
105
  for (const mood of MOODS) {
61
- const result = await runTool(
62
- agentDef,
63
- "recommend",
64
- { category, mood },
65
- createToolContext(),
66
- );
106
+ const result = await run("recommend", { category, mood });
67
107
  expect((result as { picks: string[] }).picks, `${category}/${mood}`).not.toHaveLength(0);
68
108
  }
69
109
  }
70
110
  });
71
111
 
112
+ test("the schema accepts a category/mood pair from the enums", async () => {
113
+ const parsed = await parseToolInput<{ category: string; mood: string }>(agentDef, "recommend", {
114
+ category: "movie",
115
+ mood: "cozy",
116
+ });
117
+ expect(parsed).toEqual({ category: "movie", mood: "cozy" });
118
+ });
119
+
72
120
  test("a mood outside the enum is refused by the schema", async () => {
73
121
  // The wire boundary: an LLM tool call is untyped, so the schema is the only
74
122
  // thing between a hallucinated mood and an index into `undefined`.
75
- const schema = agentDef.tools?.recommend?.inputSchema;
76
- if (!schema) throw new Error("recommend has no input schema");
77
- const bad = await schema["~standard"].validate({ category: "movie", mood: "melancholy" });
78
- expect(bad.issues).toBeDefined();
123
+ // `toolInputIssues` is the SDK's own ask — `~standard` is a vendor wire
124
+ // contract, and the detail a hand-rolled version gets wrong first is that
125
+ // `.validate` may be sync or async, so a missing `await` leaves `.issues`
126
+ // undefined and the negative test passes for the wrong reason.
127
+ expect(
128
+ await toolInputIssues(agentDef, "recommend", { category: "movie", mood: "melancholy" }),
129
+ ).toBeDefined();
79
130
  });
80
131
  });
@@ -1,7 +1,12 @@
1
1
  import { agent } from "@alexkroman1/aai";
2
+ import { nightProjection } from "./shared.ts";
2
3
 
3
4
  export default agent({
4
5
  name: "Night Owl",
6
+ // The night's recommendation log, pushed to the client after every tool
7
+ // call. The page renders `useAgentState(nightProjection)` and keeps no copy
8
+ // of its own, so a reload resumes with the list intact.
9
+ syncState: nightProjection,
5
10
  greeting:
6
11
  "Hey there, night owl. Try asking me for a cozy movie recommendation, or tell me what time you need to wake up and I'll calculate the best time to fall asleep.",
7
12
  builtinTools: ["run_code"],