@alexkroman1/aai-cli 7.0.0 → 8.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +1 -0
  2. package/dist/{_bundler-B4RqNF5Z.mjs → _bundler-CDuYl5Gb.mjs} +1 -1
  3. package/dist/{_dev-server-CiKFH9qw.mjs → _dev-server-Dt3TCjhj.mjs} +6 -2
  4. package/dist/{_init-CE4HKdgV.mjs → _init-BhEfQ6Yi.mjs} +1 -1
  5. package/dist/_resource-commands.d.ts +23 -0
  6. package/dist/{_slug-api-nRFaBEPJ.mjs → _slug-api-z-USxPtL.mjs} +14 -10
  7. package/dist/_slug-api.d.ts +5 -2
  8. package/dist/{_templates-FI3xPEMj.mjs → _templates-4WcKOjS5.mjs} +4 -5
  9. package/dist/_templates.d.ts +3 -4
  10. package/dist/{build-6jfaakgc.mjs → build-BBxf8vom.mjs} +2 -2
  11. package/dist/cli.mjs +338 -346
  12. package/dist/{deploy-DvYeD8ia.mjs → deploy-DRC8jWr8.mjs} +1 -1
  13. package/dist/{dev-B59JIfWO.mjs → dev-QuuOOUtv.mjs} +1 -1
  14. package/dist/eval-DfQ8cm5p.mjs +56 -0
  15. package/dist/eval.d.ts +43 -0
  16. package/dist/{init-CpJVjEg0.mjs → init-CUo87gcI.mjs} +2 -2
  17. package/dist/{logs-9q-psa4q.mjs → logs-CQ2T46dr.mjs} +1 -1
  18. package/dist/scaffold/CLAUDE.md +409 -68
  19. package/dist/scaffold/package.json +5 -4
  20. package/dist/scaffold/server.mjs +10 -3
  21. package/dist/scaffold/tsconfig.json +0 -1
  22. package/dist/{secret-CVvSLIDV.mjs → secret-Ct34kdgU.mjs} +1 -1
  23. package/dist/{storage-BvUrnvM3.mjs → storage-i_79ZwAR.mjs} +1 -1
  24. package/dist/{studio--MUV0cid.mjs → studio-CzTzPqe1.mjs} +1 -1
  25. package/dist/templates/briefing-desk/agent.test.ts +350 -0
  26. package/dist/templates/briefing-desk/agent.ts +30 -0
  27. package/dist/templates/briefing-desk/shared.ts +200 -0
  28. package/dist/templates/briefing-desk/system-prompt.md +39 -0
  29. package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
  30. package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
  31. package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
  32. package/dist/templates/call-audit/agent.eval.test.ts +474 -0
  33. package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
  34. package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
  35. package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
  36. package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
  37. package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
  38. package/dist/templates/infocom-adventure/agent.test.ts +49 -18
  39. package/dist/templates/infocom-adventure/agent.ts +21 -0
  40. package/dist/templates/infocom-adventure/shared.ts +20 -9
  41. package/dist/templates/infocom-adventure/system-prompt.md +1 -1
  42. package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
  43. package/dist/templates/link-digest/agent.eval.test.ts +195 -0
  44. package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
  45. package/dist/templates/night-owl/agent.eval.test.ts +272 -0
  46. package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
  47. package/dist/templates/personal-finance/system-prompt.md +4 -0
  48. package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
  49. package/dist/templates/pipeline-simple/agent.ts +2 -2
  50. package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
  51. package/dist/templates/pizza-ordering/agent.test.ts +15 -0
  52. package/dist/templates/pizza-ordering/shared.ts +20 -1
  53. package/dist/templates/pizza-ordering/system-prompt.md +6 -0
  54. package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
  55. package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
  56. package/dist/templates/podcast-digest/agent.test.ts +30 -59
  57. package/dist/templates/podcast-digest/agent.ts +1 -1
  58. package/dist/templates/podcast-digest/workflows/digest.ts +52 -19
  59. package/dist/templates/podcast-digest/workflows/slack.ts +48 -169
  60. package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
  61. package/dist/templates/redline/agent.eval.test.ts +209 -0
  62. package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
  63. package/dist/templates/retail/address.ts +87 -1
  64. package/dist/templates/retail/agent.eval.test.ts +314 -0
  65. package/dist/templates/retail/agent.test.ts +171 -91
  66. package/dist/templates/retail/agent.ts +35 -1
  67. package/dist/templates/retail/cancel.ts +116 -0
  68. package/dist/templates/retail/client.tsx +39 -0
  69. package/dist/templates/retail/payment.ts +107 -0
  70. package/dist/templates/retail/pending.test.ts +196 -0
  71. package/dist/templates/retail/pending.ts +137 -0
  72. package/dist/templates/retail/registry.test.ts +119 -8
  73. package/dist/templates/retail/returns.ts +102 -0
  74. package/dist/templates/retail/shared.test.ts +42 -0
  75. package/dist/templates/retail/shared.ts +35 -0
  76. package/dist/templates/retail/store.test.ts +6 -4
  77. package/dist/templates/retail/store.ts +113 -57
  78. package/dist/templates/retail/swap.test.ts +4 -4
  79. package/dist/templates/retail/swap.ts +248 -15
  80. package/dist/templates/retail/system-prompt.md +62 -30
  81. package/dist/templates/retail/tools/cancel_change.ts +33 -0
  82. package/dist/templates/retail/tools/cancel_pending_order.ts +15 -59
  83. package/dist/templates/retail/tools/confirm_change.ts +42 -0
  84. package/dist/templates/retail/tools/exchange_delivered_order_items.ts +21 -66
  85. package/dist/templates/retail/tools/modify_pending_order_address.ts +12 -24
  86. package/dist/templates/retail/tools/modify_pending_order_items.ts +22 -60
  87. package/dist/templates/retail/tools/modify_pending_order_payment.ts +15 -77
  88. package/dist/templates/retail/tools/modify_user_address.ts +12 -15
  89. package/dist/templates/retail/tools/return_delivered_order_items.ts +16 -71
  90. package/dist/templates/retail/tools/transfer_to_human_agents.ts +4 -2
  91. package/dist/templates/simple/agent.eval.test.ts +54 -0
  92. package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
  93. package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
  94. package/dist/templates/support-line/agent.eval.test.ts +273 -0
  95. package/dist/templates/support-line/system-prompt.md +2 -1
  96. package/dist/templates/support-line/tools/answer_question.ts +56 -7
  97. package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
  98. package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
  99. package/dist/templates/travel-concierge/agent.test.ts +110 -10
  100. package/dist/templates/travel-concierge/agent.ts +18 -9
  101. package/dist/templates/travel-concierge/shared.ts +110 -9
  102. package/dist/templates/travel-concierge/system-prompt.md +7 -1
  103. package/dist/templates/travel-concierge/tools/book_car_rental.ts +6 -3
  104. package/dist/templates/travel-concierge/tools/book_excursion.ts +6 -3
  105. package/dist/templates/travel-concierge/tools/book_hotel.ts +6 -3
  106. package/dist/templates/travel-concierge/tools/cancel_ticket.ts +6 -3
  107. package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
  108. package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
  109. package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
  110. package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
  111. package/dist/templates/travel-concierge/tools/update_ticket.ts +6 -3
  112. package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
  113. package/dist/templates/web-researcher/system-prompt.md +15 -4
  114. package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
  115. package/dist/test.d.ts +29 -5
  116. package/dist/{worker-bundler-COxnqstQ.mjs → worker-bundler-CCVEDjm1.mjs} +45 -25
  117. package/dist/worker-bundler.mjs +1 -1
  118. package/dist/{workflow-D2AQf2Pl.mjs → workflow-BflATMmU.mjs} +19 -11
  119. package/package.json +5 -5
  120. package/dist/eject-C9WJyyr2.mjs +0 -116
  121. package/dist/eject.d.ts +0 -60
  122. package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
@@ -1,22 +1,20 @@
1
1
  import { isToolFailure } from "@alexkroman1/aai";
2
2
  import { z } from "zod";
3
- import { OrderIdField, resolveOrder } from "../resolve.ts";
4
- import {
5
- authenticatedUser,
6
- findPaymentMethod,
7
- isGiftCard,
8
- money,
9
- retailTool,
10
- setFocus,
11
- } from "../store.ts";
3
+ import { planPayment } from "../payment.ts";
4
+ import { stageAction } from "../pending.ts";
5
+ import { OrderIdField } from "../resolve.ts";
6
+ import { retailTool, setFocus } from "../store.ts";
12
7
 
13
8
  export default retailTool({
14
9
  name: "modify_pending_order_payment",
15
10
  when: "serving",
11
+ send: { type: "STAGED" },
16
12
  description:
17
- "Change which payment method a pending order is charged to. The new method must be different " +
18
- "from the current one, and a gift card must hold enough to cover the whole order. The original " +
19
- "method is refunded. Read the change back and get an explicit yes before calling this.",
13
+ "STAGE a change to which payment method a pending order is charged to this does NOT change " +
14
+ "it. The new method must be different from the current one, and a gift card must hold enough " +
15
+ "to cover the whole order; the original method is refunded. The change comes back as a " +
16
+ "sentence to read to the caller; nothing happens until you hear an explicit yes and call " +
17
+ "confirm_change.",
20
18
  inputSchema: z.object({
21
19
  order_id: OrderIdField,
22
20
  payment_method_id: z
@@ -25,70 +23,10 @@ export default retailTool({
25
23
  .describe("The new payment method id, e.g. 'gift_card_0000000'"),
26
24
  }),
27
25
  execute: (args, state) => {
28
- const user = authenticatedUser(state);
29
- if (isToolFailure(user)) return user;
30
-
31
- const order = resolveOrder(state, args.order_id);
32
- if (isToolFailure(order)) return order;
33
- setFocus(state, { orderId: order.order_id });
34
-
35
- if (!order.status.startsWith("pending")) {
36
- return {
37
- error: `Order ${order.order_id} is ${order.status}, and only a pending order's payment method can be changed.`,
38
- };
39
- }
40
-
41
- const newMethod = findPaymentMethod(user, args.payment_method_id);
42
- if (isToolFailure(newMethod)) return newMethod;
43
-
44
- const original = order.payment_history[0];
45
- if (order.payment_history.length !== 1 || original?.transaction_type !== "payment") {
46
- return {
47
- error: `Order ${order.order_id} does not have exactly one payment on record, so its payment method cannot be changed.`,
48
- };
49
- }
50
- if (original.payment_method_id === args.payment_method_id) {
51
- return {
52
- error: `Order ${order.order_id} is already paid with ${args.payment_method_id}. The new method must be different.`,
53
- };
54
- }
55
-
56
- const amount = original.amount;
57
- if (isGiftCard(newMethod) && newMethod.balance < amount) {
58
- return {
59
- error: `Gift card ${args.payment_method_id}'s balance ($${newMethod.balance.toFixed(2)}) does not cover the $${amount.toFixed(2)} order total.`,
60
- };
61
- }
62
-
63
- order.payment_history.push(
64
- {
65
- transaction_type: "payment",
66
- amount,
67
- payment_method_id: args.payment_method_id,
68
- },
69
- {
70
- transaction_type: "refund",
71
- amount,
72
- payment_method_id: original.payment_method_id,
73
- },
74
- );
75
-
76
- if (isGiftCard(newMethod)) {
77
- newMethod.balance = money(newMethod.balance - amount);
78
- }
79
- const oldMethod = user.payment_methods[original.payment_method_id];
80
- if (oldMethod && isGiftCard(oldMethod)) {
81
- oldMethod.balance = money(oldMethod.balance + amount);
82
- }
83
-
84
- return {
85
- order_id: order.order_id,
86
- status: order.status,
87
- amount,
88
- paid_with: args.payment_method_id,
89
- refunded_to: original.payment_method_id,
90
- message: `Order ${order.order_id} is now charged to ${args.payment_method_id}, and $${amount.toFixed(2)} is going back to ${original.payment_method_id}.`,
91
- };
26
+ const plan = planPayment(state, args.order_id, args.payment_method_id);
27
+ if (isToolFailure(plan)) return plan;
28
+ setFocus(state, { orderId: plan.orderId });
29
+ return stageAction(state, { kind: "modify_pending_order_payment", plan });
92
30
  },
93
- summary: (_args, result) => `re-paid ${result.order_id}`,
31
+ summary: (_args, result) => `staged: ${result.read_back}`,
94
32
  });
@@ -1,29 +1,26 @@
1
1
  import { isToolFailure } from "@alexkroman1/aai";
2
2
  import { z } from "zod";
3
- import { AddressFields, formatAddress, toAddress } from "../address.ts";
4
- import { requireOwnUser, retailTool } from "../store.ts";
3
+ import { AddressFields, planUserAddress } from "../address.ts";
4
+ import { stageAction } from "../pending.ts";
5
+ import { retailTool } from "../store.ts";
5
6
 
6
7
  export default retailTool({
7
8
  name: "modify_user_address",
8
9
  when: "serving",
10
+ send: { type: "STAGED" },
9
11
  description:
10
- "Change the customer's default address, used for future orders. Read the new address back and " +
11
- "get an explicit yes before calling this. This does not change the address on any existing " +
12
- "order use modify_pending_order_address for that.",
12
+ "STAGE a change to the customer's default address, used for future orders this does NOT " +
13
+ "change it. The new address comes back as a sentence to read to the caller; nothing happens " +
14
+ "until you hear an explicit yes and call confirm_change. This does not change the address on " +
15
+ "any existing order — use modify_pending_order_address for that.",
13
16
  inputSchema: z.object({
14
17
  user_id: z.string().max(100).describe("The user id, e.g. 'sara_doe_496'"),
15
18
  ...AddressFields,
16
19
  }),
17
20
  execute: (args, state) => {
18
- const user = requireOwnUser(state, args.user_id);
19
- if (isToolFailure(user)) return user;
20
-
21
- user.address = toAddress(args);
22
- return {
23
- user_id: user.user_id,
24
- address: user.address,
25
- message: `Default address updated to ${formatAddress(user.address)}. Existing orders keep their own shipping addresses.`,
26
- };
21
+ const plan = planUserAddress(state, args.user_id, args);
22
+ if (isToolFailure(plan)) return plan;
23
+ return stageAction(state, { kind: "modify_user_address", plan });
27
24
  },
28
- summary: (_args, result) => `re-addressed ${result.user_id}`,
25
+ summary: (_args, result) => `staged: ${result.read_back}`,
29
26
  });
@@ -1,23 +1,21 @@
1
1
  import { isToolFailure } from "@alexkroman1/aai";
2
2
  import { z } from "zod";
3
- import { OrderIdField, resolveOrder } from "../resolve.ts";
4
- import {
5
- authenticatedUser,
6
- findPaymentMethod,
7
- isGiftCard,
8
- retailTool,
9
- setFocus,
10
- } from "../store.ts";
3
+ import { stageAction } from "../pending.ts";
4
+ import { OrderIdField } from "../resolve.ts";
5
+ import { planReturn } from "../returns.ts";
6
+ import { retailTool, setFocus } from "../store.ts";
11
7
 
12
8
  export default retailTool({
13
9
  name: "return_delivered_order_items",
14
10
  when: "serving",
11
+ send: { type: "STAGED" },
15
12
  description:
16
- "Request a return of items from a delivered order. Only a 'delivered' order can be returned, " +
17
- "and only once. The refund must go to the order's ORIGINAL payment method or to one of the " +
18
- "customer's gift cards. Confirm the exact item list and the refund destination with an " +
19
- "explicit yes before calling this. The customer gets an email explaining how to send the " +
20
- "items back.",
13
+ "STAGE a return of items from a delivered order this does NOT request anything. Only a " +
14
+ "'delivered' order can be returned, and only once. The refund must go to the order's ORIGINAL " +
15
+ "payment method or to one of the customer's gift cards. The exact item list and the refund " +
16
+ "destination come back as a sentence to read to the caller; nothing happens until you hear an " +
17
+ "explicit yes and call confirm_change. The customer then gets an email explaining how to send " +
18
+ "the items back.",
21
19
  inputSchema: z.object({
22
20
  order_id: OrderIdField,
23
21
  item_ids: z
@@ -30,63 +28,10 @@ export default retailTool({
30
28
  .describe("Where the refund goes — the original method, or one of their gift cards"),
31
29
  }),
32
30
  execute: (args, state) => {
33
- const user = authenticatedUser(state);
34
- if (isToolFailure(user)) return user;
35
-
36
- const order = resolveOrder(state, args.order_id);
37
- if (isToolFailure(order)) return order;
38
- setFocus(state, { orderId: order.order_id });
39
-
40
- if (order.status !== "delivered") {
41
- return {
42
- error: `Order ${order.order_id} is ${order.status}. Only a delivered order can be returned, and only once.`,
43
- };
44
- }
45
-
46
- const method = findPaymentMethod(user, args.payment_method_id);
47
- if (isToolFailure(method)) return method;
48
-
49
- const originalMethodId = order.payment_history[0]?.payment_method_id;
50
- if (!isGiftCard(method) && args.payment_method_id !== originalMethodId) {
51
- return {
52
- error: `A refund must go to the original payment method (${originalMethodId}) or to a gift card. ${args.payment_method_id} is neither.`,
53
- };
54
- }
55
-
56
- if (args.item_ids.length === 0) {
57
- return { error: "No items were listed to return." };
58
- }
59
- const held = new Map<string, number>();
60
- for (const item of order.items) {
61
- held.set(item.item_id, (held.get(item.item_id) ?? 0) + 1);
62
- }
63
- const asked = new Map<string, number>();
64
- for (const itemId of args.item_ids) {
65
- asked.set(itemId, (asked.get(itemId) ?? 0) + 1);
66
- }
67
- for (const [itemId, count] of asked) {
68
- const available = held.get(itemId) ?? 0;
69
- if (count > available) {
70
- return {
71
- error: `Order ${order.order_id} holds ${available} of item ${itemId}, but ${count} were listed for return.`,
72
- };
73
- }
74
- }
75
-
76
- order.status = "return requested";
77
- order.return_items = [...args.item_ids].sort();
78
- order.return_payment_method_id = args.payment_method_id;
79
-
80
- const names = order.return_items
81
- .map((id) => order.items.find((item) => item.item_id === id)?.name ?? id)
82
- .join(", ");
83
- return {
84
- order_id: order.order_id,
85
- status: order.status,
86
- return_items: order.return_items,
87
- refund_to: args.payment_method_id,
88
- message: `Return requested on ${order.order_id} for ${names}. The customer will get an email with return instructions, and the refund goes to ${args.payment_method_id} once the items arrive.`,
89
- };
31
+ const plan = planReturn(state, args.order_id, args.item_ids, args.payment_method_id);
32
+ if (isToolFailure(plan)) return plan;
33
+ setFocus(state, { orderId: plan.orderId });
34
+ return stageAction(state, { kind: "return_delivered_order_items", plan });
90
35
  },
91
- summary: (_args, result) => `return requested on ${result.order_id}`,
36
+ summary: (_args, result) => `staged: ${result.read_back}`,
92
37
  });
@@ -5,8 +5,10 @@ export default retailTool({
5
5
  name: "transfer_to_human_agents",
6
6
  description:
7
7
  "Hand the caller to a human agent. Use this ONLY when the caller explicitly asks for a human, " +
8
- "or when their request cannot be handled with the other tools and the policy. Call this FIRST, " +
9
- "then say 'You are being transferred to a human agent. Please hold on.' and nothing else.",
8
+ "or when their request cannot be handled with the other tools and the policy. The caller is " +
9
+ "transferred by THIS CALL and by nothing else: call it before you say anything about a " +
10
+ "transfer, and never say 'You are being transferred' until it has answered. Once it has, say " +
11
+ "'You are being transferred to a human agent. Please hold on.' and nothing else.",
10
12
  inputSchema: z.object({
11
13
  summary: z.string().max(2000).describe("A short summary of the caller's issue for the human"),
12
14
  }),
@@ -0,0 +1,54 @@
1
+ // An EVAL: does the agent actually behave? Run it with `aai eval`.
2
+ //
3
+ // A test asserts about the config (see agent.test.ts — it never calls a model).
4
+ // An eval drives the real thing: a real session, the real tool executor, the
5
+ // real event stream, with only the microphone and the speaker faked.
6
+ //
7
+ // `describeEval` picks the model for you and says which it picked:
8
+ //
9
+ // * with a provider key — a LIVE model. This spends tokens, takes a few
10
+ // seconds a case, and is a NOISY instrument: a model is probabilistic, so
11
+ // one failure is a question, not a verdict. Re-run before believing either
12
+ // answer.
13
+ // * without one — a SCRIPTED model answering each case's `stubReply`. The
14
+ // agent, the session and this file all really run, so what it proves is
15
+ // that the wiring works. It proves nothing about what the agent SAYS.
16
+ //
17
+ // What no eval here can see: anything below the audio boundary — where the
18
+ // agent decides you stopped talking, how it handles being interrupted, whether
19
+ // two sentences merged into one turn. Those need real paced audio.
20
+ import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
21
+ import { expect } from "vitest";
22
+ import agentDef from "./agent.ts";
23
+
24
+ describeEval(agentDef, (test) => {
25
+ test(
26
+ "answers a question in its own voice",
27
+ async ({ session }) => {
28
+ // `say()` hands back THAT turn, so the claim is about the reply to this
29
+ // question — not about everything said so far, which already includes
30
+ // the agent's greeting.
31
+ const turn = await session.say("What is the capital of France?");
32
+
33
+ expect(turn.completed).toBe(true);
34
+ expect(turn.text).toMatch(/paris/i);
35
+ // This agent has no tools, so reaching for one would be a real finding.
36
+ expect(turn.toolCalls).toEqual([]);
37
+ },
38
+ { stubReply: "Paris is the capital of France." },
39
+ );
40
+
41
+ test(
42
+ "keeps the thread across two turns",
43
+ async ({ session }) => {
44
+ await session.say("My name is Sam.");
45
+ const turn = await session.say("What did I say my name was?");
46
+
47
+ expect(turn.text).toMatch(/sam/i);
48
+ expect(session.events().some((e) => e.type === "error.reported")).toBe(false);
49
+ },
50
+ // One reply per turn: the second is the one under test, and a stub that
51
+ // answered the first would fail the case it is supposed to let run.
52
+ { stubReply: ["Nice to meet you, Sam.", "You said your name was Sam."] },
53
+ );
54
+ });
@@ -0,0 +1,320 @@
1
+ /// <reference types="vite/client" />
2
+
3
+ // An EVAL: does the story machine actually hold? Run it with `aai eval`.
4
+ //
5
+ // `agent.test.ts` drives each tool directly against a context it made itself,
6
+ // which is where the dice arithmetic and the consequence tables belong. What it
7
+ // cannot reach is the thing this template really is: a campaign and a POSITION
8
+ // that both have to survive a turn boundary, in one session, with a model
9
+ // deciding what to call. Every case here is one of those:
10
+ //
11
+ // * the campaign and the flow both outlive the turn that created them,
12
+ // * a standing roll is spent when the scene moves on,
13
+ // * and an emptied pair of tracks really ends the story — the `after` hook
14
+ // writes `gameOver`, the tool reports it in the same call, and the final
15
+ // state refuses everything afterwards.
16
+ //
17
+ // Two modes, announced by `describeEval` on every run:
18
+ //
19
+ // * with ASSEMBLYAI_API_KEY — a LIVE model, which has to choose these tools
20
+ // for itself from what the player said.
21
+ // * without one — a SCRIPTED model whose tool calls REALLY EXECUTE, so the
22
+ // campaign really changes and the flow really moves. That is a genuine
23
+ // multi-turn state test with no model in it.
24
+ //
25
+ // What no eval here can see: anything below the audio boundary — endpointing,
26
+ // barge-in, a sentence split across two turns. Those need real paced audio.
27
+
28
+ import { withSystemPrompt } from "@alexkroman1/aai/manifest";
29
+ import { withDiscoveredTools } from "@alexkroman1/aai/testing";
30
+ import { type EvalTurn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
31
+ import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
32
+ import { expect } from "vitest";
33
+ import { z } from "zod";
34
+ import authoredAgent from "./agent.ts";
35
+ import systemPrompt from "./system-prompt.md?raw";
36
+
37
+ /**
38
+ * The def a DEPLOYED agent runs, assembled the way the build assembles it: the
39
+ * authored export, plus what `tools/` declares, plus `system-prompt.md`.
40
+ *
41
+ * The prompt is not optional here the way it is in a config test. It is the only
42
+ * thing that tells a live model to set the whole game up in ONE call, and an
43
+ * eval run against the framework default prompt would measure an agent nobody
44
+ * deployed.
45
+ */
46
+ const agentDef = withSystemPrompt(
47
+ withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
48
+ systemPrompt,
49
+ );
50
+
51
+ /**
52
+ * What each tool this file drives answers, off the wire.
53
+ *
54
+ * `tool.completed` carries a tool result as a JSON STRING, so a case either
55
+ * casts it or validates it — and a cast is silent exactly when a tool's result
56
+ * changed shape underneath the eval, which is the regression an eval exists to
57
+ * catch. `toolResultIn` takes a schema for that reason, and these name only the
58
+ * fields asserted below: a gated tool's `{ state, done }` position, plus the
59
+ * `result` it nests its own answer under.
60
+ */
61
+ const Setup = z.object({
62
+ state: z.string(),
63
+ initialized: z.boolean(),
64
+ playerName: z.string(),
65
+ });
66
+ const Status = z.object({
67
+ state: z.string(),
68
+ done: z.boolean(),
69
+ gameOver: z.boolean(),
70
+ playerName: z.string(),
71
+ });
72
+ const Roll = z.object({
73
+ state: z.string(),
74
+ result: z.object({ actionDice: z.array(z.number()), challengeDice: z.array(z.number()) }),
75
+ });
76
+ const Settled = z.object({
77
+ state: z.string(),
78
+ done: z.boolean(),
79
+ result: z.object({ gameOver: z.boolean() }),
80
+ });
81
+ const Refusal = z.object({ error: z.string() });
82
+
83
+ /**
84
+ * What the ONE call to `name` answered on this turn.
85
+ *
86
+ * `toolResultIn` is the SDK's reader: it throws rather than returning
87
+ * undefined, and names what the agent called instead — "it called something
88
+ * else" is the finding, and a case that read `undefined` off a missing call
89
+ * would assert against nothing. A turn that called it TWICE is refused too,
90
+ * rather than silently answered with the first.
91
+ */
92
+ const answerOf = <T>(turn: EvalTurn, name: string, schema: z.ZodType<T>): T =>
93
+ toolResultIn(turn.toolCalls, name, schema);
94
+
95
+ /**
96
+ * What EVERY call to `name` answered on this turn, in call order.
97
+ *
98
+ * The plural form is for a claim about calls that may legitimately not have
99
+ * happened — a refusal the narrator may have pre-empted by not calling at all.
100
+ * `toolResultIn` over a ONE-CALL list per call: the name is that call's own, so
101
+ * the reader's "no such call" and "two calls" throws are unreachable and what
102
+ * is left is the parse, the schema, and the "never completed" failure.
103
+ */
104
+ const answersOf = <T>(turn: EvalTurn, name: string, schema: z.ZodType<T>): T[] =>
105
+ turn.toolCalls
106
+ .filter((c) => c.name === name)
107
+ .map((call) => toolResultIn([call], call.name, schema));
108
+
109
+ /**
110
+ * Every field `setup_character` requires, for the SCRIPTED runs.
111
+ *
112
+ * A live model generates these itself from one sentence — that is the
113
+ * template's one-turn setup — so this exists only so a keyless run reaches
114
+ * `playing` and the later turns have a campaign to be about.
115
+ */
116
+ const SERA = {
117
+ genre: "dark_fantasy",
118
+ tone: "melancholic",
119
+ archetype: "outsider_loner",
120
+ playerName: "Sera",
121
+ characterConcept: "A burned-out lamplighter who keeps the last road lit",
122
+ settingDescription:
123
+ "The Ashen Hollow, a valley of cold chimneys where the lamps have been going out one by one.",
124
+ startingLocation: "The Lamplighter's Shed",
125
+ locationDesc: "A shed of oil cans and broken wicks at the edge of the hollow.",
126
+ timeOfDay: "late_evening",
127
+ openingSituation: "The last lamp on the north road has gone dark, and something is moving on it.",
128
+ npc1Name: "Old Ivo",
129
+ npc1Desc: "A wick-cutter who remembers when the hollow was lit end to end",
130
+ npc1Disposition: "neutral",
131
+ npc1Agenda: "Keep the shed's oil ledger balanced, whatever it costs",
132
+ threatClockName: "The Dark Road",
133
+ threatClockDesc: "The hollow goes fully dark and the road is lost",
134
+ } as const;
135
+
136
+ describeEval(agentDef, (test) => {
137
+ test(
138
+ "the character and the story's position both outlive the setup turn",
139
+ async ({ session }) => {
140
+ const opening = await session.say(
141
+ "Dark fantasy, melancholic. My character is Sera, a burned-out lamplighter. Begin.",
142
+ );
143
+ const setup = answerOf(opening, "setup_character", Setup);
144
+ // `setup_character` drives the flow itself (`reset` then `SETUP`) and
145
+ // reports the position it landed in. `awaitingSetup` here would mean the
146
+ // roll tools are still refusing with a full character sheet in the slot.
147
+ expect(setup.state).toBe("playing.awaitingRoll");
148
+ expect(setup.initialized).toBe(true);
149
+ expect(setup.playerName).toMatch(/sera/i);
150
+
151
+ // A SECOND turn, which is the only place either half can be shown to have
152
+ // survived: the campaign lives in a session slot and the position in a
153
+ // dialog beside it, and `check_state` is the one tool that reports both.
154
+ const status = await session.say(
155
+ "Check the state and remind me who I am and where the story is.",
156
+ );
157
+ const now = answerOf(status, "check_state", Status);
158
+ expect(now.playerName).toBe(setup.playerName);
159
+ expect(now.state).toBe("playing.awaitingRoll");
160
+ expect(now.gameOver).toBe(false);
161
+ expect(status.completed).toBe(true);
162
+ },
163
+ {
164
+ stubReply: [
165
+ { tool: "setup_character", args: SERA },
166
+ "The last lamp on the north road has gone out. What do you do, Sera?",
167
+ { tool: "check_state" },
168
+ "You are Sera, a burned-out lamplighter, and the story is waiting on your next move.",
169
+ ],
170
+ },
171
+ );
172
+
173
+ test(
174
+ "a roll leaves a standing roll, and moving the scene on spends it",
175
+ async ({ session }) => {
176
+ await session.say(
177
+ "Dark fantasy, melancholic. My character is Sera, a burned-out lamplighter. Begin.",
178
+ );
179
+
180
+ const climb = await session.say("I climb the storm-lashed tower wall to reach the lamp.");
181
+ const rolled = answerOf(climb, "action_roll", Roll);
182
+ // The dice come from code, never from the narrator — two action dice and
183
+ // two challenge dice, on every roll.
184
+ expect(rolled.result.actionDice).toHaveLength(2);
185
+ expect(rolled.result.challengeDice).toHaveLength(2);
186
+ // `rollResolved` is what keeps `burn_momentum` reachable. A roll that left
187
+ // the flow in `awaitingRoll` would close the burn window instantly.
188
+ expect(rolled.state).toBe("playing.rollResolved");
189
+
190
+ const moved = await session.say(
191
+ "Nothing risky now. Just record that I am at the Drowned Steps and log the scene.",
192
+ );
193
+ const settled = answerOf(moved, "update_state", Settled);
194
+ // `update_state` sends SETTLED: the scene has moved on, so the standing
195
+ // roll is spent and the burn window is shut.
196
+ expect(settled.state).toBe("playing.awaitingRoll");
197
+ expect(settled.result.gameOver).toBe(false);
198
+ },
199
+ {
200
+ stubReply: [
201
+ { tool: "setup_character", args: SERA },
202
+ "The last lamp on the north road has gone out. What do you do, Sera?",
203
+ {
204
+ tool: "action_roll",
205
+ args: {
206
+ move: "face_danger",
207
+ stat: "edge",
208
+ position: "risky",
209
+ effect: "standard",
210
+ purpose: "climb the storm-lashed tower wall",
211
+ },
212
+ },
213
+ "You haul yourself up the wet stone, one handhold at a time.",
214
+ {
215
+ tool: "update_state",
216
+ args: { location: "The Drowned Steps", logEntry: "Descended to the drowned steps" },
217
+ },
218
+ "You take the drowned steps down, and the water closes over your boots.",
219
+ ],
220
+ },
221
+ );
222
+
223
+ test(
224
+ "both tracks empty ends the story, and the ending sticks",
225
+ async ({ session }) => {
226
+ await session.say(
227
+ "Dark fantasy, melancholic. My character is Sera, a burned-out lamplighter. Begin.",
228
+ );
229
+
230
+ const down = await session.say(
231
+ "## Correction. In that last scene I lost everything: my health is zero and my " +
232
+ "spirit is zero. Sync the state to match.",
233
+ );
234
+ const downed = answerOf(down, "update_state", Settled);
235
+ // The flag is DERIVED where it is reported. `gameOver` is written by the
236
+ // slot's `after` hook, which runs only once this body has returned — so a
237
+ // tool that copied `state.gameOver` into its own result would report the
238
+ // value from BEFORE the update that emptied the tracks, and would report
239
+ // `false` right here.
240
+ expect(downed.result.gameOver).toBe(true);
241
+ // And the flag is what the transition reads: `sendFrom` turns it into
242
+ // DOWNED, whose target is a `final` state.
243
+ expect(downed.state).toBe("gameOver");
244
+ expect(downed.done).toBe(true);
245
+
246
+ // And the ending sticks. `gameOver` is `final`, so nothing delivered to
247
+ // the flow can move it — a later turn still reads a finished story, which
248
+ // is what an `on: { SETUP }` on that state used to make look untrue.
249
+ const asked = await session.say("Check the state. Is the story over?");
250
+ const ended = answerOf(asked, "check_state", Status);
251
+ expect(ended.state).toBe("gameOver");
252
+ expect(ended.done).toBe(true);
253
+ expect(ended.gameOver).toBe(true);
254
+ },
255
+ {
256
+ stubReply: [
257
+ { tool: "setup_character", args: SERA },
258
+ "The last lamp on the north road has gone out. What do you do, Sera?",
259
+ { tool: "update_state", args: { health: 0, spirit: 0, logEntry: "Sera's lamp goes out" } },
260
+ "Your hands stop shaking, because there is nothing left in them.",
261
+ { tool: "check_state" },
262
+ "The story is over. Say the word and we will begin another.",
263
+ ],
264
+ },
265
+ );
266
+
267
+ test(
268
+ "an ended story refuses to be played",
269
+ async ({ session }) => {
270
+ await session.say(
271
+ "Dark fantasy, melancholic. My character is Sera, a burned-out lamplighter. Begin.",
272
+ );
273
+ const down = await session.say(
274
+ "## Correction. In that last scene I lost everything: my health is zero and my " +
275
+ "spirit is zero. Sync the state to match.",
276
+ );
277
+ expect(answerOf(down, "update_state", Settled).state).toBe("gameOver");
278
+
279
+ const after = await session.say("I refuse to die. Roll to fight on.");
280
+ const attempts = answersOf(after, "action_roll", Refusal);
281
+ // The gate is only OBSERVABLE if something calls the gated tool, and the
282
+ // script is what guarantees that — hence the exact count. `gameOver` was
283
+ // once a flag nothing acted on, so a player with both tracks empty could
284
+ // keep rolling for as long as they liked; what must never happen is a roll
285
+ // that RESOLVES.
286
+ expect(attempts).toHaveLength(1);
287
+ for (const attempt of attempts) {
288
+ expect(attempt.error).toMatch(/not available yet/i);
289
+ expect(attempt.error).toContain("gameOver");
290
+ }
291
+ expect(after.completed).toBe(true);
292
+ },
293
+ // Scripted only, and `{ scripted: true }` rather than a loop that accepts
294
+ // zero attempts: the narrator's own instruction on `gameOver` tells it not
295
+ // to roll, so a live model correctly declines and the refusal is never
296
+ // provoked — which used to leave this claim asserting nothing on the runs
297
+ // that cost money. The honest live version of "it declined" is the case
298
+ // above, which reads the position rather than a refusal.
299
+ {
300
+ scripted: true,
301
+ stubReply: [
302
+ { tool: "setup_character", args: SERA },
303
+ "The last lamp on the north road has gone out. What do you do, Sera?",
304
+ { tool: "update_state", args: { health: 0, spirit: 0, logEntry: "Sera's lamp goes out" } },
305
+ "Your hands stop shaking, because there is nothing left in them.",
306
+ {
307
+ tool: "action_roll",
308
+ args: {
309
+ move: "endure_harm",
310
+ stat: "iron",
311
+ position: "desperate",
312
+ effect: "limited",
313
+ purpose: "fight on with nothing left",
314
+ },
315
+ },
316
+ "There is nothing left to roll for. The hollow goes dark.",
317
+ ],
318
+ },
319
+ );
320
+ });