@alexkroman1/aai-cli 6.11.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/README.md +1 -0
  2. package/dist/{_agent-DpH2pBJd.mjs → _agent-BTwZJu4P.mjs} +2 -2
  3. package/dist/{_bundler-C2BDwU7r.mjs → _bundler-CDuYl5Gb.mjs} +2 -2
  4. package/dist/_dev-env.d.ts +22 -2
  5. package/dist/{_dev-server-LLLWnFBs.mjs → _dev-server-Me06wgeF.mjs} +38 -47
  6. package/dist/_dev-server.d.ts +0 -17
  7. package/dist/{_init-C57bWAQ8.mjs → _init-BhEfQ6Yi.mjs} +3 -2
  8. package/dist/{_preflight-BtfaYtbE.mjs → _preflight-Co1G9Jww.mjs} +1 -1
  9. package/dist/_preflight.d.ts +1 -1
  10. package/dist/_resource-commands.d.ts +23 -0
  11. package/dist/{_slug-api-BxrsJXVI.mjs → _slug-api-z-USxPtL.mjs} +16 -12
  12. package/dist/_slug-api.d.ts +5 -2
  13. package/dist/{_studio-U3jhKlA3.mjs → _studio-DR5yzZ1T.mjs} +2 -1
  14. package/dist/{_templates-Dxb_P2Wz.mjs → _templates-4WcKOjS5.mjs} +5 -6
  15. package/dist/_templates.d.ts +3 -4
  16. package/dist/_workflow-scan.d.ts +108 -0
  17. package/dist/{build-Dfza2pRI.mjs → build-BBxf8vom.mjs} +6 -3
  18. package/dist/cli.mjs +346 -349
  19. package/dist/{client-bundler-BEIqgOtd.mjs → client-bundler-CWnG42cU.mjs} +2 -1
  20. package/dist/client-bundler.mjs +1 -1
  21. package/dist/{delete-vbpjGzqs.mjs → delete-Dl_LSfvI.mjs} +3 -3
  22. package/dist/{deploy-B8lubiRT.mjs → deploy-DRC8jWr8.mjs} +5 -5
  23. package/dist/{dev-Uq5ujP8a.mjs → dev-BvcctCn2.mjs} +1 -1
  24. package/dist/eval-DfQ8cm5p.mjs +56 -0
  25. package/dist/eval.d.ts +43 -0
  26. package/dist/{init-l6xfU4xX.mjs → init-CUo87gcI.mjs} +3 -3
  27. package/dist/{login-C71-qz8F.mjs → login-BBuM1sxH.mjs} +3 -4
  28. package/dist/{logs-BWVGSjLR.mjs → logs-CQ2T46dr.mjs} +2 -2
  29. package/dist/project-config.mjs +1 -1
  30. package/dist/scaffold/CLAUDE.md +442 -80
  31. package/dist/scaffold/package.json +6 -3
  32. package/dist/scaffold/server.mjs +11 -4
  33. package/dist/scaffold/tsconfig.json +0 -1
  34. package/dist/{secret-BuMuFR4B.mjs → secret-Ct34kdgU.mjs} +2 -2
  35. package/dist/{storage-DzRZ-eCw.mjs → storage-i_79ZwAR.mjs} +2 -2
  36. package/dist/{studio-BCNUpDgP.mjs → studio-CzTzPqe1.mjs} +4 -4
  37. package/dist/templates/briefing-desk/agent.test.ts +350 -0
  38. package/dist/templates/briefing-desk/agent.ts +30 -0
  39. package/dist/templates/briefing-desk/shared.ts +200 -0
  40. package/dist/templates/briefing-desk/system-prompt.md +39 -0
  41. package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
  42. package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
  43. package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
  44. package/dist/templates/call-audit/agent.eval.test.ts +474 -0
  45. package/dist/templates/call-audit/agent.test.ts +57 -204
  46. package/dist/templates/call-audit/agent.ts +32 -19
  47. package/dist/templates/call-audit/client.tsx +17 -54
  48. package/dist/templates/call-audit/workflows/audit.ts +11 -18
  49. package/dist/templates/call-audit/workflows/ingest.ts +106 -114
  50. package/dist/templates/call-audit/workflows/media.ts +2 -12
  51. package/dist/templates/call-audit/workflows/summarize.ts +47 -52
  52. package/dist/templates/call-audit/workflows/sync-api.ts +7 -7
  53. package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
  54. package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
  55. package/dist/templates/dispatch-center/agent.test.ts +149 -25
  56. package/dist/templates/dispatch-center/client.tsx +239 -129
  57. package/dist/templates/dispatch-center/shared.ts +99 -1
  58. package/dist/templates/dispatch-center/system-prompt.md +3 -1
  59. package/dist/templates/dispatch-center/tools/incident_add_note.ts +16 -13
  60. package/dist/templates/dispatch-center/tools/incident_create.ts +19 -1
  61. package/dist/templates/dispatch-center/tools/incident_escalate.ts +68 -59
  62. package/dist/templates/dispatch-center/tools/incident_triage.ts +57 -43
  63. package/dist/templates/dispatch-center/tools/incident_update_status.ts +49 -40
  64. package/dist/templates/dispatch-center/tools/ops_run_scenario.ts +8 -2
  65. package/dist/templates/dispatch-center/tools/resources_dispatch.ts +69 -52
  66. package/dist/templates/dispatch-center/tools/resources_update_status.ts +45 -41
  67. package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
  68. package/dist/templates/embedded-assets/agent.test.ts +17 -14
  69. package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
  70. package/dist/templates/health-assistant/agent.test.ts +22 -4
  71. package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
  72. package/dist/templates/infocom-adventure/agent.test.ts +53 -24
  73. package/dist/templates/infocom-adventure/agent.ts +21 -0
  74. package/dist/templates/infocom-adventure/client.tsx +246 -164
  75. package/dist/templates/infocom-adventure/shared.ts +20 -9
  76. package/dist/templates/infocom-adventure/system-prompt.md +1 -1
  77. package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
  78. package/dist/templates/link-digest/agent.eval.test.ts +195 -0
  79. package/dist/templates/link-digest/agent.test.ts +24 -19
  80. package/dist/templates/link-digest/client.tsx +47 -61
  81. package/dist/templates/link-digest/workflows/digest.ts +19 -31
  82. package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
  83. package/dist/templates/night-owl/agent.eval.test.ts +272 -0
  84. package/dist/templates/night-owl/agent.test.ts +70 -19
  85. package/dist/templates/night-owl/agent.ts +5 -0
  86. package/dist/templates/night-owl/client.tsx +56 -56
  87. package/dist/templates/night-owl/shared.ts +24 -0
  88. package/dist/templates/night-owl/tools/recommend.ts +18 -6
  89. package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
  90. package/dist/templates/personal-finance/system-prompt.md +4 -0
  91. package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
  92. package/dist/templates/pipeline-simple/agent.ts +2 -2
  93. package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
  94. package/dist/templates/pizza-ordering/agent.test.ts +42 -27
  95. package/dist/templates/pizza-ordering/client.tsx +9 -26
  96. package/dist/templates/pizza-ordering/shared.ts +20 -1
  97. package/dist/templates/pizza-ordering/system-prompt.md +6 -0
  98. package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
  99. package/dist/templates/plan-and-execute/agent.test.ts +99 -58
  100. package/dist/templates/plan-and-execute/agent.ts +1 -1
  101. package/dist/templates/plan-and-execute/client.tsx +12 -15
  102. package/dist/templates/plan-and-execute/shared.ts +71 -2
  103. package/dist/templates/plan-and-execute/tools/plan_status.ts +18 -6
  104. package/dist/templates/plan-and-execute/tools/revise_plan.ts +18 -10
  105. package/dist/templates/plan-and-execute/tools/start_plan.ts +13 -2
  106. package/dist/templates/plan-and-execute/tools/work_next_step.ts +70 -22
  107. package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
  108. package/dist/templates/podcast-digest/agent.test.ts +717 -0
  109. package/dist/templates/podcast-digest/agent.ts +139 -0
  110. package/dist/templates/podcast-digest/client.tsx +154 -0
  111. package/dist/templates/podcast-digest/workflows/digest.ts +444 -0
  112. package/dist/templates/podcast-digest/workflows/feeds.ts +507 -0
  113. package/dist/templates/podcast-digest/workflows/slack.ts +88 -0
  114. package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
  115. package/dist/templates/recap-workflow/agent.test.ts +97 -94
  116. package/dist/templates/recap-workflow/tools/recap_progress.ts +9 -12
  117. package/dist/templates/recap-workflow/tools/recap_status.ts +8 -9
  118. package/dist/templates/recap-workflow/tools/request_recap.ts +2 -1
  119. package/dist/templates/recap-workflow/workflows/recap.ts +36 -40
  120. package/dist/templates/redline/agent.eval.test.ts +209 -0
  121. package/dist/templates/redline/agent.test.ts +17 -15
  122. package/dist/templates/redline/client.tsx +12 -12
  123. package/dist/templates/redline/workflows/redline.ts +19 -31
  124. package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
  125. package/dist/templates/research-workflow/agent.test.ts +60 -59
  126. package/dist/templates/research-workflow/tools/research_progress.ts +9 -12
  127. package/dist/templates/research-workflow/tools/research_status.ts +9 -11
  128. package/dist/templates/research-workflow/workflows/research.ts +44 -61
  129. package/dist/templates/retail/address.ts +87 -1
  130. package/dist/templates/retail/agent.eval.test.ts +314 -0
  131. package/dist/templates/retail/agent.test.ts +195 -112
  132. package/dist/templates/retail/agent.ts +35 -1
  133. package/dist/templates/retail/cancel.ts +116 -0
  134. package/dist/templates/retail/client.tsx +265 -117
  135. package/dist/templates/retail/payment.ts +107 -0
  136. package/dist/templates/retail/pending.test.ts +196 -0
  137. package/dist/templates/retail/pending.ts +137 -0
  138. package/dist/templates/retail/registry.test.ts +150 -7
  139. package/dist/templates/retail/returns.ts +102 -0
  140. package/dist/templates/retail/shared.test.ts +42 -0
  141. package/dist/templates/retail/shared.ts +35 -0
  142. package/dist/templates/retail/store.test.ts +84 -15
  143. package/dist/templates/retail/store.ts +231 -48
  144. package/dist/templates/retail/swap.test.ts +4 -4
  145. package/dist/templates/retail/swap.ts +248 -15
  146. package/dist/templates/retail/system-prompt.md +69 -28
  147. package/dist/templates/retail/tools/cancel_change.ts +33 -0
  148. package/dist/templates/retail/tools/cancel_pending_order.ts +16 -60
  149. package/dist/templates/retail/tools/confirm_change.ts +42 -0
  150. package/dist/templates/retail/tools/exchange_delivered_order_items.ts +22 -67
  151. package/dist/templates/retail/tools/find_user_id_by_email.ts +8 -5
  152. package/dist/templates/retail/tools/find_user_id_by_name_zip.ts +4 -5
  153. package/dist/templates/retail/tools/get_item_details.ts +3 -4
  154. package/dist/templates/retail/tools/get_order_details.ts +3 -4
  155. package/dist/templates/retail/tools/get_product_details.ts +3 -4
  156. package/dist/templates/retail/tools/get_user_details.ts +2 -2
  157. package/dist/templates/retail/tools/list_all_product_types.ts +5 -5
  158. package/dist/templates/retail/tools/modify_pending_order_address.ts +13 -25
  159. package/dist/templates/retail/tools/modify_pending_order_items.ts +23 -62
  160. package/dist/templates/retail/tools/modify_pending_order_payment.ts +16 -78
  161. package/dist/templates/retail/tools/modify_user_address.ts +13 -16
  162. package/dist/templates/retail/tools/return_delivered_order_items.ts +17 -72
  163. package/dist/templates/retail/tools/transfer_to_human_agents.ts +12 -6
  164. package/dist/templates/simple/agent.eval.test.ts +54 -0
  165. package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
  166. package/dist/templates/solo-rpg/agent.test.ts +227 -48
  167. package/dist/templates/solo-rpg/client.tsx +38 -37
  168. package/dist/templates/solo-rpg/shared.ts +145 -19
  169. package/dist/templates/solo-rpg/system-prompt.md +3 -2
  170. package/dist/templates/solo-rpg/tools/action_roll.ts +82 -63
  171. package/dist/templates/solo-rpg/tools/burn_momentum.ts +85 -50
  172. package/dist/templates/solo-rpg/tools/check_state.ts +24 -7
  173. package/dist/templates/solo-rpg/tools/load_game.ts +13 -1
  174. package/dist/templates/solo-rpg/tools/save_game.ts +16 -7
  175. package/dist/templates/solo-rpg/tools/setup_character.ts +22 -2
  176. package/dist/templates/solo-rpg/tools/update_state.ts +113 -100
  177. package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
  178. package/dist/templates/spoken-summary/agent.test.ts +68 -97
  179. package/dist/templates/spoken-summary/agent.ts +30 -17
  180. package/dist/templates/spoken-summary/client.tsx +10 -50
  181. package/dist/templates/spoken-summary/workflows/summarize.ts +17 -22
  182. package/dist/templates/spoken-summary/workflows/transcribe.ts +19 -26
  183. package/dist/templates/support-line/agent.eval.test.ts +273 -0
  184. package/dist/templates/support-line/agent.test.ts +11 -16
  185. package/dist/templates/support-line/agent.ts +1 -1
  186. package/dist/templates/support-line/client.tsx +9 -9
  187. package/dist/templates/support-line/nodes.ts +100 -0
  188. package/dist/templates/support-line/procedure.ts +407 -0
  189. package/dist/templates/support-line/system-prompt.md +2 -1
  190. package/dist/templates/support-line/tools/answer_question.ts +66 -4
  191. package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
  192. package/dist/templates/transcription-workflow/agent.test.ts +94 -194
  193. package/dist/templates/transcription-workflow/agent.ts +1 -1
  194. package/dist/templates/transcription-workflow/client.tsx +17 -42
  195. package/dist/templates/transcription-workflow/workflows/batch.ts +19 -22
  196. package/dist/templates/transcription-workflow/workflows/normalize.ts +90 -186
  197. package/dist/templates/transcription-workflow/workflows/stitch.ts +0 -11
  198. package/dist/templates/transcription-workflow/workflows/stream.ts +4 -5
  199. package/dist/templates/transcription-workflow/workflows/sync-api.ts +6 -6
  200. package/dist/templates/transcription-workflow/workflows/transcribe.ts +32 -26
  201. package/dist/templates/transcription-workflow/workflows/wav.ts +0 -31
  202. package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
  203. package/dist/templates/travel-concierge/agent.test.ts +174 -43
  204. package/dist/templates/travel-concierge/agent.ts +18 -9
  205. package/dist/templates/travel-concierge/client.tsx +11 -23
  206. package/dist/templates/travel-concierge/routing.ts +34 -15
  207. package/dist/templates/travel-concierge/shared.ts +180 -12
  208. package/dist/templates/travel-concierge/system-prompt.md +7 -1
  209. package/dist/templates/travel-concierge/tools/book_car_rental.ts +8 -5
  210. package/dist/templates/travel-concierge/tools/book_excursion.ts +8 -5
  211. package/dist/templates/travel-concierge/tools/book_hotel.ts +8 -5
  212. package/dist/templates/travel-concierge/tools/cancel_ticket.ts +8 -5
  213. package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
  214. package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
  215. package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
  216. package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
  217. package/dist/templates/travel-concierge/tools/update_ticket.ts +8 -5
  218. package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
  219. package/dist/templates/web-researcher/system-prompt.md +15 -4
  220. package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
  221. package/dist/test.d.ts +29 -5
  222. package/dist/{worker-bundler-CGD4r8Kc.mjs → worker-bundler-CCVEDjm1.mjs} +241 -3
  223. package/dist/worker-bundler.mjs +1 -1
  224. package/dist/{workflow-CFpxOFfQ.mjs → workflow-BflATMmU.mjs} +47 -12
  225. package/dist/workflow-bundler.d.ts +6 -1
  226. package/dist/workflow.d.ts +1 -1
  227. package/package.json +5 -4
  228. package/dist/eject-31gjtaHF.mjs +0 -116
  229. package/dist/eject.d.ts +0 -60
  230. package/dist/templates/call-audit/workflows/temp-media.ts +0 -138
  231. package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
  232. package/dist/templates/support-line/graph.ts +0 -224
  233. package/dist/{_config-D_s09e7g.mjs → _config-CmJOFsAP.mjs} +1 -1
  234. /package/dist/templates/plan-and-execute/{graph.ts → procedure.ts} +0 -0
@@ -1,6 +1,26 @@
1
- import { errorMessage, isToolFailure, tool, toolFailure } from "@alexkroman1/aai";
2
- import { executeStep, replanNode } from "../graph.ts";
3
- import { liveSearch, noteRevision, type PastStep, planSlot, recordStep } from "../shared.ts";
1
+ import { errorMessage, type ToolFailure, toolFailure } from "@alexkroman1/aai";
2
+ import { executeStep, replanNode } from "../procedure.ts";
3
+ import {
4
+ liveSearch,
5
+ noteRevision,
6
+ type PastStep,
7
+ planFlow,
8
+ planSlot,
9
+ recordStep,
10
+ } from "../shared.ts";
11
+
12
+ /**
13
+ * What the claim window decided, as a DISCRIMINATED union.
14
+ *
15
+ * The three arms used to be told apart by `"done" in claimed`, which stopped
16
+ * working once the outcome had a declared type: union normalization gives every
17
+ * arm the other arms' keys as `?: never`, and `in` cannot rule out a key that is
18
+ * optional-never. A `kind` tag is both narrower and easier to read.
19
+ */
20
+ type Claim =
21
+ | { kind: "failed"; failure: ToolFailure }
22
+ | { kind: "dry" }
23
+ | { kind: "step"; step: string; objective: string; pastSteps: PastStep[] };
4
24
 
5
25
  /**
6
26
  * One turn of their execute→replan loop: do the head step, then let the
@@ -22,33 +42,54 @@ import { liveSearch, noteRevision, type PastStep, planSlot, recordStep } from ".
22
42
  * behind each other to get the same guarantee. A step whose work then FAILS is
23
43
  * put back, because a failed model call is worth retrying and a silently dropped
24
44
  * step leaves the caller with a plan that skipped something.
45
+ *
46
+ * **`when: "working"` replaced two hand-rolled guards** — the `!plan.objective`
47
+ * check and the already-answered one — and neither was a data check: both asked
48
+ * where the conversation was. See {@link planFlow}. The flow is what refuses now,
49
+ * naming the state and quoting its instruction, so this body only ever runs when
50
+ * there is a plan in progress.
51
+ *
52
+ * **`ANSWERED` is sent only when the replanner produced a RESPONSE.** A plan
53
+ * that merely ran out of steps stays in `working`: nothing was concluded, and
54
+ * the caller can still revise or add to it. Conflating the two would announce an
55
+ * answer that does not exist.
25
56
  */
26
- export default tool({
57
+ export default planFlow.tool({
27
58
  description:
28
59
  "Do the next step of the plan and report what it found. Call this once per " +
29
60
  "step — never in a loop. Say a short 'let me look into that' first, since " +
30
61
  "the step may take a few seconds.",
62
+ when: "working",
31
63
  async execute(_args, ctx) {
32
64
  // The whole read-and-claim, in one window nothing can interleave with.
33
- const claimed = planSlot.update(ctx, (plan) => {
34
- if (!plan.objective) return toolFailure("There is no plan yet use start_plan first.");
35
- if (plan.response) {
36
- return {
37
- done: true as const,
38
- response: plan.response,
39
- message: "The plan is already finished.",
40
- };
65
+ const claimed = planSlot.update(ctx, (plan): Claim => {
66
+ // `when: "working"` means there IS a plan, so this arm is unreachable by
67
+ // the flow's own guarantee — kept because the slot and the flow are two
68
+ // values, and a plan cleared by something else should refuse rather than
69
+ // claim a step off an empty objective.
70
+ if (!plan.objective) {
71
+ return { kind: "failed", failure: toolFailure("There is no plan yet — use start_plan.") };
41
72
  }
42
73
  const step = plan.plan.shift();
43
- if (!step) {
44
- return {
45
- done: true as const,
46
- message: "No steps are left. Ask the caller what they want next.",
47
- };
48
- }
49
- return { step, objective: plan.objective, pastSteps: [...plan.pastSteps] as PastStep[] };
74
+ if (!step) return { kind: "dry" };
75
+ return {
76
+ kind: "step",
77
+ step,
78
+ objective: plan.objective,
79
+ pastSteps: [...plan.pastSteps] as PastStep[],
80
+ };
50
81
  });
51
- if (isToolFailure(claimed) || "done" in claimed) return claimed;
82
+ if (claimed.kind === "failed") return claimed.failure;
83
+ if (claimed.kind === "dry") {
84
+ // `response: undefined` is spelled out rather than omitted: `sendFrom`
85
+ // below reads that field to decide whether to send ANSWERED, and a key
86
+ // missing from one arm of a union is not readable on the union at all.
87
+ return {
88
+ finished: true,
89
+ response: undefined,
90
+ message: "No steps are left. Ask the caller what they want next.",
91
+ };
92
+ }
52
93
  const { step, objective, pastSteps } = claimed;
53
94
 
54
95
  try {
@@ -69,7 +110,7 @@ export default tool({
69
110
  plan.plan = [];
70
111
  noteRevision(plan, `Finished after ${plan.pastSteps.length} step(s)`);
71
112
  return {
72
- done: true,
113
+ finished: true,
73
114
  step,
74
115
  result: outcome.result,
75
116
  searches: outcome.searches,
@@ -82,7 +123,7 @@ export default tool({
82
123
  plan.plan = act.steps;
83
124
  if (changed) noteRevision(plan, `Replanned to ${act.steps.length} step(s) after: ${step}`);
84
125
  return {
85
- done: false,
126
+ finished: false,
86
127
  step,
87
128
  result: outcome.result,
88
129
  searches: outcome.searches,
@@ -98,4 +139,11 @@ export default tool({
98
139
  return toolFailure(`That step could not be worked: ${errorMessage(err)}`);
99
140
  }
100
141
  },
142
+ // Written BELOW `execute` deliberately. `sendFrom`'s parameter is
143
+ // `Exclude<NoInfer<R>, ToolFailure>`, and `NoInfer` keeps it from bidding on
144
+ // `R` — but this body's own return type is itself an inference
145
+ // (`planSlot.update`'s), so with `sendFrom` first there is no candidate to
146
+ // contextually type it against and the parameter lands as `unknown`.
147
+ sendFrom: (outcome) =>
148
+ outcome.response === undefined ? undefined : ({ type: "ANSWERED" } as const),
101
149
  });
@@ -0,0 +1,411 @@
1
+ // Copyright 2026 the AAI authors. MIT license.
2
+ // An EVAL for a WORKFLOW APP: does the run actually do the work? Run it with
3
+ // `aai eval`.
4
+ //
5
+ // `agent.test.ts` asserts about the declaration, drives each step on its own,
6
+ // and covers the feed scraping and the Slack rendering as pure functions. This
7
+ // drives the WHOLE BODY — `dailyDigestFlow` from the top — and what it is here
8
+ // to check is the one thing no per-step spec can see: that **the run is the
9
+ // schedule**. There is no cron in this template. A durable `sleep()` between
10
+ // digests IS the scheduler, so "does it post three digests two hours apart"
11
+ // is a question about the BODY's loop and about nothing else.
12
+ //
13
+ // EVERY CASE HERE IS SCRIPTED IN BOTH MODES, which is unusual and deliberate.
14
+ // Two reasons, and neither is convenience:
15
+ //
16
+ // * **Every leg of this run is HTTP through one slot.** The feeds, the two
17
+ // transcription calls, the model and the Slack post all go through
18
+ // `stepFetch`, and publishing a fake REPLACES — so there is no arrangement
19
+ // in which some legs are real and the Slack post is not. A live run would
20
+ // have to POST a summarized digest to a real Slack workspace, which a
21
+ // template eval may not do.
22
+ // * **The claims are arithmetic.** How many sleeps a run of three digests
23
+ // asks for, which episode was polled how many times, what a digest says
24
+ // about an episode nobody could transcribe — a live provider can neither
25
+ // confirm nor deny any of them. It can only be asked and then have its
26
+ // answer accepted, which is not evidence.
27
+ //
28
+ // So a LIVE run of this file makes no provider call and costs nothing. That is
29
+ // the honest report rather than a gap being papered over: what a live run would
30
+ // add here is a real transcript, and `spoken-summary` and `call-audit` both
31
+ // measure that against a real recording already.
32
+ //
33
+ // WHAT NO EVAL HERE COVERS: durability — which for THIS template is most of
34
+ // what it is for. Imported through vitest with no bundler in the path, a
35
+ // `"use workflow"` body is an ordinary async function, so the multi-day
36
+ // suspension that makes a digest arrive tomorrow is not exercised; the sleep is
37
+ // RECORDED and skipped. `run.slept` below is that admission written as an
38
+ // assertion, and it is the only way to check a seven-day schedule without
39
+ // waiting a week. `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier that
40
+ // really suspends and resumes a run.
41
+ import { TRANSCRIBE_API } from "@alexkroman1/aai/step";
42
+ import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
43
+ import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
44
+ import { expect } from "vitest";
45
+ import agentDef, { dailyDigest } from "./agent.ts";
46
+ import { MAX_POLL_ATTEMPTS, POLL_DELAY, scheduleIntervalMs } from "./workflows/digest.ts";
47
+
48
+ /** The feed every case reads. Not a real host — nothing here leaves the process. */
49
+ const FEED_URL = "https://feeds.example.test/rebuild.xml";
50
+
51
+ /** A classic incoming webhook, which takes Block Kit. */
52
+ const SLACK_WEBHOOK = "https://hooks.slack.com/services/T00000000/B00000000/eval";
53
+
54
+ /**
55
+ * The three episodes, newest first — and each transcript carries a SENTINEL
56
+ * word nothing else in the run produces.
57
+ *
58
+ * That is what makes "the right transcript reached the right episode" an
59
+ * assertion rather than a hope: the scripted model echoes whichever sentinel it
60
+ * was shown, so a batch that crossed two episodes' transcripts shows up as the
61
+ * wrong word in the wrong digest entry. With N jobs in flight finishing out of
62
+ * order, that is exactly the mistake worth catching.
63
+ */
64
+ const EPISODES = [
65
+ {
66
+ title: "The migration nobody owns",
67
+ audioUrl: "https://cdn.example.test/rebuild-13.mp3",
68
+ published: "Wed, 13 Aug 2026 09:00:00 GMT",
69
+ sentinel: "quokka",
70
+ },
71
+ {
72
+ title: "Cutting the release train in half",
73
+ audioUrl: "https://cdn.example.test/rebuild-12.mp3",
74
+ published: "Tue, 12 Aug 2026 09:00:00 GMT",
75
+ sentinel: "narwhal",
76
+ },
77
+ {
78
+ title: "Hiring for the platform team",
79
+ audioUrl: "https://cdn.example.test/rebuild-11.mp3",
80
+ published: "Mon, 11 Aug 2026 09:00:00 GMT",
81
+ sentinel: "pangolin",
82
+ },
83
+ ] as const;
84
+
85
+ /** A podcast RSS document: an `<rss>` root and an `<enclosure url=>` per item. */
86
+ const FEED_XML = `<?xml version="1.0" encoding="UTF-8"?>
87
+ <rss version="2.0"><channel>
88
+ <title>The Rebuild</title>
89
+ ${EPISODES.map(
90
+ (episode) => `<item>
91
+ <title>${episode.title}</title>
92
+ <link>https://example.test/${episode.sentinel}</link>
93
+ <guid isPermaLink="false">${episode.sentinel}</guid>
94
+ <pubDate>${episode.published}</pubDate>
95
+ <enclosure url="${episode.audioUrl}" type="audio/mpeg" length="1234"/>
96
+ </item>`,
97
+ ).join("\n")}
98
+ </channel></rss>`;
99
+
100
+ /** How one episode's transcription behaves. */
101
+ type EpisodeScript = {
102
+ /** Polls that answer "still working" before this job completes. Defaults to 0. */
103
+ pendingPolls?: number;
104
+ /** Refuse the SUBMIT with this status instead. A 4xx is terminal. */
105
+ submitStatus?: number;
106
+ };
107
+
108
+ /**
109
+ * Answer the whole world in memory: the feed, the two transcription calls, the
110
+ * model, and Slack.
111
+ *
112
+ * ONE handler, because publishing a `stepFetch` REPLACES — and this run has
113
+ * four different far sides, so routing by URL is the only shape available.
114
+ *
115
+ * **The transcription legs are hand-routed here rather than handed to
116
+ * `installStubTranscribe`, and that is a gap rather than a preference.** That
117
+ * fake counts `pendingPolls` GLOBALLY and stages a `failure` per LEG, so
118
+ * neither "these two episodes finish on different rounds" nor "this ONE episode
119
+ * is broken" is expressible through it — and both are this template's subject,
120
+ * it being the only one that has a BATCH of jobs in flight. The URLs are built
121
+ * from the SDK's own `TRANSCRIBE_API` constant, so a case still cannot pass
122
+ * because the fake and the step agree on a typo.
123
+ */
124
+ function scriptWorld(
125
+ scripts: Readonly<Record<string, EpisodeScript>> = {},
126
+ slack: { status?: number; body?: unknown } = {},
127
+ ) {
128
+ /** Minted job id → the audio URL it was submitted for. */
129
+ const jobs = new Map<string, string>();
130
+ /** Job id → polls answered so far. */
131
+ const polls = new Map<string, number>();
132
+ let minted = 0;
133
+
134
+ return installStubStepFetch((request) => {
135
+ if (request.url === FEED_URL) return { body: FEED_XML };
136
+
137
+ if (request.url === `${TRANSCRIBE_API}/v2/transcript` && request.method === "POST") {
138
+ const audioUrl = String(
139
+ (JSON.parse(String(request.body ?? "{}")) as { audio_url?: string }).audio_url,
140
+ );
141
+ const script = scripts[audioUrl] ?? {};
142
+ if (script.submitStatus !== undefined) {
143
+ return { status: script.submitStatus, body: { error: `no audio at ${audioUrl}` } };
144
+ }
145
+ minted += 1;
146
+ const id = `job_${minted}`;
147
+ jobs.set(id, audioUrl);
148
+ return { body: { id } };
149
+ }
150
+
151
+ if (request.url.startsWith(`${TRANSCRIBE_API}/v2/transcript/`)) {
152
+ const id = request.url.slice(request.url.lastIndexOf("/") + 1);
153
+ const seen = polls.get(id) ?? 0;
154
+ polls.set(id, seen + 1);
155
+ const audioUrl = jobs.get(id) ?? "";
156
+ if (seen < (scripts[audioUrl]?.pendingPolls ?? 0)) return { body: { status: "processing" } };
157
+ const episode = EPISODES.find((one) => one.audioUrl === audioUrl);
158
+ return {
159
+ body: {
160
+ status: "completed",
161
+ text: `The hosts spent the whole hour on the ${episode?.sentinel ?? "unknown"}.`,
162
+ audio_duration: 1800,
163
+ },
164
+ };
165
+ }
166
+
167
+ if (request.url.includes("/chat/completions")) {
168
+ // The reply ECHOES whichever sentinel the prompt carried, which is what
169
+ // makes the transcript→episode correlation assertable end to end.
170
+ const prompt = String(request.body ?? "");
171
+ const seen = EPISODES.find((one) => prompt.includes(one.sentinel))?.sentinel ?? "nothing";
172
+ return {
173
+ body: {
174
+ choices: [
175
+ {
176
+ message: {
177
+ content: JSON.stringify({
178
+ summary: `An hour about the ${seen}.`,
179
+ keyPoints: [`The ${seen} is the decision`, "Nobody owns the follow-up"],
180
+ }),
181
+ },
182
+ },
183
+ ],
184
+ },
185
+ };
186
+ }
187
+
188
+ if (request.url.startsWith("https://hooks.slack.com/")) {
189
+ return { status: slack.status ?? 200, body: slack.body ?? "ok" };
190
+ }
191
+
192
+ return { status: 404, body: { error: `no route for ${request.method} ${request.url}` } };
193
+ });
194
+ }
195
+
196
+ /** The bodies of every Slack post the run made, parsed. */
197
+ function slackPosts(world: ReturnType<typeof scriptWorld>): Record<string, unknown>[] {
198
+ return world.calls
199
+ .filter((call) => call.url.startsWith("https://hooks.slack.com/"))
200
+ .map((call) => JSON.parse(String(call.body ?? "{}")) as Record<string, unknown>);
201
+ }
202
+
203
+ /** How many times the job for `audioUrl` was polled. */
204
+ function pollsFor(world: ReturnType<typeof scriptWorld>, audioUrl: string): number {
205
+ const submitted = world.calls.filter(
206
+ (call) => call.url === `${TRANSCRIBE_API}/v2/transcript` && call.method === "POST",
207
+ );
208
+ const at = submitted.findIndex((call) => String(call.body ?? "").includes(audioUrl));
209
+ if (at < 0) return 0;
210
+ return world.calls.filter((call) => call.url.endsWith(`/v2/transcript/job_${at + 1}`)).length;
211
+ }
212
+
213
+ /** The whole input, with only the fields a case cares about spelled out. */
214
+ function input(overrides: Partial<Parameters<typeof dailyDigest.run>[0]> = {}) {
215
+ return {
216
+ podcastChannels: FEED_URL,
217
+ slackWebhookUrl: SLACK_WEBHOOK,
218
+ slackWorkflowTextParam: "text",
219
+ maxEpisodesPerDigest: 2,
220
+ intervalEvery: 1,
221
+ intervalUnit: "days" as const,
222
+ daysToRun: 1,
223
+ ...overrides,
224
+ };
225
+ }
226
+
227
+ describeWorkflowEval(agentDef, (test) => {
228
+ test("the RUN is the schedule: N digests, N-1 recorded sleeps, no cron", async ({ app }) => {
229
+ // The case this template exists for. Three digests two hours apart is
230
+ // ordinarily six hours of wall clock; here the durable waits are RECORDED
231
+ // rather than taken, which is the only way to assert a schedule at all.
232
+ const world = scriptWorld();
233
+
234
+ const run = await app.run(
235
+ dailyDigest,
236
+ input({ daysToRun: 3, intervalEvery: 2, intervalUnit: "hours" }),
237
+ );
238
+
239
+ // The error FIRST, so a failed run names its own reason instead of
240
+ // reporting "expected 'failed' to be 'completed'".
241
+ expect(run.error).toBeUndefined();
242
+ expect(run.status).toBe("completed");
243
+ const output = run.output;
244
+ if (output === undefined) expect.fail("a completed run must carry an output");
245
+
246
+ // THE INVARIANT: one sleep BETWEEN digests and none after the last, because
247
+ // a run that has delivered everything it owes should end rather than sleep
248
+ // for two hours and then end.
249
+ const interval = scheduleIntervalMs(2, "hours");
250
+ expect(run.slept).toEqual([{ duration: interval }, { duration: interval }]);
251
+ expect(run.slept).toHaveLength(output.digestsScheduled - 1);
252
+ // And no poll waits are mixed in: every job finished on its first poll, so
253
+ // every recorded sleep above is a SCHEDULE sleep.
254
+ expect(run.slept.every((one) => one.duration === interval)).toBe(true);
255
+
256
+ expect(output.digestsScheduled).toBe(3);
257
+ expect(output.digestsSent).toBe(3);
258
+ expect(output.scheduleInterval).toBe("2 hours");
259
+ expect(output.deliveryTarget).toBe("Slack webhook");
260
+
261
+ // Three posts, numbered, each a Block Kit payload with the notification
262
+ // line an incoming webhook needs.
263
+ const posts = slackPosts(world);
264
+ expect(posts).toHaveLength(3);
265
+ expect(posts.map((post) => post.text)).toEqual([
266
+ "Podcast digest 1/3: 2 episode summaries",
267
+ "Podcast digest 2/3: 2 episode summaries",
268
+ "Podcast digest 3/3: 2 episode summaries",
269
+ ]);
270
+ for (const post of posts) expect(Array.isArray(post.blocks)).toBe(true);
271
+
272
+ // The last digest is what the page renders, and its clock came from a STEP
273
+ // — a `new Date()` in the body would answer differently on every replay.
274
+ expect(output.lastDigest?.slackStatus).toBe("ok");
275
+ expect(Number.isFinite(Date.parse(output.lastDigest?.sentAt ?? ""))).toBe(true);
276
+ // Two episodes, newest first, and each carrying ITS OWN transcript.
277
+ expect(output.lastDigest?.episodes.map((one) => one.title)).toEqual([
278
+ EPISODES[0].title,
279
+ EPISODES[1].title,
280
+ ]);
281
+ expect(output.lastDigest?.episodes[0]?.summary).toContain(EPISODES[0].sentinel);
282
+ expect(output.lastDigest?.episodes[1]?.summary).toContain(EPISODES[1].sentinel);
283
+
284
+ // The feed is re-read once per digest, which is what makes a repeating run
285
+ // pick up what is NEW rather than re-posting yesterday's list.
286
+ expect(world.calls.filter((call) => call.url === FEED_URL)).toHaveLength(3);
287
+ expect(run.reported.filter((line) => line === "Finding recent podcast episodes.")).toHaveLength(
288
+ 3,
289
+ );
290
+ });
291
+
292
+ test("a finished episode is never polled again, so the slow one holds nothing up", async ({
293
+ app,
294
+ }) => {
295
+ // The batch poll loop, which is this template's one genuinely new mechanism:
296
+ // N episodes in flight finish out of ORDER, so `pending` has to SHRINK. A
297
+ // loop that waited for all N every round would poll the finished episode
298
+ // four times too — cheap here and, with `maxEpisodesPerDigest` up to 20 and
299
+ // a twenty-second wait per round, the difference between a digest arriving
300
+ // and a digest timing out.
301
+ //
302
+ // The NEWEST episode is the slow one deliberately: it makes completion
303
+ // order and publication order disagree, which is the other half of what
304
+ // this case pins.
305
+ const world = scriptWorld({
306
+ [EPISODES[0].audioUrl]: { pendingPolls: 3 },
307
+ [EPISODES[1].audioUrl]: { pendingPolls: 0 },
308
+ });
309
+
310
+ const run = await app.run(dailyDigest, input());
311
+
312
+ expect(run.error).toBeUndefined();
313
+ const output = run.output;
314
+ if (output === undefined) expect.fail("a completed run must carry an output");
315
+
316
+ // The fast episode was polled ONCE and then dropped out of `pending`; the
317
+ // slow one took four rounds. Five polls, not eight.
318
+ expect(pollsFor(world, EPISODES[1].audioUrl)).toBe(1);
319
+ expect(pollsFor(world, EPISODES[0].audioUrl)).toBe(4);
320
+ expect(world.calls.filter((call) => call.url.includes("/v2/transcript/"))).toHaveLength(5);
321
+ // Three waits for four rounds — asked for, and recorded rather than taken.
322
+ expect(run.slept).toEqual([
323
+ { duration: POLL_DELAY },
324
+ { duration: POLL_DELAY },
325
+ { duration: POLL_DELAY },
326
+ ]);
327
+
328
+ // And the digest is in PUBLICATION order, not completion order. The feed is
329
+ // sorted newest-first for a reason, and a reader should not be able to tell
330
+ // which episode the provider happened to finish first.
331
+ expect(output.lastDigest?.episodes.map((one) => one.title)).toEqual([
332
+ EPISODES[0].title,
333
+ EPISODES[1].title,
334
+ ]);
335
+ // Each entry still carries its own transcript, which is the assertion that
336
+ // a batch crossing two episodes' results would fail.
337
+ expect(output.lastDigest?.episodes[0]?.summary).toContain(EPISODES[0].sentinel);
338
+ expect(output.lastDigest?.episodes[1]?.summary).toContain(EPISODES[1].sentinel);
339
+ expect(output.lastDigest?.episodes.every((one) => one.transcriptSource === "assemblyai")).toBe(
340
+ true,
341
+ );
342
+ });
343
+
344
+ test("one broken episode does not sink the digest, and the message says why", async ({ app }) => {
345
+ // The partial-failure policy, driven end to end: a 400 on ONE episode's
346
+ // submit is terminal — the same URL answers the same way on the fourth
347
+ // attempt — so it becomes an `unavailable` VALUE rather than a throw. The
348
+ // digest of the other episode still goes out.
349
+ const world = scriptWorld({ [EPISODES[0].audioUrl]: { submitStatus: 400 } });
350
+
351
+ const run = await app.run(dailyDigest, input());
352
+
353
+ expect(run.error).toBeUndefined();
354
+ expect(run.status).toBe("completed");
355
+ const episodes = run.output?.lastDigest?.episodes ?? [];
356
+ expect(episodes).toHaveLength(2);
357
+
358
+ // The broken one is an ENTRY with a stated reason, not a gap. Four
359
+ // summaries and silence looks like the feed simply had four episodes.
360
+ const broken = episodes[0];
361
+ expect(broken?.title).toBe(EPISODES[0].title);
362
+ expect(broken?.transcriptSource).toBe("unavailable");
363
+ expect(broken?.summary).toContain("could not be transcribed");
364
+ expect(broken?.summary).toContain(EPISODES[0].audioUrl);
365
+ expect(broken?.keyPoints).toEqual(["No transcript was available to summarize."]);
366
+
367
+ // The good one is untouched.
368
+ expect(episodes[1]?.transcriptSource).toBe("assemblyai");
369
+ expect(episodes[1]?.summary).toContain(EPISODES[1].sentinel);
370
+
371
+ // The model was asked about ONE episode, not two — a broken episode costs
372
+ // nothing beyond the submit that refused it.
373
+ expect(world.calls.filter((call) => call.url.includes("/chat/completions"))).toHaveLength(1);
374
+ // And the reason reaches SLACK, which is the only place a reader will see
375
+ // it. A digest that hid the failure is the outcome this refuses.
376
+ expect(JSON.stringify(slackPosts(world)[0])).toContain("could not be transcribed");
377
+ });
378
+
379
+ test("a job that never finishes degrades instead of replaying forever", async ({ app }) => {
380
+ // The poll budget is bounded, and running out of it is NOT an error: a
381
+ // partial digest beats none, and the reason names where the transcript
382
+ // still is. An unbounded loop is the failure this replaces — a run the
383
+ // platform would replay for as long as the provider stayed quiet.
384
+ const world = scriptWorld({
385
+ // Past the budget, so this job is never done.
386
+ [EPISODES[0].audioUrl]: { pendingPolls: MAX_POLL_ATTEMPTS + 1 },
387
+ [EPISODES[1].audioUrl]: { pendingPolls: 0 },
388
+ });
389
+
390
+ const run = await app.run(dailyDigest, input());
391
+
392
+ expect(run.error).toBeUndefined();
393
+ expect(run.status).toBe("completed");
394
+ const episodes = run.output?.lastDigest?.episodes ?? [];
395
+
396
+ const stuck = episodes[0];
397
+ expect(stuck?.transcriptSource).toBe("unavailable");
398
+ // It names the CHECK COUNT and where the transcript can still be read,
399
+ // which is the difference between "we gave up" and "we gave up, here it is".
400
+ expect(stuck?.summary).toContain(`after ${MAX_POLL_ATTEMPTS} checks`);
401
+ expect(stuck?.summary).toContain(`${TRANSCRIBE_API}/v2/transcript/`);
402
+ // The other episode still shipped.
403
+ expect(episodes[1]?.transcriptSource).toBe("assemblyai");
404
+
405
+ // The budget really bounded it: one round per attempt, and the fast episode
406
+ // dropped out after the first.
407
+ expect(pollsFor(world, EPISODES[0].audioUrl)).toBe(MAX_POLL_ATTEMPTS);
408
+ expect(pollsFor(world, EPISODES[1].audioUrl)).toBe(1);
409
+ expect(run.slept).toHaveLength(MAX_POLL_ATTEMPTS);
410
+ });
411
+ });