@alexkroman1/aai-cli 7.0.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +1 -0
  2. package/dist/{_bundler-B4RqNF5Z.mjs → _bundler-CDuYl5Gb.mjs} +1 -1
  3. package/dist/{_dev-server-CiKFH9qw.mjs → _dev-server-Me06wgeF.mjs} +2 -2
  4. package/dist/{_init-CE4HKdgV.mjs → _init-BhEfQ6Yi.mjs} +1 -1
  5. package/dist/_resource-commands.d.ts +23 -0
  6. package/dist/{_slug-api-nRFaBEPJ.mjs → _slug-api-z-USxPtL.mjs} +14 -10
  7. package/dist/_slug-api.d.ts +5 -2
  8. package/dist/{_templates-FI3xPEMj.mjs → _templates-4WcKOjS5.mjs} +4 -5
  9. package/dist/_templates.d.ts +3 -4
  10. package/dist/{build-6jfaakgc.mjs → build-BBxf8vom.mjs} +2 -2
  11. package/dist/cli.mjs +338 -346
  12. package/dist/{deploy-DvYeD8ia.mjs → deploy-DRC8jWr8.mjs} +1 -1
  13. package/dist/{dev-B59JIfWO.mjs → dev-BvcctCn2.mjs} +1 -1
  14. package/dist/eval-DfQ8cm5p.mjs +56 -0
  15. package/dist/eval.d.ts +43 -0
  16. package/dist/{init-CpJVjEg0.mjs → init-CUo87gcI.mjs} +2 -2
  17. package/dist/{logs-9q-psa4q.mjs → logs-CQ2T46dr.mjs} +1 -1
  18. package/dist/scaffold/CLAUDE.md +409 -68
  19. package/dist/scaffold/package.json +5 -4
  20. package/dist/scaffold/server.mjs +10 -3
  21. package/dist/scaffold/tsconfig.json +0 -1
  22. package/dist/{secret-CVvSLIDV.mjs → secret-Ct34kdgU.mjs} +1 -1
  23. package/dist/{storage-BvUrnvM3.mjs → storage-i_79ZwAR.mjs} +1 -1
  24. package/dist/{studio--MUV0cid.mjs → studio-CzTzPqe1.mjs} +1 -1
  25. package/dist/templates/briefing-desk/agent.test.ts +350 -0
  26. package/dist/templates/briefing-desk/agent.ts +30 -0
  27. package/dist/templates/briefing-desk/shared.ts +200 -0
  28. package/dist/templates/briefing-desk/system-prompt.md +39 -0
  29. package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
  30. package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
  31. package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
  32. package/dist/templates/call-audit/agent.eval.test.ts +474 -0
  33. package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
  34. package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
  35. package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
  36. package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
  37. package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
  38. package/dist/templates/infocom-adventure/agent.test.ts +49 -18
  39. package/dist/templates/infocom-adventure/agent.ts +21 -0
  40. package/dist/templates/infocom-adventure/shared.ts +20 -9
  41. package/dist/templates/infocom-adventure/system-prompt.md +1 -1
  42. package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
  43. package/dist/templates/link-digest/agent.eval.test.ts +195 -0
  44. package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
  45. package/dist/templates/night-owl/agent.eval.test.ts +272 -0
  46. package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
  47. package/dist/templates/personal-finance/system-prompt.md +4 -0
  48. package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
  49. package/dist/templates/pipeline-simple/agent.ts +2 -2
  50. package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
  51. package/dist/templates/pizza-ordering/agent.test.ts +15 -0
  52. package/dist/templates/pizza-ordering/shared.ts +20 -1
  53. package/dist/templates/pizza-ordering/system-prompt.md +6 -0
  54. package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
  55. package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
  56. package/dist/templates/podcast-digest/agent.test.ts +30 -59
  57. package/dist/templates/podcast-digest/agent.ts +1 -1
  58. package/dist/templates/podcast-digest/workflows/digest.ts +52 -19
  59. package/dist/templates/podcast-digest/workflows/slack.ts +48 -169
  60. package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
  61. package/dist/templates/redline/agent.eval.test.ts +209 -0
  62. package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
  63. package/dist/templates/retail/address.ts +87 -1
  64. package/dist/templates/retail/agent.eval.test.ts +314 -0
  65. package/dist/templates/retail/agent.test.ts +171 -91
  66. package/dist/templates/retail/agent.ts +35 -1
  67. package/dist/templates/retail/cancel.ts +116 -0
  68. package/dist/templates/retail/client.tsx +39 -0
  69. package/dist/templates/retail/payment.ts +107 -0
  70. package/dist/templates/retail/pending.test.ts +196 -0
  71. package/dist/templates/retail/pending.ts +137 -0
  72. package/dist/templates/retail/registry.test.ts +119 -8
  73. package/dist/templates/retail/returns.ts +102 -0
  74. package/dist/templates/retail/shared.test.ts +42 -0
  75. package/dist/templates/retail/shared.ts +35 -0
  76. package/dist/templates/retail/store.test.ts +6 -4
  77. package/dist/templates/retail/store.ts +113 -57
  78. package/dist/templates/retail/swap.test.ts +4 -4
  79. package/dist/templates/retail/swap.ts +248 -15
  80. package/dist/templates/retail/system-prompt.md +62 -30
  81. package/dist/templates/retail/tools/cancel_change.ts +33 -0
  82. package/dist/templates/retail/tools/cancel_pending_order.ts +15 -59
  83. package/dist/templates/retail/tools/confirm_change.ts +42 -0
  84. package/dist/templates/retail/tools/exchange_delivered_order_items.ts +21 -66
  85. package/dist/templates/retail/tools/modify_pending_order_address.ts +12 -24
  86. package/dist/templates/retail/tools/modify_pending_order_items.ts +22 -60
  87. package/dist/templates/retail/tools/modify_pending_order_payment.ts +15 -77
  88. package/dist/templates/retail/tools/modify_user_address.ts +12 -15
  89. package/dist/templates/retail/tools/return_delivered_order_items.ts +16 -71
  90. package/dist/templates/retail/tools/transfer_to_human_agents.ts +4 -2
  91. package/dist/templates/simple/agent.eval.test.ts +54 -0
  92. package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
  93. package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
  94. package/dist/templates/support-line/agent.eval.test.ts +273 -0
  95. package/dist/templates/support-line/system-prompt.md +2 -1
  96. package/dist/templates/support-line/tools/answer_question.ts +56 -7
  97. package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
  98. package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
  99. package/dist/templates/travel-concierge/agent.test.ts +110 -10
  100. package/dist/templates/travel-concierge/agent.ts +18 -9
  101. package/dist/templates/travel-concierge/shared.ts +110 -9
  102. package/dist/templates/travel-concierge/system-prompt.md +7 -1
  103. package/dist/templates/travel-concierge/tools/book_car_rental.ts +6 -3
  104. package/dist/templates/travel-concierge/tools/book_excursion.ts +6 -3
  105. package/dist/templates/travel-concierge/tools/book_hotel.ts +6 -3
  106. package/dist/templates/travel-concierge/tools/cancel_ticket.ts +6 -3
  107. package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
  108. package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
  109. package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
  110. package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
  111. package/dist/templates/travel-concierge/tools/update_ticket.ts +6 -3
  112. package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
  113. package/dist/templates/web-researcher/system-prompt.md +15 -4
  114. package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
  115. package/dist/test.d.ts +29 -5
  116. package/dist/{worker-bundler-COxnqstQ.mjs → worker-bundler-CCVEDjm1.mjs} +45 -25
  117. package/dist/worker-bundler.mjs +1 -1
  118. package/dist/{workflow-D2AQf2Pl.mjs → workflow-BflATMmU.mjs} +19 -11
  119. package/package.json +5 -5
  120. package/dist/eject-C9WJyyr2.mjs +0 -116
  121. package/dist/eject.d.ts +0 -60
  122. package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
@@ -0,0 +1,209 @@
1
+ // Copyright 2026 the AAI authors. MIT license.
2
+ // An EVAL for a WORKFLOW APP: does the run actually do the work? Run it with
3
+ // `aai eval`.
4
+ //
5
+ // `agent.test.ts` asserts about the declaration and drives the three steps one
6
+ // at a time. This drives the WHOLE BODY — `redlineFlow` from the top — and the
7
+ // thing it is here to check is the LOOP: the critic returns a verdict, the body
8
+ // breaks on it, and `shipped` says which of the two stopped it. That decision is
9
+ // the whole point of this template, and it is the one thing a per-step spec
10
+ // structurally cannot see.
11
+ //
12
+ // `describeWorkflowEval` picks the providers for you and says which it picked:
13
+ //
14
+ // * with `ASSEMBLYAI_API_KEY` — a LIVE run: a real model writes, a real model
15
+ // critiques, and a real model revises if it is asked to. That spends tokens,
16
+ // and a model is a NOISY instrument — one failure is a question, not a
17
+ // verdict. Re-run before believing either answer.
18
+ // * without one — a SCRIPTED run. The body, the loop and the three steps all
19
+ // really execute; only the gateway is answered in memory.
20
+ //
21
+ // Two of the three cases below are SCRIPTED IN BOTH MODES on purpose, and say so
22
+ // where they are: their claims are about the loop's arithmetic and about what a
23
+ // stage was SHOWN, and a live model cannot be asked to make either of those
24
+ // true — it can only be asked and then have its answer accepted, which is not
25
+ // evidence.
26
+ //
27
+ // WHAT NO EVAL HERE COVERS: durability. Imported through vitest with no bundler
28
+ // in the path, a `"use workflow"` body is an ordinary async function — no
29
+ // journal, no replay, and no per-step retry, so a rate-limited live run FAILS
30
+ // where a deployed one would have ridden it out. The tier that really resumes a
31
+ // run is `aai-cli`'s `dev-workflow.scenario.test.ts`.
32
+ import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
33
+ import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
34
+ import { expect } from "vitest";
35
+ import agentDef, { MAX_ROUNDS, redline } from "./agent.ts";
36
+
37
+ /** A brief with a word in it nothing else would produce, so the draft is checkable. */
38
+ const BRIEF = "Explain why our on-call rotation is moving to a two-week quokka cycle";
39
+ /** The one point every stage must be shown — `briefBlock` is what carries it. */
40
+ const MUST_COVER = ["Nobody carries the pager two weeks running"];
41
+
42
+ /** What the writer and the reviser are scripted to hand back. */
43
+ const DRAFT = [
44
+ "The on-call rotation is moving to a two-week quokka cycle.",
45
+ "Nobody carries the pager two weeks running: the second week is review and follow-up.",
46
+ ].join(" ");
47
+
48
+ const critique = (verdict: "ship" | "revise", score = 8): string =>
49
+ JSON.stringify({
50
+ verdict,
51
+ score,
52
+ notes: verdict === "ship" ? [] : ["Say what happens to the handover", "Name the start date"],
53
+ });
54
+
55
+ /** One gateway reply, in the envelope `stepGenerate` reads. */
56
+ const reply = (content: string) => ({ body: { choices: [{ message: { content } }] } });
57
+
58
+ /**
59
+ * Answer the gateway with `contents`, in order, and record what each stage asked.
60
+ *
61
+ * The last reply repeats, matching `stubGateway`'s convention — a loop cannot
62
+ * know how many calls it will make, and a script that ran out mid-loop would
63
+ * fail on the script rather than on the code. `installStubStepFetch` rather than
64
+ * `installStubGateway`: `stepGenerate` goes through the published `stepFetch`
65
+ * slot, and a published slot BEATS a stubbed global, so stubbing the global here
66
+ * would test a path production does not take.
67
+ */
68
+ function scriptGateway(contents: readonly string[]) {
69
+ let next = 0;
70
+ const fetched = installStubStepFetch(() => {
71
+ const content = contents.at(Math.min(next, contents.length - 1)) ?? "";
72
+ next += 1;
73
+ return reply(content);
74
+ });
75
+ return fetched;
76
+ }
77
+
78
+ /** Every prompt the gateway was sent, in call order. */
79
+ function promptsOf(fetched: ReturnType<typeof scriptGateway>): string[] {
80
+ return fetched.calls.map((call) => String(call.body ?? ""));
81
+ }
82
+
83
+ describeWorkflowEval(agentDef, (test) => {
84
+ test("runs a round and lets the CRITIC decide whether there is another", async ({
85
+ app,
86
+ mode,
87
+ }) => {
88
+ // One round in live mode, deliberately: the claim is about the loop's exit,
89
+ // and three long-form model calls are enough to make it.
90
+ if (mode === "stub") scriptGateway([DRAFT, critique("ship")]);
91
+
92
+ const run = await app.run(redline, {
93
+ brief: BRIEF,
94
+ audience: "engineers",
95
+ rounds: 1,
96
+ mustCover: MUST_COVER,
97
+ });
98
+
99
+ // The error first, so a failed run names its own reason.
100
+ expect(run.error).toBeUndefined();
101
+ expect(run.status).toBe("completed");
102
+
103
+ const output = run.output;
104
+ if (output === undefined) expect.fail("a completed run must carry an output");
105
+ expect(output.roundsRun).toBe(1);
106
+ expect(output.rounds).toHaveLength(1);
107
+
108
+ const last = output.rounds.at(-1);
109
+ if (last === undefined) expect.fail("a run of one round must record it");
110
+ // THE INVARIANT the template exists for: `shipped` is true exactly when the
111
+ // critic said so, and a shipped round revised nothing after it. Get either
112
+ // half wrong and the loop spends a model call it did not need, or stops one
113
+ // short — both of which read as a working run.
114
+ expect(["ship", "revise"]).toContain(last.critique.verdict);
115
+ expect(output.shipped).toBe(last.critique.verdict === "ship");
116
+ if (output.shipped) expect(last.revisedWords).toBeUndefined();
117
+ else expect(last.revisedWords).toBeTypeOf("number");
118
+
119
+ // The score is CLAMPED at the call site rather than by the schema, so a
120
+ // model answering 11 still lands in range.
121
+ expect(last.critique.score).toBeGreaterThanOrEqual(1);
122
+ expect(last.critique.score).toBeLessThanOrEqual(10);
123
+ expect(last.critique.notes.length).toBeLessThanOrEqual(3);
124
+
125
+ // It wrote about the brief it was given, not about writing in general.
126
+ expect(output.draft).toMatch(/quokka/i);
127
+ expect(output.words).toBeGreaterThan(5);
128
+
129
+ // Both stages narrated, and the round is numbered — which is what a page
130
+ // watching the run renders.
131
+ expect(run.reported[0]).toBe("Writing the first draft for engineers.");
132
+ expect(run.reported).toContain("Round 1: reading it back critically.");
133
+ // Nothing durable was asked for, so nothing was skipped: this body's only
134
+ // waits are its model calls.
135
+ expect(run.slept).toEqual([]);
136
+ });
137
+
138
+ test("stops on the ROUND BUDGET when the critic never ships", async ({ app }) => {
139
+ // Scripted in both modes: a live critic cannot be made to refuse three times
140
+ // running, and asking it and then accepting whatever it says is not evidence
141
+ // about the budget. What this pins is the loop's arithmetic — the half a
142
+ // live case cannot reach.
143
+ const fetched = scriptGateway([
144
+ DRAFT,
145
+ critique("revise", 4),
146
+ `${DRAFT} It starts on the first Monday of the month.`,
147
+ critique("revise", 5),
148
+ `${DRAFT} It starts on the first Monday, and the handover is a written note.`,
149
+ critique("revise", 6),
150
+ ]);
151
+
152
+ const run = await app.run(redline, {
153
+ brief: BRIEF,
154
+ audience: "executives",
155
+ rounds: MAX_ROUNDS,
156
+ mustCover: MUST_COVER,
157
+ });
158
+
159
+ expect(run.error).toBeUndefined();
160
+ const output = run.output;
161
+ if (output === undefined) expect.fail("a completed run must carry an output");
162
+
163
+ // The BUDGET stopped it, not the critic — which is the field a page reads to
164
+ // say "this is as good as it got" rather than "this is finished".
165
+ expect(output.shipped).toBe(false);
166
+ expect(output.roundsRun).toBe(MAX_ROUNDS);
167
+ // Every round revised, because none of them shipped.
168
+ expect(
169
+ output.rounds.map((round) => round.revisedWords).every((n) => typeof n === "number"),
170
+ ).toBe(true);
171
+ // One draft plus a critique-and-revise pair per round. A loop that critiqued
172
+ // twice, or revised the round it shipped, changes this number.
173
+ expect(fetched.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
174
+
175
+ // `briefBlock` is what keeps the three stages from drifting apart, and this is
176
+ // the assertion behind that claim: the writer, the critic AND the reviser were
177
+ // all shown the same brief and the same must-cover point.
178
+ const prompts = promptsOf(fetched);
179
+ expect(prompts).toHaveLength(1 + 2 * MAX_ROUNDS);
180
+ for (const prompt of prompts) {
181
+ expect(prompt).toContain("quokka");
182
+ expect(prompt).toContain(MUST_COVER[0]);
183
+ }
184
+ // The critic and the reviser were also shown the DRAFT, which the writer
185
+ // could not have been.
186
+ expect(prompts[1]).toContain("The on-call rotation is moving");
187
+ expect(prompts[2]).toContain("The critique");
188
+ });
189
+
190
+ test("refuses a brief that is only whitespace, terminally", async ({ app }) => {
191
+ // No model is reached on this path in either mode, so it costs nothing live.
192
+ // The case exists because the schema's `.min(20)` counts CHARACTERS: twenty
193
+ // spaces validate at `start()` and arrive at the writer as nothing to write
194
+ // from, which is why `writeDraft` carries its own `FatalError`.
195
+ const run = await app.run(redline, {
196
+ brief: " ".repeat(40),
197
+ audience: "general readers",
198
+ rounds: 1,
199
+ mustCover: [],
200
+ });
201
+
202
+ expect(run.status).toBe("failed");
203
+ expect(run.error).toMatch(/too short to write from/i);
204
+ expect(run.output).toBeUndefined();
205
+ // It failed before narrating anything, which is the ordering the guard
206
+ // implies: the check is the first thing in the step.
207
+ expect(run.reported).toEqual([]);
208
+ });
209
+ });
@@ -0,0 +1,329 @@
1
+ /// <reference types="vite/client" />
2
+
3
+ // An EVAL: does this desk actually behave? Run it with `aai eval`.
4
+ //
5
+ // `agent.test.ts` drives the four tools against a STUBBED `ctx.workflows` and
6
+ // the five steps against a stubbed `fetch`. Neither can say whether the AGENT —
7
+ // a model, reading this system prompt, holding these four tools — hands the
8
+ // work off at all, or whether the run it starts is findable afterwards. That is
9
+ // what this file is for, and it is only possible because `describeEval` opens a
10
+ // real workflow engine per case and hands its client to the session: before
11
+ // that, a tool calling `ctx.workflows.start` was a tool an eval could not
12
+ // execute.
13
+ //
14
+ // Two boundaries this file is careful about, because a case that implied either
15
+ // would be the worse failure:
16
+ //
17
+ // * **The engine is not durable.** No journal, no replay, no retry, and a
18
+ // `sleep()` is RECORDED rather than taken (which is why the review wait
19
+ // below is an assertion about what the body ASKED for). Nothing here says
20
+ // anything about resume, and `aai-cli`'s `dev-workflow.scenario.test.ts` is
21
+ // the tier that does.
22
+ // * **A step's HTTP is scripted, in BOTH modes.** The live half of an eval is
23
+ // the SESSION's model — which tool the desk reaches for, and when. The
24
+ // run's own five-to-twelve model calls and its web searches are answered
25
+ // from `MODEL_SCRIPT` through the published `stepFetch` slot, so a case is
26
+ // deterministic, free, and cannot fail on a DuckDuckGo 403 or a rate limit
27
+ // the engine's inert `maxRetries` could not ride out. What the researcher's
28
+ // search loop does with what it finds is `agent.test.ts`'s subject.
29
+ //
30
+ // And what no eval here can see at all: anything below the audio boundary —
31
+ // endpointing, barge-in, whether two sentences merged into one turn.
32
+ import { withDiscoveredTools } from "@alexkroman1/aai/testing";
33
+ import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
34
+ import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
35
+ import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
36
+ import { expect } from "vitest";
37
+ import { z } from "zod";
38
+ import authoredAgent from "./agent.ts";
39
+ import { research } from "./shared.ts";
40
+
41
+ /**
42
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
43
+ *
44
+ * Load-bearing, and not applied by `agent()` — the BUILD is what enumerates
45
+ * `tools/`, so an eval driving the raw default export would measure a desk with
46
+ * no tools at all and every assertion below would mean nothing. This template
47
+ * declares its prompt inline in `agent.ts`, so there is no `system-prompt.md`
48
+ * to layer on with `withSystemPrompt`.
49
+ *
50
+ * The glob is written here rather than reached for from a shared helper because
51
+ * this file SHIPS — see `agent.test.ts`.
52
+ */
53
+ const agentDef = withDiscoveredTools(
54
+ authoredAgent,
55
+ import.meta.glob("./tools/*.ts", { eager: true }),
56
+ );
57
+
58
+ /**
59
+ * The key the run's steps read with `requireStepEnv`.
60
+ *
61
+ * Passed as the agent env so the eval's workflow engine publishes it: the
62
+ * gateway call below is answered by a fake, but `stepGenerate` asks for the key
63
+ * BEFORE it makes the request, so a run with no key fails on the missing
64
+ * credential rather than reaching the script. The ENVIRONMENT and nothing else —
65
+ * a template may not read a developer's CLI config.
66
+ */
67
+ const EVAL_ENV = { ASSEMBLYAI_API_KEY: process.env.ASSEMBLYAI_API_KEY ?? "eval-scripted-key" };
68
+
69
+ /** The one angle the scripted planner comes back with. */
70
+ const ANGLE = "What second-hand cargo bikes actually sell for";
71
+
72
+ /** The written report the scripted `writeReport` produces. */
73
+ const REPORT_BODY =
74
+ "## Second-hand cargo bikes in Amsterdam\n\nAsking prices cluster between 1,200 and 2,400 EUR.";
75
+
76
+ /** The two sentences the desk reads down the phone. */
77
+ const SPOKEN_SUMMARY =
78
+ "Second-hand cargo bikes in Amsterdam mostly go for between one and two and a half thousand euros.";
79
+
80
+ /**
81
+ * What the run's model calls are answered with, in the order the body asks.
82
+ *
83
+ * Six calls, and the ORDER is the assertion: brief, angles, the researcher's
84
+ * one action, the gap pass, the report, the spoken summary. A stage that
85
+ * disappeared or moved would hand a later stage an earlier reply, which is
86
+ * exactly what the two content assertions in "writes the report" catch.
87
+ *
88
+ * The researcher's action is `stop` on purpose: `search` and `read` go through
89
+ * `webSearch`/`visitWebpage`, whose fetch is the global one and not the step
90
+ * slot, so scripting them is not available here — a case that let them run
91
+ * would really search the web.
92
+ */
93
+ const MODEL_SCRIPT: readonly string[] = [
94
+ JSON.stringify({
95
+ brief: "What a used cargo bike costs in Amsterdam, and where they are sold.",
96
+ criteria: ["typical asking prices", "where people buy them"],
97
+ }),
98
+ JSON.stringify({ angles: [ANGLE] }),
99
+ JSON.stringify({ action: "stop", why: "the budget is better spent elsewhere" }),
100
+ JSON.stringify({ angles: [] }),
101
+ REPORT_BODY,
102
+ SPOKEN_SUMMARY,
103
+ ];
104
+
105
+ /** A scripted step HTTP surface, and the gate that holds its first answer. */
106
+ type ScriptedSteps = {
107
+ /** Every step request, in order — the gateway calls the run really made. */
108
+ readonly calls: readonly { url: string; method: string }[];
109
+ /** Let a held first answer through. Idempotent. */
110
+ readonly release: () => void;
111
+ };
112
+
113
+ /**
114
+ * Answer the run's model calls from {@link MODEL_SCRIPT}.
115
+ *
116
+ * Through `stepFetch`'s published slot rather than `vi.stubGlobal("fetch")`,
117
+ * which is what a step really dials — and which leaves the SESSION's own model
118
+ * on the live path, so a live case still measures the agent. Anything that is
119
+ * not the gateway THROWS rather than answering 200: an unexpected request from
120
+ * a step is a finding, and a silent empty body would be read as a model that
121
+ * said nothing.
122
+ *
123
+ * `hold` keeps the FIRST answer pending, which is the only way to observe a run
124
+ * that is still going: a durable `sleep` is skipped here, so an unheld run
125
+ * finishes in milliseconds.
126
+ */
127
+ function scriptSteps(options: { hold?: boolean } = {}): ScriptedSteps {
128
+ const gate = Promise.withResolvers<void>();
129
+ let served = 0;
130
+ const stub = installStubStepFetch(async (request) => {
131
+ if (!request.url.includes("/chat/completions")) {
132
+ throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
133
+ }
134
+ const at = served++;
135
+ if (options.hold === true && at === 0) await gate.promise;
136
+ const content = MODEL_SCRIPT[Math.min(at, MODEL_SCRIPT.length - 1)];
137
+ return { body: { choices: [{ message: { content } }] } };
138
+ });
139
+ return { calls: stub.calls, release: () => gate.resolve() };
140
+ }
141
+
142
+ /** `request_research`'s answer when it really started something. */
143
+ const Started = z.object({
144
+ started: z.literal(true),
145
+ runId: z.string().min(1),
146
+ topic: z.string().min(1),
147
+ });
148
+
149
+ /**
150
+ * The run id a `request_research` call reported.
151
+ *
152
+ * Parsed rather than regexed: a tool result reaches the event stream as a
153
+ * serialized string, and a shape that stopped matching should fail HERE naming
154
+ * the field instead of handing the next assertion `undefined`.
155
+ */
156
+ function startedRunId(calls: readonly EvalToolCall[]): string {
157
+ const call = calls.find((one) => one.name === "request_research");
158
+ if (call?.result === undefined) {
159
+ throw new Error(
160
+ `the desk called no request_research: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
161
+ );
162
+ }
163
+ return Started.parse(JSON.parse(call.result)).runId;
164
+ }
165
+
166
+ /** Every tool call in this turn that READS a run, whichever the model picked. */
167
+ function readbacks(calls: readonly EvalToolCall[]): readonly EvalToolCall[] {
168
+ return calls.filter((one) => one.name === "research_status" || one.name === "research_progress");
169
+ }
170
+
171
+ /** The one utterance every case that starts work opens with. */
172
+ const ASK = "Please research the second-hand cargo bike market in Amsterdam for me.";
173
+
174
+ /** The scripted session turn that answers {@link ASK}. */
175
+ const START_TURN = [
176
+ { tool: "request_research", args: { topic: "the second-hand cargo bike market in Amsterdam" } },
177
+ "I've started looking into that — I'll let you know when it lands.",
178
+ ] as const;
179
+
180
+ /**
181
+ * Let the run finish before the case ends.
182
+ *
183
+ * Not tidiness: the scripted `stepFetch` is unpublished when the test that
184
+ * installed it finishes, so a body still mid-flight would make its next model
185
+ * call against whatever the next case publishes — or against the real gateway.
186
+ */
187
+ async function drain(workflows: EvalWorkflows | undefined, steps: ScriptedSteps): Promise<void> {
188
+ steps.release();
189
+ for (const run of await (workflows?.runs() ?? [])) {
190
+ await workflows?.settle(run.runId, research);
191
+ }
192
+ }
193
+
194
+ describeEval(
195
+ agentDef,
196
+ (test) => {
197
+ test(
198
+ "hands the topic to a run and answers the turn without waiting for it",
199
+ async ({ session, workflows }) => {
200
+ // Held, so the run cannot possibly have finished by the time the desk
201
+ // replies — which is the whole claim of the handoff shape.
202
+ const steps = scriptSteps({ hold: true });
203
+
204
+ const turn = await session.say(ASK);
205
+
206
+ const runId = startedRunId(turn.toolCalls);
207
+ expect(turn.completed).toBe(true);
208
+ // The topic it passed is the caller's, not a paraphrase of the prompt.
209
+ const asked = turn.toolCalls.find((one) => one.name === "request_research");
210
+ expect(String(asked?.args.topic)).toMatch(/cargo bike/i);
211
+
212
+ // The run is REAL: the engine started it, under the name the agent
213
+ // declares, and it is still going now that the turn has ended.
214
+ const runs = await (workflows?.runs() ?? []);
215
+ const started = runs.find((one) => one.runId === runId);
216
+ expect(started?.workflow).toBe("research");
217
+ expect(started?.status).toBe("running");
218
+ // And it really began work — the brief stage narrated before its model
219
+ // call, which is the request this case is holding.
220
+ expect(started?.reported.join("\n")).toMatch(/really asking/);
221
+
222
+ await drain(workflows, steps);
223
+ },
224
+ { stubReply: [...START_TURN] },
225
+ );
226
+
227
+ test(
228
+ "the run really writes the report, and asks for the review wait",
229
+ async ({ session, workflows }) => {
230
+ const steps = scriptSteps();
231
+
232
+ const turn = await session.say(ASK);
233
+ const runId = startedRunId(turn.toolCalls);
234
+ const run = await workflows?.settle(runId, research);
235
+
236
+ // What a completed run reports as `output` is what the agent reads back
237
+ // and what the announcement is built from, so every field is asserted.
238
+ expect(run?.status).toBe("completed");
239
+ expect(run?.output?.report).toBe(REPORT_BODY);
240
+ expect(run?.output?.summary).toBe(SPOKEN_SUMMARY);
241
+ expect(run?.output?.angles).toEqual([ANGLE]);
242
+ expect(run?.output?.filedAt).toBe("filed");
243
+
244
+ // The five stages, in order, off the run's own narration — which is
245
+ // also what `research_progress` reads back down the phone. A stage that
246
+ // stopped reporting is a caller who is told nothing for minutes.
247
+ const narration = run?.reported.join("\n") ?? "";
248
+ expect(narration).toMatch(/really asking/);
249
+ expect(narration).toMatch(/Researching 1 angle/);
250
+ expect(narration).toMatch(new RegExp(`Looking into: ${ANGLE}`));
251
+ expect(narration).toMatch(/writing it up/);
252
+ expect(narration).toMatch(/Writing up 1 angle/);
253
+ expect(run?.reported.at(-1)).toBe("Filing the findings.");
254
+
255
+ // The review wait, ASKED FOR and not taken: this engine records a
256
+ // durable `sleep` rather than suspending, so what a case can honestly
257
+ // claim is that the body asked — and that is the assertion that fails
258
+ // if the suspension is ever deleted.
259
+ expect(run?.slept).toEqual([{ duration: "30 seconds" }]);
260
+
261
+ // Six model calls, all through the step slot: the fan-out's width came
262
+ // from a journaled stage rather than from anything the body recomputed.
263
+ expect(steps.calls).toHaveLength(MODEL_SCRIPT.length);
264
+ },
265
+ { stubReply: [...START_TURN] },
266
+ );
267
+
268
+ test(
269
+ "reads the live run back rather than guessing at it",
270
+ async ({ session, workflows }) => {
271
+ const steps = scriptSteps({ hold: true });
272
+
273
+ const started = await session.say(ASK);
274
+ const runId = startedRunId(started.toolCalls);
275
+ const turn = await session.say("What's it doing right now?");
276
+
277
+ // WHICH of the two readback tools the model picks is its business —
278
+ // the prompt offers both — so the claim is about what it was told:
279
+ // either the run's own latest progress line or its status, and never
280
+ // an answer the desk invented.
281
+ const read = readbacks(turn.toolCalls);
282
+ expect(read.length).toBeGreaterThan(0);
283
+ const answered = read.map((one) => one.result ?? "").join("\n");
284
+ expect(answered).toMatch(/really asking|Still working on it/);
285
+ expect(answered).not.toMatch(/Nothing started yet/);
286
+
287
+ // The load-bearing half: that readback happened while the run was
288
+ // genuinely in flight, which is the only state these two tools exist
289
+ // for.
290
+ const runs = await (workflows?.runs() ?? []);
291
+ expect(runs.find((one) => one.runId === runId)?.status).toBe("running");
292
+
293
+ await drain(workflows, steps);
294
+ },
295
+ {
296
+ stubReply: [
297
+ ...START_TURN,
298
+ { tool: "research_progress", args: {} },
299
+ "It's still working out what the question really is.",
300
+ ],
301
+ },
302
+ );
303
+
304
+ test(
305
+ "says nothing is running when nothing is, and starts nothing to find out",
306
+ async ({ session, workflows }) => {
307
+ const steps = scriptSteps();
308
+
309
+ const turn = await session.say("Any news on that research I asked for?");
310
+
311
+ const read = readbacks(turn.toolCalls);
312
+ expect(read.length).toBeGreaterThan(0);
313
+ expect(read.map((one) => one.result ?? "").join("\n")).toMatch(/Nothing started yet/);
314
+ // A question is not a request: asking after work nobody asked for must
315
+ // not put a run — and a research pass's worth of model calls — on the
316
+ // account.
317
+ expect(await (workflows?.runs() ?? [])).toEqual([]);
318
+ expect(steps.calls).toEqual([]);
319
+ },
320
+ {
321
+ stubReply: [
322
+ { tool: "research_status", args: {} },
323
+ "Nothing has been started yet — want me to look into something?",
324
+ ],
325
+ },
326
+ );
327
+ },
328
+ { env: EVAL_ENV },
329
+ );
@@ -1,5 +1,14 @@
1
+ /**
2
+ * Addresses: the shared schema fields, and the two address changes as
3
+ * plan/apply pairs (see `cancel.ts` for why every mutating action is split
4
+ * that way).
5
+ */
6
+
7
+ import { isToolFailure, type ToolFailure } from "@alexkroman1/aai";
1
8
  import { z } from "zod";
2
- import type { Address } from "./shared.ts";
9
+ import { resolveOrder } from "./resolve.ts";
10
+ import type { Address, RetailState } from "./shared.ts";
11
+ import { requireOwnUser } from "./store.ts";
3
12
 
4
13
  /** Spread into both address tools' schemas. Two hand-copied zod shapes is how
5
14
  * the order address and the profile address drift apart. */
@@ -27,3 +36,80 @@ export function formatAddress(address: Address): string {
27
36
  const lines = [address.address1, address.address2].filter(Boolean).join(", ");
28
37
  return `${lines}, ${address.city} ${address.state} ${address.zip}, ${address.country}`;
29
38
  }
39
+
40
+ // ─── One order's shipping address ────────────────────────────────────────────
41
+
42
+ export interface OrderAddressPlan {
43
+ readBack: string;
44
+ orderId: string;
45
+ address: Address;
46
+ }
47
+
48
+ export function planOrderAddress(
49
+ state: RetailState,
50
+ spokenOrderId: string,
51
+ address: Address,
52
+ ): OrderAddressPlan | ToolFailure {
53
+ const order = resolveOrder(state, spokenOrderId);
54
+ if (isToolFailure(order)) return order;
55
+
56
+ // Any pending variant is fine here — unlike cancel and modify-items, which
57
+ // require exactly 'pending'. Re-addressing a modified order is harmless.
58
+ if (!order.status.startsWith("pending")) {
59
+ return {
60
+ error: `Order ${order.order_id} is ${order.status}, and only a pending order's address can be changed.`,
61
+ };
62
+ }
63
+
64
+ return {
65
+ readBack: `ship order ${order.order_id} to ${formatAddress(address)} instead`,
66
+ orderId: order.order_id,
67
+ address: toAddress(address),
68
+ };
69
+ }
70
+
71
+ export function applyOrderAddress(state: RetailState, plan: OrderAddressPlan) {
72
+ const order = state.store.orders[plan.orderId];
73
+ if (order) order.address = plan.address;
74
+ return {
75
+ order_id: plan.orderId,
76
+ status: order?.status ?? "pending",
77
+ address: plan.address,
78
+ message: `Order ${plan.orderId} now ships to ${formatAddress(plan.address)}.`,
79
+ };
80
+ }
81
+
82
+ // ─── The customer's default address ──────────────────────────────────────────
83
+
84
+ export interface UserAddressPlan {
85
+ readBack: string;
86
+ userId: string;
87
+ address: Address;
88
+ }
89
+
90
+ export function planUserAddress(
91
+ state: RetailState,
92
+ userId: string,
93
+ address: Address,
94
+ ): UserAddressPlan | ToolFailure {
95
+ const user = requireOwnUser(state, userId);
96
+ if (isToolFailure(user)) return user;
97
+
98
+ return {
99
+ readBack:
100
+ `change your default address for future orders to ${formatAddress(address)} ` +
101
+ "(existing orders keep their own)",
102
+ userId: user.user_id,
103
+ address: toAddress(address),
104
+ };
105
+ }
106
+
107
+ export function applyUserAddress(state: RetailState, plan: UserAddressPlan) {
108
+ const user = state.store.users[plan.userId];
109
+ if (user) user.address = plan.address;
110
+ return {
111
+ user_id: plan.userId,
112
+ address: plan.address,
113
+ message: `Default address updated to ${formatAddress(plan.address)}. Existing orders keep their own shipping addresses.`,
114
+ };
115
+ }