@alexkroman1/aai-cli 7.0.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/{_bundler-B4RqNF5Z.mjs → _bundler-CDuYl5Gb.mjs} +1 -1
- package/dist/{_dev-server-CiKFH9qw.mjs → _dev-server-Me06wgeF.mjs} +2 -2
- package/dist/{_init-CE4HKdgV.mjs → _init-BhEfQ6Yi.mjs} +1 -1
- package/dist/_resource-commands.d.ts +23 -0
- package/dist/{_slug-api-nRFaBEPJ.mjs → _slug-api-z-USxPtL.mjs} +14 -10
- package/dist/_slug-api.d.ts +5 -2
- package/dist/{_templates-FI3xPEMj.mjs → _templates-4WcKOjS5.mjs} +4 -5
- package/dist/_templates.d.ts +3 -4
- package/dist/{build-6jfaakgc.mjs → build-BBxf8vom.mjs} +2 -2
- package/dist/cli.mjs +338 -346
- package/dist/{deploy-DvYeD8ia.mjs → deploy-DRC8jWr8.mjs} +1 -1
- package/dist/{dev-B59JIfWO.mjs → dev-BvcctCn2.mjs} +1 -1
- package/dist/eval-DfQ8cm5p.mjs +56 -0
- package/dist/eval.d.ts +43 -0
- package/dist/{init-CpJVjEg0.mjs → init-CUo87gcI.mjs} +2 -2
- package/dist/{logs-9q-psa4q.mjs → logs-CQ2T46dr.mjs} +1 -1
- package/dist/scaffold/CLAUDE.md +409 -68
- package/dist/scaffold/package.json +5 -4
- package/dist/scaffold/server.mjs +10 -3
- package/dist/scaffold/tsconfig.json +0 -1
- package/dist/{secret-CVvSLIDV.mjs → secret-Ct34kdgU.mjs} +1 -1
- package/dist/{storage-BvUrnvM3.mjs → storage-i_79ZwAR.mjs} +1 -1
- package/dist/{studio--MUV0cid.mjs → studio-CzTzPqe1.mjs} +1 -1
- package/dist/templates/briefing-desk/agent.test.ts +350 -0
- package/dist/templates/briefing-desk/agent.ts +30 -0
- package/dist/templates/briefing-desk/shared.ts +200 -0
- package/dist/templates/briefing-desk/system-prompt.md +39 -0
- package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
- package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
- package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
- package/dist/templates/call-audit/agent.eval.test.ts +474 -0
- package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
- package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
- package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
- package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
- package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
- package/dist/templates/infocom-adventure/agent.test.ts +49 -18
- package/dist/templates/infocom-adventure/agent.ts +21 -0
- package/dist/templates/infocom-adventure/shared.ts +20 -9
- package/dist/templates/infocom-adventure/system-prompt.md +1 -1
- package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
- package/dist/templates/link-digest/agent.eval.test.ts +195 -0
- package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
- package/dist/templates/night-owl/agent.eval.test.ts +272 -0
- package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
- package/dist/templates/personal-finance/system-prompt.md +4 -0
- package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
- package/dist/templates/pipeline-simple/agent.ts +2 -2
- package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
- package/dist/templates/pizza-ordering/agent.test.ts +15 -0
- package/dist/templates/pizza-ordering/shared.ts +20 -1
- package/dist/templates/pizza-ordering/system-prompt.md +6 -0
- package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
- package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
- package/dist/templates/podcast-digest/agent.test.ts +30 -59
- package/dist/templates/podcast-digest/agent.ts +1 -1
- package/dist/templates/podcast-digest/workflows/digest.ts +52 -19
- package/dist/templates/podcast-digest/workflows/slack.ts +48 -169
- package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
- package/dist/templates/redline/agent.eval.test.ts +209 -0
- package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
- package/dist/templates/retail/address.ts +87 -1
- package/dist/templates/retail/agent.eval.test.ts +314 -0
- package/dist/templates/retail/agent.test.ts +171 -91
- package/dist/templates/retail/agent.ts +35 -1
- package/dist/templates/retail/cancel.ts +116 -0
- package/dist/templates/retail/client.tsx +39 -0
- package/dist/templates/retail/payment.ts +107 -0
- package/dist/templates/retail/pending.test.ts +196 -0
- package/dist/templates/retail/pending.ts +137 -0
- package/dist/templates/retail/registry.test.ts +119 -8
- package/dist/templates/retail/returns.ts +102 -0
- package/dist/templates/retail/shared.test.ts +42 -0
- package/dist/templates/retail/shared.ts +35 -0
- package/dist/templates/retail/store.test.ts +6 -4
- package/dist/templates/retail/store.ts +113 -57
- package/dist/templates/retail/swap.test.ts +4 -4
- package/dist/templates/retail/swap.ts +248 -15
- package/dist/templates/retail/system-prompt.md +62 -30
- package/dist/templates/retail/tools/cancel_change.ts +33 -0
- package/dist/templates/retail/tools/cancel_pending_order.ts +15 -59
- package/dist/templates/retail/tools/confirm_change.ts +42 -0
- package/dist/templates/retail/tools/exchange_delivered_order_items.ts +21 -66
- package/dist/templates/retail/tools/modify_pending_order_address.ts +12 -24
- package/dist/templates/retail/tools/modify_pending_order_items.ts +22 -60
- package/dist/templates/retail/tools/modify_pending_order_payment.ts +15 -77
- package/dist/templates/retail/tools/modify_user_address.ts +12 -15
- package/dist/templates/retail/tools/return_delivered_order_items.ts +16 -71
- package/dist/templates/retail/tools/transfer_to_human_agents.ts +4 -2
- package/dist/templates/simple/agent.eval.test.ts +54 -0
- package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
- package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
- package/dist/templates/support-line/agent.eval.test.ts +273 -0
- package/dist/templates/support-line/system-prompt.md +2 -1
- package/dist/templates/support-line/tools/answer_question.ts +56 -7
- package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
- package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
- package/dist/templates/travel-concierge/agent.test.ts +110 -10
- package/dist/templates/travel-concierge/agent.ts +18 -9
- package/dist/templates/travel-concierge/shared.ts +110 -9
- package/dist/templates/travel-concierge/system-prompt.md +7 -1
- package/dist/templates/travel-concierge/tools/book_car_rental.ts +6 -3
- package/dist/templates/travel-concierge/tools/book_excursion.ts +6 -3
- package/dist/templates/travel-concierge/tools/book_hotel.ts +6 -3
- package/dist/templates/travel-concierge/tools/cancel_ticket.ts +6 -3
- package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
- package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
- package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
- package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
- package/dist/templates/travel-concierge/tools/update_ticket.ts +6 -3
- package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
- package/dist/templates/web-researcher/system-prompt.md +15 -4
- package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
- package/dist/test.d.ts +29 -5
- package/dist/{worker-bundler-COxnqstQ.mjs → worker-bundler-CCVEDjm1.mjs} +45 -25
- package/dist/worker-bundler.mjs +1 -1
- package/dist/{workflow-D2AQf2Pl.mjs → workflow-BflATMmU.mjs} +19 -11
- package/package.json +5 -5
- package/dist/eject-C9WJyyr2.mjs +0 -116
- package/dist/eject.d.ts +0 -60
- package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// Copyright 2026 the AAI authors. MIT license.
|
|
2
|
+
// An EVAL for a WORKFLOW APP: does the run actually do the work? Run it with
|
|
3
|
+
// `aai eval`.
|
|
4
|
+
//
|
|
5
|
+
// `agent.test.ts` asserts about the declaration and drives the three steps one
|
|
6
|
+
// at a time. This drives the WHOLE BODY — `redlineFlow` from the top — and the
|
|
7
|
+
// thing it is here to check is the LOOP: the critic returns a verdict, the body
|
|
8
|
+
// breaks on it, and `shipped` says which of the two stopped it. That decision is
|
|
9
|
+
// the whole point of this template, and it is the one thing a per-step spec
|
|
10
|
+
// structurally cannot see.
|
|
11
|
+
//
|
|
12
|
+
// `describeWorkflowEval` picks the providers for you and says which it picked:
|
|
13
|
+
//
|
|
14
|
+
// * with `ASSEMBLYAI_API_KEY` — a LIVE run: a real model writes, a real model
|
|
15
|
+
// critiques, and a real model revises if it is asked to. That spends tokens,
|
|
16
|
+
// and a model is a NOISY instrument — one failure is a question, not a
|
|
17
|
+
// verdict. Re-run before believing either answer.
|
|
18
|
+
// * without one — a SCRIPTED run. The body, the loop and the three steps all
|
|
19
|
+
// really execute; only the gateway is answered in memory.
|
|
20
|
+
//
|
|
21
|
+
// Two of the three cases below are SCRIPTED IN BOTH MODES on purpose, and say so
|
|
22
|
+
// where they are: their claims are about the loop's arithmetic and about what a
|
|
23
|
+
// stage was SHOWN, and a live model cannot be asked to make either of those
|
|
24
|
+
// true — it can only be asked and then have its answer accepted, which is not
|
|
25
|
+
// evidence.
|
|
26
|
+
//
|
|
27
|
+
// WHAT NO EVAL HERE COVERS: durability. Imported through vitest with no bundler
|
|
28
|
+
// in the path, a `"use workflow"` body is an ordinary async function — no
|
|
29
|
+
// journal, no replay, and no per-step retry, so a rate-limited live run FAILS
|
|
30
|
+
// where a deployed one would have ridden it out. The tier that really resumes a
|
|
31
|
+
// run is `aai-cli`'s `dev-workflow.scenario.test.ts`.
|
|
32
|
+
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
33
|
+
import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
34
|
+
import { expect } from "vitest";
|
|
35
|
+
import agentDef, { MAX_ROUNDS, redline } from "./agent.ts";
|
|
36
|
+
|
|
37
|
+
/** A brief with a word in it nothing else would produce, so the draft is checkable. */
|
|
38
|
+
const BRIEF = "Explain why our on-call rotation is moving to a two-week quokka cycle";
|
|
39
|
+
/** The one point every stage must be shown — `briefBlock` is what carries it. */
|
|
40
|
+
const MUST_COVER = ["Nobody carries the pager two weeks running"];
|
|
41
|
+
|
|
42
|
+
/** What the writer and the reviser are scripted to hand back. */
|
|
43
|
+
const DRAFT = [
|
|
44
|
+
"The on-call rotation is moving to a two-week quokka cycle.",
|
|
45
|
+
"Nobody carries the pager two weeks running: the second week is review and follow-up.",
|
|
46
|
+
].join(" ");
|
|
47
|
+
|
|
48
|
+
const critique = (verdict: "ship" | "revise", score = 8): string =>
|
|
49
|
+
JSON.stringify({
|
|
50
|
+
verdict,
|
|
51
|
+
score,
|
|
52
|
+
notes: verdict === "ship" ? [] : ["Say what happens to the handover", "Name the start date"],
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
/** One gateway reply, in the envelope `stepGenerate` reads. */
|
|
56
|
+
const reply = (content: string) => ({ body: { choices: [{ message: { content } }] } });
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Answer the gateway with `contents`, in order, and record what each stage asked.
|
|
60
|
+
*
|
|
61
|
+
* The last reply repeats, matching `stubGateway`'s convention — a loop cannot
|
|
62
|
+
* know how many calls it will make, and a script that ran out mid-loop would
|
|
63
|
+
* fail on the script rather than on the code. `installStubStepFetch` rather than
|
|
64
|
+
* `installStubGateway`: `stepGenerate` goes through the published `stepFetch`
|
|
65
|
+
* slot, and a published slot BEATS a stubbed global, so stubbing the global here
|
|
66
|
+
* would test a path production does not take.
|
|
67
|
+
*/
|
|
68
|
+
function scriptGateway(contents: readonly string[]) {
|
|
69
|
+
let next = 0;
|
|
70
|
+
const fetched = installStubStepFetch(() => {
|
|
71
|
+
const content = contents.at(Math.min(next, contents.length - 1)) ?? "";
|
|
72
|
+
next += 1;
|
|
73
|
+
return reply(content);
|
|
74
|
+
});
|
|
75
|
+
return fetched;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Every prompt the gateway was sent, in call order. */
|
|
79
|
+
function promptsOf(fetched: ReturnType<typeof scriptGateway>): string[] {
|
|
80
|
+
return fetched.calls.map((call) => String(call.body ?? ""));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describeWorkflowEval(agentDef, (test) => {
|
|
84
|
+
test("runs a round and lets the CRITIC decide whether there is another", async ({
|
|
85
|
+
app,
|
|
86
|
+
mode,
|
|
87
|
+
}) => {
|
|
88
|
+
// One round in live mode, deliberately: the claim is about the loop's exit,
|
|
89
|
+
// and three long-form model calls are enough to make it.
|
|
90
|
+
if (mode === "stub") scriptGateway([DRAFT, critique("ship")]);
|
|
91
|
+
|
|
92
|
+
const run = await app.run(redline, {
|
|
93
|
+
brief: BRIEF,
|
|
94
|
+
audience: "engineers",
|
|
95
|
+
rounds: 1,
|
|
96
|
+
mustCover: MUST_COVER,
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
// The error first, so a failed run names its own reason.
|
|
100
|
+
expect(run.error).toBeUndefined();
|
|
101
|
+
expect(run.status).toBe("completed");
|
|
102
|
+
|
|
103
|
+
const output = run.output;
|
|
104
|
+
if (output === undefined) expect.fail("a completed run must carry an output");
|
|
105
|
+
expect(output.roundsRun).toBe(1);
|
|
106
|
+
expect(output.rounds).toHaveLength(1);
|
|
107
|
+
|
|
108
|
+
const last = output.rounds.at(-1);
|
|
109
|
+
if (last === undefined) expect.fail("a run of one round must record it");
|
|
110
|
+
// THE INVARIANT the template exists for: `shipped` is true exactly when the
|
|
111
|
+
// critic said so, and a shipped round revised nothing after it. Get either
|
|
112
|
+
// half wrong and the loop spends a model call it did not need, or stops one
|
|
113
|
+
// short — both of which read as a working run.
|
|
114
|
+
expect(["ship", "revise"]).toContain(last.critique.verdict);
|
|
115
|
+
expect(output.shipped).toBe(last.critique.verdict === "ship");
|
|
116
|
+
if (output.shipped) expect(last.revisedWords).toBeUndefined();
|
|
117
|
+
else expect(last.revisedWords).toBeTypeOf("number");
|
|
118
|
+
|
|
119
|
+
// The score is CLAMPED at the call site rather than by the schema, so a
|
|
120
|
+
// model answering 11 still lands in range.
|
|
121
|
+
expect(last.critique.score).toBeGreaterThanOrEqual(1);
|
|
122
|
+
expect(last.critique.score).toBeLessThanOrEqual(10);
|
|
123
|
+
expect(last.critique.notes.length).toBeLessThanOrEqual(3);
|
|
124
|
+
|
|
125
|
+
// It wrote about the brief it was given, not about writing in general.
|
|
126
|
+
expect(output.draft).toMatch(/quokka/i);
|
|
127
|
+
expect(output.words).toBeGreaterThan(5);
|
|
128
|
+
|
|
129
|
+
// Both stages narrated, and the round is numbered — which is what a page
|
|
130
|
+
// watching the run renders.
|
|
131
|
+
expect(run.reported[0]).toBe("Writing the first draft for engineers.");
|
|
132
|
+
expect(run.reported).toContain("Round 1: reading it back critically.");
|
|
133
|
+
// Nothing durable was asked for, so nothing was skipped: this body's only
|
|
134
|
+
// waits are its model calls.
|
|
135
|
+
expect(run.slept).toEqual([]);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("stops on the ROUND BUDGET when the critic never ships", async ({ app }) => {
|
|
139
|
+
// Scripted in both modes: a live critic cannot be made to refuse three times
|
|
140
|
+
// running, and asking it and then accepting whatever it says is not evidence
|
|
141
|
+
// about the budget. What this pins is the loop's arithmetic — the half a
|
|
142
|
+
// live case cannot reach.
|
|
143
|
+
const fetched = scriptGateway([
|
|
144
|
+
DRAFT,
|
|
145
|
+
critique("revise", 4),
|
|
146
|
+
`${DRAFT} It starts on the first Monday of the month.`,
|
|
147
|
+
critique("revise", 5),
|
|
148
|
+
`${DRAFT} It starts on the first Monday, and the handover is a written note.`,
|
|
149
|
+
critique("revise", 6),
|
|
150
|
+
]);
|
|
151
|
+
|
|
152
|
+
const run = await app.run(redline, {
|
|
153
|
+
brief: BRIEF,
|
|
154
|
+
audience: "executives",
|
|
155
|
+
rounds: MAX_ROUNDS,
|
|
156
|
+
mustCover: MUST_COVER,
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
expect(run.error).toBeUndefined();
|
|
160
|
+
const output = run.output;
|
|
161
|
+
if (output === undefined) expect.fail("a completed run must carry an output");
|
|
162
|
+
|
|
163
|
+
// The BUDGET stopped it, not the critic — which is the field a page reads to
|
|
164
|
+
// say "this is as good as it got" rather than "this is finished".
|
|
165
|
+
expect(output.shipped).toBe(false);
|
|
166
|
+
expect(output.roundsRun).toBe(MAX_ROUNDS);
|
|
167
|
+
// Every round revised, because none of them shipped.
|
|
168
|
+
expect(
|
|
169
|
+
output.rounds.map((round) => round.revisedWords).every((n) => typeof n === "number"),
|
|
170
|
+
).toBe(true);
|
|
171
|
+
// One draft plus a critique-and-revise pair per round. A loop that critiqued
|
|
172
|
+
// twice, or revised the round it shipped, changes this number.
|
|
173
|
+
expect(fetched.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
|
|
174
|
+
|
|
175
|
+
// `briefBlock` is what keeps the three stages from drifting apart, and this is
|
|
176
|
+
// the assertion behind that claim: the writer, the critic AND the reviser were
|
|
177
|
+
// all shown the same brief and the same must-cover point.
|
|
178
|
+
const prompts = promptsOf(fetched);
|
|
179
|
+
expect(prompts).toHaveLength(1 + 2 * MAX_ROUNDS);
|
|
180
|
+
for (const prompt of prompts) {
|
|
181
|
+
expect(prompt).toContain("quokka");
|
|
182
|
+
expect(prompt).toContain(MUST_COVER[0]);
|
|
183
|
+
}
|
|
184
|
+
// The critic and the reviser were also shown the DRAFT, which the writer
|
|
185
|
+
// could not have been.
|
|
186
|
+
expect(prompts[1]).toContain("The on-call rotation is moving");
|
|
187
|
+
expect(prompts[2]).toContain("The critique");
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("refuses a brief that is only whitespace, terminally", async ({ app }) => {
|
|
191
|
+
// No model is reached on this path in either mode, so it costs nothing live.
|
|
192
|
+
// The case exists because the schema's `.min(20)` counts CHARACTERS: twenty
|
|
193
|
+
// spaces validate at `start()` and arrive at the writer as nothing to write
|
|
194
|
+
// from, which is why `writeDraft` carries its own `FatalError`.
|
|
195
|
+
const run = await app.run(redline, {
|
|
196
|
+
brief: " ".repeat(40),
|
|
197
|
+
audience: "general readers",
|
|
198
|
+
rounds: 1,
|
|
199
|
+
mustCover: [],
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
expect(run.status).toBe("failed");
|
|
203
|
+
expect(run.error).toMatch(/too short to write from/i);
|
|
204
|
+
expect(run.output).toBeUndefined();
|
|
205
|
+
// It failed before narrating anything, which is the ordering the guard
|
|
206
|
+
// implies: the check is the first thing in the step.
|
|
207
|
+
expect(run.reported).toEqual([]);
|
|
208
|
+
});
|
|
209
|
+
});
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
/// <reference types="vite/client" />
|
|
2
|
+
|
|
3
|
+
// An EVAL: does this desk actually behave? Run it with `aai eval`.
|
|
4
|
+
//
|
|
5
|
+
// `agent.test.ts` drives the four tools against a STUBBED `ctx.workflows` and
|
|
6
|
+
// the five steps against a stubbed `fetch`. Neither can say whether the AGENT —
|
|
7
|
+
// a model, reading this system prompt, holding these four tools — hands the
|
|
8
|
+
// work off at all, or whether the run it starts is findable afterwards. That is
|
|
9
|
+
// what this file is for, and it is only possible because `describeEval` opens a
|
|
10
|
+
// real workflow engine per case and hands its client to the session: before
|
|
11
|
+
// that, a tool calling `ctx.workflows.start` was a tool an eval could not
|
|
12
|
+
// execute.
|
|
13
|
+
//
|
|
14
|
+
// Two boundaries this file is careful about, because a case that implied either
|
|
15
|
+
// would be the worse failure:
|
|
16
|
+
//
|
|
17
|
+
// * **The engine is not durable.** No journal, no replay, no retry, and a
|
|
18
|
+
// `sleep()` is RECORDED rather than taken (which is why the review wait
|
|
19
|
+
// below is an assertion about what the body ASKED for). Nothing here says
|
|
20
|
+
// anything about resume, and `aai-cli`'s `dev-workflow.scenario.test.ts` is
|
|
21
|
+
// the tier that does.
|
|
22
|
+
// * **A step's HTTP is scripted, in BOTH modes.** The live half of an eval is
|
|
23
|
+
// the SESSION's model — which tool the desk reaches for, and when. The
|
|
24
|
+
// run's own five-to-twelve model calls and its web searches are answered
|
|
25
|
+
// from `MODEL_SCRIPT` through the published `stepFetch` slot, so a case is
|
|
26
|
+
// deterministic, free, and cannot fail on a DuckDuckGo 403 or a rate limit
|
|
27
|
+
// the engine's inert `maxRetries` could not ride out. What the researcher's
|
|
28
|
+
// search loop does with what it finds is `agent.test.ts`'s subject.
|
|
29
|
+
//
|
|
30
|
+
// And what no eval here can see at all: anything below the audio boundary —
|
|
31
|
+
// endpointing, barge-in, whether two sentences merged into one turn.
|
|
32
|
+
import { withDiscoveredTools } from "@alexkroman1/aai/testing";
|
|
33
|
+
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
34
|
+
import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
|
|
35
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
36
|
+
import { expect } from "vitest";
|
|
37
|
+
import { z } from "zod";
|
|
38
|
+
import authoredAgent from "./agent.ts";
|
|
39
|
+
import { research } from "./shared.ts";
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
43
|
+
*
|
|
44
|
+
* Load-bearing, and not applied by `agent()` — the BUILD is what enumerates
|
|
45
|
+
* `tools/`, so an eval driving the raw default export would measure a desk with
|
|
46
|
+
* no tools at all and every assertion below would mean nothing. This template
|
|
47
|
+
* declares its prompt inline in `agent.ts`, so there is no `system-prompt.md`
|
|
48
|
+
* to layer on with `withSystemPrompt`.
|
|
49
|
+
*
|
|
50
|
+
* The glob is written here rather than reached for from a shared helper because
|
|
51
|
+
* this file SHIPS — see `agent.test.ts`.
|
|
52
|
+
*/
|
|
53
|
+
const agentDef = withDiscoveredTools(
|
|
54
|
+
authoredAgent,
|
|
55
|
+
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The key the run's steps read with `requireStepEnv`.
|
|
60
|
+
*
|
|
61
|
+
* Passed as the agent env so the eval's workflow engine publishes it: the
|
|
62
|
+
* gateway call below is answered by a fake, but `stepGenerate` asks for the key
|
|
63
|
+
* BEFORE it makes the request, so a run with no key fails on the missing
|
|
64
|
+
* credential rather than reaching the script. The ENVIRONMENT and nothing else —
|
|
65
|
+
* a template may not read a developer's CLI config.
|
|
66
|
+
*/
|
|
67
|
+
const EVAL_ENV = { ASSEMBLYAI_API_KEY: process.env.ASSEMBLYAI_API_KEY ?? "eval-scripted-key" };
|
|
68
|
+
|
|
69
|
+
/** The one angle the scripted planner comes back with. */
|
|
70
|
+
const ANGLE = "What second-hand cargo bikes actually sell for";
|
|
71
|
+
|
|
72
|
+
/** The written report the scripted `writeReport` produces. */
|
|
73
|
+
const REPORT_BODY =
|
|
74
|
+
"## Second-hand cargo bikes in Amsterdam\n\nAsking prices cluster between 1,200 and 2,400 EUR.";
|
|
75
|
+
|
|
76
|
+
/** The two sentences the desk reads down the phone. */
|
|
77
|
+
const SPOKEN_SUMMARY =
|
|
78
|
+
"Second-hand cargo bikes in Amsterdam mostly go for between one and two and a half thousand euros.";
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* What the run's model calls are answered with, in the order the body asks.
|
|
82
|
+
*
|
|
83
|
+
* Six calls, and the ORDER is the assertion: brief, angles, the researcher's
|
|
84
|
+
* one action, the gap pass, the report, the spoken summary. A stage that
|
|
85
|
+
* disappeared or moved would hand a later stage an earlier reply, which is
|
|
86
|
+
* exactly what the two content assertions in "writes the report" catch.
|
|
87
|
+
*
|
|
88
|
+
* The researcher's action is `stop` on purpose: `search` and `read` go through
|
|
89
|
+
* `webSearch`/`visitWebpage`, whose fetch is the global one and not the step
|
|
90
|
+
* slot, so scripting them is not available here — a case that let them run
|
|
91
|
+
* would really search the web.
|
|
92
|
+
*/
|
|
93
|
+
const MODEL_SCRIPT: readonly string[] = [
|
|
94
|
+
JSON.stringify({
|
|
95
|
+
brief: "What a used cargo bike costs in Amsterdam, and where they are sold.",
|
|
96
|
+
criteria: ["typical asking prices", "where people buy them"],
|
|
97
|
+
}),
|
|
98
|
+
JSON.stringify({ angles: [ANGLE] }),
|
|
99
|
+
JSON.stringify({ action: "stop", why: "the budget is better spent elsewhere" }),
|
|
100
|
+
JSON.stringify({ angles: [] }),
|
|
101
|
+
REPORT_BODY,
|
|
102
|
+
SPOKEN_SUMMARY,
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
/** A scripted step HTTP surface, and the gate that holds its first answer. */
|
|
106
|
+
type ScriptedSteps = {
|
|
107
|
+
/** Every step request, in order — the gateway calls the run really made. */
|
|
108
|
+
readonly calls: readonly { url: string; method: string }[];
|
|
109
|
+
/** Let a held first answer through. Idempotent. */
|
|
110
|
+
readonly release: () => void;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Answer the run's model calls from {@link MODEL_SCRIPT}.
|
|
115
|
+
*
|
|
116
|
+
* Through `stepFetch`'s published slot rather than `vi.stubGlobal("fetch")`,
|
|
117
|
+
* which is what a step really dials — and which leaves the SESSION's own model
|
|
118
|
+
* on the live path, so a live case still measures the agent. Anything that is
|
|
119
|
+
* not the gateway THROWS rather than answering 200: an unexpected request from
|
|
120
|
+
* a step is a finding, and a silent empty body would be read as a model that
|
|
121
|
+
* said nothing.
|
|
122
|
+
*
|
|
123
|
+
* `hold` keeps the FIRST answer pending, which is the only way to observe a run
|
|
124
|
+
* that is still going: a durable `sleep` is skipped here, so an unheld run
|
|
125
|
+
* finishes in milliseconds.
|
|
126
|
+
*/
|
|
127
|
+
function scriptSteps(options: { hold?: boolean } = {}): ScriptedSteps {
|
|
128
|
+
const gate = Promise.withResolvers<void>();
|
|
129
|
+
let served = 0;
|
|
130
|
+
const stub = installStubStepFetch(async (request) => {
|
|
131
|
+
if (!request.url.includes("/chat/completions")) {
|
|
132
|
+
throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
|
|
133
|
+
}
|
|
134
|
+
const at = served++;
|
|
135
|
+
if (options.hold === true && at === 0) await gate.promise;
|
|
136
|
+
const content = MODEL_SCRIPT[Math.min(at, MODEL_SCRIPT.length - 1)];
|
|
137
|
+
return { body: { choices: [{ message: { content } }] } };
|
|
138
|
+
});
|
|
139
|
+
return { calls: stub.calls, release: () => gate.resolve() };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** `request_research`'s answer when it really started something. */
|
|
143
|
+
const Started = z.object({
|
|
144
|
+
started: z.literal(true),
|
|
145
|
+
runId: z.string().min(1),
|
|
146
|
+
topic: z.string().min(1),
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The run id a `request_research` call reported.
|
|
151
|
+
*
|
|
152
|
+
* Parsed rather than regexed: a tool result reaches the event stream as a
|
|
153
|
+
* serialized string, and a shape that stopped matching should fail HERE naming
|
|
154
|
+
* the field instead of handing the next assertion `undefined`.
|
|
155
|
+
*/
|
|
156
|
+
function startedRunId(calls: readonly EvalToolCall[]): string {
|
|
157
|
+
const call = calls.find((one) => one.name === "request_research");
|
|
158
|
+
if (call?.result === undefined) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
`the desk called no request_research: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
return Started.parse(JSON.parse(call.result)).runId;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Every tool call in this turn that READS a run, whichever the model picked. */
|
|
167
|
+
function readbacks(calls: readonly EvalToolCall[]): readonly EvalToolCall[] {
|
|
168
|
+
return calls.filter((one) => one.name === "research_status" || one.name === "research_progress");
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** The one utterance every case that starts work opens with. */
|
|
172
|
+
const ASK = "Please research the second-hand cargo bike market in Amsterdam for me.";
|
|
173
|
+
|
|
174
|
+
/** The scripted session turn that answers {@link ASK}. */
|
|
175
|
+
const START_TURN = [
|
|
176
|
+
{ tool: "request_research", args: { topic: "the second-hand cargo bike market in Amsterdam" } },
|
|
177
|
+
"I've started looking into that — I'll let you know when it lands.",
|
|
178
|
+
] as const;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Let the run finish before the case ends.
|
|
182
|
+
*
|
|
183
|
+
* Not tidiness: the scripted `stepFetch` is unpublished when the test that
|
|
184
|
+
* installed it finishes, so a body still mid-flight would make its next model
|
|
185
|
+
* call against whatever the next case publishes — or against the real gateway.
|
|
186
|
+
*/
|
|
187
|
+
async function drain(workflows: EvalWorkflows | undefined, steps: ScriptedSteps): Promise<void> {
|
|
188
|
+
steps.release();
|
|
189
|
+
for (const run of await (workflows?.runs() ?? [])) {
|
|
190
|
+
await workflows?.settle(run.runId, research);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
describeEval(
|
|
195
|
+
agentDef,
|
|
196
|
+
(test) => {
|
|
197
|
+
test(
|
|
198
|
+
"hands the topic to a run and answers the turn without waiting for it",
|
|
199
|
+
async ({ session, workflows }) => {
|
|
200
|
+
// Held, so the run cannot possibly have finished by the time the desk
|
|
201
|
+
// replies — which is the whole claim of the handoff shape.
|
|
202
|
+
const steps = scriptSteps({ hold: true });
|
|
203
|
+
|
|
204
|
+
const turn = await session.say(ASK);
|
|
205
|
+
|
|
206
|
+
const runId = startedRunId(turn.toolCalls);
|
|
207
|
+
expect(turn.completed).toBe(true);
|
|
208
|
+
// The topic it passed is the caller's, not a paraphrase of the prompt.
|
|
209
|
+
const asked = turn.toolCalls.find((one) => one.name === "request_research");
|
|
210
|
+
expect(String(asked?.args.topic)).toMatch(/cargo bike/i);
|
|
211
|
+
|
|
212
|
+
// The run is REAL: the engine started it, under the name the agent
|
|
213
|
+
// declares, and it is still going now that the turn has ended.
|
|
214
|
+
const runs = await (workflows?.runs() ?? []);
|
|
215
|
+
const started = runs.find((one) => one.runId === runId);
|
|
216
|
+
expect(started?.workflow).toBe("research");
|
|
217
|
+
expect(started?.status).toBe("running");
|
|
218
|
+
// And it really began work — the brief stage narrated before its model
|
|
219
|
+
// call, which is the request this case is holding.
|
|
220
|
+
expect(started?.reported.join("\n")).toMatch(/really asking/);
|
|
221
|
+
|
|
222
|
+
await drain(workflows, steps);
|
|
223
|
+
},
|
|
224
|
+
{ stubReply: [...START_TURN] },
|
|
225
|
+
);
|
|
226
|
+
|
|
227
|
+
test(
|
|
228
|
+
"the run really writes the report, and asks for the review wait",
|
|
229
|
+
async ({ session, workflows }) => {
|
|
230
|
+
const steps = scriptSteps();
|
|
231
|
+
|
|
232
|
+
const turn = await session.say(ASK);
|
|
233
|
+
const runId = startedRunId(turn.toolCalls);
|
|
234
|
+
const run = await workflows?.settle(runId, research);
|
|
235
|
+
|
|
236
|
+
// What a completed run reports as `output` is what the agent reads back
|
|
237
|
+
// and what the announcement is built from, so every field is asserted.
|
|
238
|
+
expect(run?.status).toBe("completed");
|
|
239
|
+
expect(run?.output?.report).toBe(REPORT_BODY);
|
|
240
|
+
expect(run?.output?.summary).toBe(SPOKEN_SUMMARY);
|
|
241
|
+
expect(run?.output?.angles).toEqual([ANGLE]);
|
|
242
|
+
expect(run?.output?.filedAt).toBe("filed");
|
|
243
|
+
|
|
244
|
+
// The five stages, in order, off the run's own narration — which is
|
|
245
|
+
// also what `research_progress` reads back down the phone. A stage that
|
|
246
|
+
// stopped reporting is a caller who is told nothing for minutes.
|
|
247
|
+
const narration = run?.reported.join("\n") ?? "";
|
|
248
|
+
expect(narration).toMatch(/really asking/);
|
|
249
|
+
expect(narration).toMatch(/Researching 1 angle/);
|
|
250
|
+
expect(narration).toMatch(new RegExp(`Looking into: ${ANGLE}`));
|
|
251
|
+
expect(narration).toMatch(/writing it up/);
|
|
252
|
+
expect(narration).toMatch(/Writing up 1 angle/);
|
|
253
|
+
expect(run?.reported.at(-1)).toBe("Filing the findings.");
|
|
254
|
+
|
|
255
|
+
// The review wait, ASKED FOR and not taken: this engine records a
|
|
256
|
+
// durable `sleep` rather than suspending, so what a case can honestly
|
|
257
|
+
// claim is that the body asked — and that is the assertion that fails
|
|
258
|
+
// if the suspension is ever deleted.
|
|
259
|
+
expect(run?.slept).toEqual([{ duration: "30 seconds" }]);
|
|
260
|
+
|
|
261
|
+
// Six model calls, all through the step slot: the fan-out's width came
|
|
262
|
+
// from a journaled stage rather than from anything the body recomputed.
|
|
263
|
+
expect(steps.calls).toHaveLength(MODEL_SCRIPT.length);
|
|
264
|
+
},
|
|
265
|
+
{ stubReply: [...START_TURN] },
|
|
266
|
+
);
|
|
267
|
+
|
|
268
|
+
test(
|
|
269
|
+
"reads the live run back rather than guessing at it",
|
|
270
|
+
async ({ session, workflows }) => {
|
|
271
|
+
const steps = scriptSteps({ hold: true });
|
|
272
|
+
|
|
273
|
+
const started = await session.say(ASK);
|
|
274
|
+
const runId = startedRunId(started.toolCalls);
|
|
275
|
+
const turn = await session.say("What's it doing right now?");
|
|
276
|
+
|
|
277
|
+
// WHICH of the two readback tools the model picks is its business —
|
|
278
|
+
// the prompt offers both — so the claim is about what it was told:
|
|
279
|
+
// either the run's own latest progress line or its status, and never
|
|
280
|
+
// an answer the desk invented.
|
|
281
|
+
const read = readbacks(turn.toolCalls);
|
|
282
|
+
expect(read.length).toBeGreaterThan(0);
|
|
283
|
+
const answered = read.map((one) => one.result ?? "").join("\n");
|
|
284
|
+
expect(answered).toMatch(/really asking|Still working on it/);
|
|
285
|
+
expect(answered).not.toMatch(/Nothing started yet/);
|
|
286
|
+
|
|
287
|
+
// The load-bearing half: that readback happened while the run was
|
|
288
|
+
// genuinely in flight, which is the only state these two tools exist
|
|
289
|
+
// for.
|
|
290
|
+
const runs = await (workflows?.runs() ?? []);
|
|
291
|
+
expect(runs.find((one) => one.runId === runId)?.status).toBe("running");
|
|
292
|
+
|
|
293
|
+
await drain(workflows, steps);
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
stubReply: [
|
|
297
|
+
...START_TURN,
|
|
298
|
+
{ tool: "research_progress", args: {} },
|
|
299
|
+
"It's still working out what the question really is.",
|
|
300
|
+
],
|
|
301
|
+
},
|
|
302
|
+
);
|
|
303
|
+
|
|
304
|
+
test(
|
|
305
|
+
"says nothing is running when nothing is, and starts nothing to find out",
|
|
306
|
+
async ({ session, workflows }) => {
|
|
307
|
+
const steps = scriptSteps();
|
|
308
|
+
|
|
309
|
+
const turn = await session.say("Any news on that research I asked for?");
|
|
310
|
+
|
|
311
|
+
const read = readbacks(turn.toolCalls);
|
|
312
|
+
expect(read.length).toBeGreaterThan(0);
|
|
313
|
+
expect(read.map((one) => one.result ?? "").join("\n")).toMatch(/Nothing started yet/);
|
|
314
|
+
// A question is not a request: asking after work nobody asked for must
|
|
315
|
+
// not put a run — and a research pass's worth of model calls — on the
|
|
316
|
+
// account.
|
|
317
|
+
expect(await (workflows?.runs() ?? [])).toEqual([]);
|
|
318
|
+
expect(steps.calls).toEqual([]);
|
|
319
|
+
},
|
|
320
|
+
{
|
|
321
|
+
stubReply: [
|
|
322
|
+
{ tool: "research_status", args: {} },
|
|
323
|
+
"Nothing has been started yet — want me to look into something?",
|
|
324
|
+
],
|
|
325
|
+
},
|
|
326
|
+
);
|
|
327
|
+
},
|
|
328
|
+
{ env: EVAL_ENV },
|
|
329
|
+
);
|
|
@@ -1,5 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Addresses: the shared schema fields, and the two address changes as
|
|
3
|
+
* plan/apply pairs (see `cancel.ts` for why every mutating action is split
|
|
4
|
+
* that way).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { isToolFailure, type ToolFailure } from "@alexkroman1/aai";
|
|
1
8
|
import { z } from "zod";
|
|
2
|
-
import
|
|
9
|
+
import { resolveOrder } from "./resolve.ts";
|
|
10
|
+
import type { Address, RetailState } from "./shared.ts";
|
|
11
|
+
import { requireOwnUser } from "./store.ts";
|
|
3
12
|
|
|
4
13
|
/** Spread into both address tools' schemas. Two hand-copied zod shapes is how
|
|
5
14
|
* the order address and the profile address drift apart. */
|
|
@@ -27,3 +36,80 @@ export function formatAddress(address: Address): string {
|
|
|
27
36
|
const lines = [address.address1, address.address2].filter(Boolean).join(", ");
|
|
28
37
|
return `${lines}, ${address.city} ${address.state} ${address.zip}, ${address.country}`;
|
|
29
38
|
}
|
|
39
|
+
|
|
40
|
+
// ─── One order's shipping address ────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
export interface OrderAddressPlan {
|
|
43
|
+
readBack: string;
|
|
44
|
+
orderId: string;
|
|
45
|
+
address: Address;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function planOrderAddress(
|
|
49
|
+
state: RetailState,
|
|
50
|
+
spokenOrderId: string,
|
|
51
|
+
address: Address,
|
|
52
|
+
): OrderAddressPlan | ToolFailure {
|
|
53
|
+
const order = resolveOrder(state, spokenOrderId);
|
|
54
|
+
if (isToolFailure(order)) return order;
|
|
55
|
+
|
|
56
|
+
// Any pending variant is fine here — unlike cancel and modify-items, which
|
|
57
|
+
// require exactly 'pending'. Re-addressing a modified order is harmless.
|
|
58
|
+
if (!order.status.startsWith("pending")) {
|
|
59
|
+
return {
|
|
60
|
+
error: `Order ${order.order_id} is ${order.status}, and only a pending order's address can be changed.`,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
readBack: `ship order ${order.order_id} to ${formatAddress(address)} instead`,
|
|
66
|
+
orderId: order.order_id,
|
|
67
|
+
address: toAddress(address),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function applyOrderAddress(state: RetailState, plan: OrderAddressPlan) {
|
|
72
|
+
const order = state.store.orders[plan.orderId];
|
|
73
|
+
if (order) order.address = plan.address;
|
|
74
|
+
return {
|
|
75
|
+
order_id: plan.orderId,
|
|
76
|
+
status: order?.status ?? "pending",
|
|
77
|
+
address: plan.address,
|
|
78
|
+
message: `Order ${plan.orderId} now ships to ${formatAddress(plan.address)}.`,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// ─── The customer's default address ──────────────────────────────────────────
|
|
83
|
+
|
|
84
|
+
export interface UserAddressPlan {
|
|
85
|
+
readBack: string;
|
|
86
|
+
userId: string;
|
|
87
|
+
address: Address;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function planUserAddress(
|
|
91
|
+
state: RetailState,
|
|
92
|
+
userId: string,
|
|
93
|
+
address: Address,
|
|
94
|
+
): UserAddressPlan | ToolFailure {
|
|
95
|
+
const user = requireOwnUser(state, userId);
|
|
96
|
+
if (isToolFailure(user)) return user;
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
readBack:
|
|
100
|
+
`change your default address for future orders to ${formatAddress(address)} ` +
|
|
101
|
+
"(existing orders keep their own)",
|
|
102
|
+
userId: user.user_id,
|
|
103
|
+
address: toAddress(address),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function applyUserAddress(state: RetailState, plan: UserAddressPlan) {
|
|
108
|
+
const user = state.store.users[plan.userId];
|
|
109
|
+
if (user) user.address = plan.address;
|
|
110
|
+
return {
|
|
111
|
+
user_id: plan.userId,
|
|
112
|
+
address: plan.address,
|
|
113
|
+
message: `Default address updated to ${formatAddress(plan.address)}. Existing orders keep their own shipping addresses.`,
|
|
114
|
+
};
|
|
115
|
+
}
|