@alexkroman1/aai-cli 6.11.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/{_agent-DpH2pBJd.mjs → _agent-BTwZJu4P.mjs} +2 -2
- package/dist/{_bundler-C2BDwU7r.mjs → _bundler-CDuYl5Gb.mjs} +2 -2
- package/dist/_dev-env.d.ts +22 -2
- package/dist/{_dev-server-LLLWnFBs.mjs → _dev-server-Me06wgeF.mjs} +38 -47
- package/dist/_dev-server.d.ts +0 -17
- package/dist/{_init-C57bWAQ8.mjs → _init-BhEfQ6Yi.mjs} +3 -2
- package/dist/{_preflight-BtfaYtbE.mjs → _preflight-Co1G9Jww.mjs} +1 -1
- package/dist/_preflight.d.ts +1 -1
- package/dist/_resource-commands.d.ts +23 -0
- package/dist/{_slug-api-BxrsJXVI.mjs → _slug-api-z-USxPtL.mjs} +16 -12
- package/dist/_slug-api.d.ts +5 -2
- package/dist/{_studio-U3jhKlA3.mjs → _studio-DR5yzZ1T.mjs} +2 -1
- package/dist/{_templates-Dxb_P2Wz.mjs → _templates-4WcKOjS5.mjs} +5 -6
- package/dist/_templates.d.ts +3 -4
- package/dist/_workflow-scan.d.ts +108 -0
- package/dist/{build-Dfza2pRI.mjs → build-BBxf8vom.mjs} +6 -3
- package/dist/cli.mjs +346 -349
- package/dist/{client-bundler-BEIqgOtd.mjs → client-bundler-CWnG42cU.mjs} +2 -1
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-vbpjGzqs.mjs → delete-Dl_LSfvI.mjs} +3 -3
- package/dist/{deploy-B8lubiRT.mjs → deploy-DRC8jWr8.mjs} +5 -5
- package/dist/{dev-Uq5ujP8a.mjs → dev-BvcctCn2.mjs} +1 -1
- package/dist/eval-DfQ8cm5p.mjs +56 -0
- package/dist/eval.d.ts +43 -0
- package/dist/{init-l6xfU4xX.mjs → init-CUo87gcI.mjs} +3 -3
- package/dist/{login-C71-qz8F.mjs → login-BBuM1sxH.mjs} +3 -4
- package/dist/{logs-BWVGSjLR.mjs → logs-CQ2T46dr.mjs} +2 -2
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/CLAUDE.md +442 -80
- package/dist/scaffold/package.json +6 -3
- package/dist/scaffold/server.mjs +11 -4
- package/dist/scaffold/tsconfig.json +0 -1
- package/dist/{secret-BuMuFR4B.mjs → secret-Ct34kdgU.mjs} +2 -2
- package/dist/{storage-DzRZ-eCw.mjs → storage-i_79ZwAR.mjs} +2 -2
- package/dist/{studio-BCNUpDgP.mjs → studio-CzTzPqe1.mjs} +4 -4
- package/dist/templates/briefing-desk/agent.test.ts +350 -0
- package/dist/templates/briefing-desk/agent.ts +30 -0
- package/dist/templates/briefing-desk/shared.ts +200 -0
- package/dist/templates/briefing-desk/system-prompt.md +39 -0
- package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
- package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
- package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
- package/dist/templates/call-audit/agent.eval.test.ts +474 -0
- package/dist/templates/call-audit/agent.test.ts +57 -204
- package/dist/templates/call-audit/agent.ts +32 -19
- package/dist/templates/call-audit/client.tsx +17 -54
- package/dist/templates/call-audit/workflows/audit.ts +11 -18
- package/dist/templates/call-audit/workflows/ingest.ts +106 -114
- package/dist/templates/call-audit/workflows/media.ts +2 -12
- package/dist/templates/call-audit/workflows/summarize.ts +47 -52
- package/dist/templates/call-audit/workflows/sync-api.ts +7 -7
- package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
- package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
- package/dist/templates/dispatch-center/agent.test.ts +149 -25
- package/dist/templates/dispatch-center/client.tsx +239 -129
- package/dist/templates/dispatch-center/shared.ts +99 -1
- package/dist/templates/dispatch-center/system-prompt.md +3 -1
- package/dist/templates/dispatch-center/tools/incident_add_note.ts +16 -13
- package/dist/templates/dispatch-center/tools/incident_create.ts +19 -1
- package/dist/templates/dispatch-center/tools/incident_escalate.ts +68 -59
- package/dist/templates/dispatch-center/tools/incident_triage.ts +57 -43
- package/dist/templates/dispatch-center/tools/incident_update_status.ts +49 -40
- package/dist/templates/dispatch-center/tools/ops_run_scenario.ts +8 -2
- package/dist/templates/dispatch-center/tools/resources_dispatch.ts +69 -52
- package/dist/templates/dispatch-center/tools/resources_update_status.ts +45 -41
- package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
- package/dist/templates/embedded-assets/agent.test.ts +17 -14
- package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
- package/dist/templates/health-assistant/agent.test.ts +22 -4
- package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
- package/dist/templates/infocom-adventure/agent.test.ts +53 -24
- package/dist/templates/infocom-adventure/agent.ts +21 -0
- package/dist/templates/infocom-adventure/client.tsx +246 -164
- package/dist/templates/infocom-adventure/shared.ts +20 -9
- package/dist/templates/infocom-adventure/system-prompt.md +1 -1
- package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
- package/dist/templates/link-digest/agent.eval.test.ts +195 -0
- package/dist/templates/link-digest/agent.test.ts +24 -19
- package/dist/templates/link-digest/client.tsx +47 -61
- package/dist/templates/link-digest/workflows/digest.ts +19 -31
- package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
- package/dist/templates/night-owl/agent.eval.test.ts +272 -0
- package/dist/templates/night-owl/agent.test.ts +70 -19
- package/dist/templates/night-owl/agent.ts +5 -0
- package/dist/templates/night-owl/client.tsx +56 -56
- package/dist/templates/night-owl/shared.ts +24 -0
- package/dist/templates/night-owl/tools/recommend.ts +18 -6
- package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
- package/dist/templates/personal-finance/system-prompt.md +4 -0
- package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
- package/dist/templates/pipeline-simple/agent.ts +2 -2
- package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
- package/dist/templates/pizza-ordering/agent.test.ts +42 -27
- package/dist/templates/pizza-ordering/client.tsx +9 -26
- package/dist/templates/pizza-ordering/shared.ts +20 -1
- package/dist/templates/pizza-ordering/system-prompt.md +6 -0
- package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
- package/dist/templates/plan-and-execute/agent.test.ts +99 -58
- package/dist/templates/plan-and-execute/agent.ts +1 -1
- package/dist/templates/plan-and-execute/client.tsx +12 -15
- package/dist/templates/plan-and-execute/shared.ts +71 -2
- package/dist/templates/plan-and-execute/tools/plan_status.ts +18 -6
- package/dist/templates/plan-and-execute/tools/revise_plan.ts +18 -10
- package/dist/templates/plan-and-execute/tools/start_plan.ts +13 -2
- package/dist/templates/plan-and-execute/tools/work_next_step.ts +70 -22
- package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
- package/dist/templates/podcast-digest/agent.test.ts +717 -0
- package/dist/templates/podcast-digest/agent.ts +139 -0
- package/dist/templates/podcast-digest/client.tsx +154 -0
- package/dist/templates/podcast-digest/workflows/digest.ts +444 -0
- package/dist/templates/podcast-digest/workflows/feeds.ts +507 -0
- package/dist/templates/podcast-digest/workflows/slack.ts +88 -0
- package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
- package/dist/templates/recap-workflow/agent.test.ts +97 -94
- package/dist/templates/recap-workflow/tools/recap_progress.ts +9 -12
- package/dist/templates/recap-workflow/tools/recap_status.ts +8 -9
- package/dist/templates/recap-workflow/tools/request_recap.ts +2 -1
- package/dist/templates/recap-workflow/workflows/recap.ts +36 -40
- package/dist/templates/redline/agent.eval.test.ts +209 -0
- package/dist/templates/redline/agent.test.ts +17 -15
- package/dist/templates/redline/client.tsx +12 -12
- package/dist/templates/redline/workflows/redline.ts +19 -31
- package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
- package/dist/templates/research-workflow/agent.test.ts +60 -59
- package/dist/templates/research-workflow/tools/research_progress.ts +9 -12
- package/dist/templates/research-workflow/tools/research_status.ts +9 -11
- package/dist/templates/research-workflow/workflows/research.ts +44 -61
- package/dist/templates/retail/address.ts +87 -1
- package/dist/templates/retail/agent.eval.test.ts +314 -0
- package/dist/templates/retail/agent.test.ts +195 -112
- package/dist/templates/retail/agent.ts +35 -1
- package/dist/templates/retail/cancel.ts +116 -0
- package/dist/templates/retail/client.tsx +265 -117
- package/dist/templates/retail/payment.ts +107 -0
- package/dist/templates/retail/pending.test.ts +196 -0
- package/dist/templates/retail/pending.ts +137 -0
- package/dist/templates/retail/registry.test.ts +150 -7
- package/dist/templates/retail/returns.ts +102 -0
- package/dist/templates/retail/shared.test.ts +42 -0
- package/dist/templates/retail/shared.ts +35 -0
- package/dist/templates/retail/store.test.ts +84 -15
- package/dist/templates/retail/store.ts +231 -48
- package/dist/templates/retail/swap.test.ts +4 -4
- package/dist/templates/retail/swap.ts +248 -15
- package/dist/templates/retail/system-prompt.md +69 -28
- package/dist/templates/retail/tools/cancel_change.ts +33 -0
- package/dist/templates/retail/tools/cancel_pending_order.ts +16 -60
- package/dist/templates/retail/tools/confirm_change.ts +42 -0
- package/dist/templates/retail/tools/exchange_delivered_order_items.ts +22 -67
- package/dist/templates/retail/tools/find_user_id_by_email.ts +8 -5
- package/dist/templates/retail/tools/find_user_id_by_name_zip.ts +4 -5
- package/dist/templates/retail/tools/get_item_details.ts +3 -4
- package/dist/templates/retail/tools/get_order_details.ts +3 -4
- package/dist/templates/retail/tools/get_product_details.ts +3 -4
- package/dist/templates/retail/tools/get_user_details.ts +2 -2
- package/dist/templates/retail/tools/list_all_product_types.ts +5 -5
- package/dist/templates/retail/tools/modify_pending_order_address.ts +13 -25
- package/dist/templates/retail/tools/modify_pending_order_items.ts +23 -62
- package/dist/templates/retail/tools/modify_pending_order_payment.ts +16 -78
- package/dist/templates/retail/tools/modify_user_address.ts +13 -16
- package/dist/templates/retail/tools/return_delivered_order_items.ts +17 -72
- package/dist/templates/retail/tools/transfer_to_human_agents.ts +12 -6
- package/dist/templates/simple/agent.eval.test.ts +54 -0
- package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
- package/dist/templates/solo-rpg/agent.test.ts +227 -48
- package/dist/templates/solo-rpg/client.tsx +38 -37
- package/dist/templates/solo-rpg/shared.ts +145 -19
- package/dist/templates/solo-rpg/system-prompt.md +3 -2
- package/dist/templates/solo-rpg/tools/action_roll.ts +82 -63
- package/dist/templates/solo-rpg/tools/burn_momentum.ts +85 -50
- package/dist/templates/solo-rpg/tools/check_state.ts +24 -7
- package/dist/templates/solo-rpg/tools/load_game.ts +13 -1
- package/dist/templates/solo-rpg/tools/save_game.ts +16 -7
- package/dist/templates/solo-rpg/tools/setup_character.ts +22 -2
- package/dist/templates/solo-rpg/tools/update_state.ts +113 -100
- package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
- package/dist/templates/spoken-summary/agent.test.ts +68 -97
- package/dist/templates/spoken-summary/agent.ts +30 -17
- package/dist/templates/spoken-summary/client.tsx +10 -50
- package/dist/templates/spoken-summary/workflows/summarize.ts +17 -22
- package/dist/templates/spoken-summary/workflows/transcribe.ts +19 -26
- package/dist/templates/support-line/agent.eval.test.ts +273 -0
- package/dist/templates/support-line/agent.test.ts +11 -16
- package/dist/templates/support-line/agent.ts +1 -1
- package/dist/templates/support-line/client.tsx +9 -9
- package/dist/templates/support-line/nodes.ts +100 -0
- package/dist/templates/support-line/procedure.ts +407 -0
- package/dist/templates/support-line/system-prompt.md +2 -1
- package/dist/templates/support-line/tools/answer_question.ts +66 -4
- package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
- package/dist/templates/transcription-workflow/agent.test.ts +94 -194
- package/dist/templates/transcription-workflow/agent.ts +1 -1
- package/dist/templates/transcription-workflow/client.tsx +17 -42
- package/dist/templates/transcription-workflow/workflows/batch.ts +19 -22
- package/dist/templates/transcription-workflow/workflows/normalize.ts +90 -186
- package/dist/templates/transcription-workflow/workflows/stitch.ts +0 -11
- package/dist/templates/transcription-workflow/workflows/stream.ts +4 -5
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +6 -6
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +32 -26
- package/dist/templates/transcription-workflow/workflows/wav.ts +0 -31
- package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
- package/dist/templates/travel-concierge/agent.test.ts +174 -43
- package/dist/templates/travel-concierge/agent.ts +18 -9
- package/dist/templates/travel-concierge/client.tsx +11 -23
- package/dist/templates/travel-concierge/routing.ts +34 -15
- package/dist/templates/travel-concierge/shared.ts +180 -12
- package/dist/templates/travel-concierge/system-prompt.md +7 -1
- package/dist/templates/travel-concierge/tools/book_car_rental.ts +8 -5
- package/dist/templates/travel-concierge/tools/book_excursion.ts +8 -5
- package/dist/templates/travel-concierge/tools/book_hotel.ts +8 -5
- package/dist/templates/travel-concierge/tools/cancel_ticket.ts +8 -5
- package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
- package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
- package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
- package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
- package/dist/templates/travel-concierge/tools/update_ticket.ts +8 -5
- package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
- package/dist/templates/web-researcher/system-prompt.md +15 -4
- package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
- package/dist/test.d.ts +29 -5
- package/dist/{worker-bundler-CGD4r8Kc.mjs → worker-bundler-CCVEDjm1.mjs} +241 -3
- package/dist/worker-bundler.mjs +1 -1
- package/dist/{workflow-CFpxOFfQ.mjs → workflow-BflATMmU.mjs} +47 -12
- package/dist/workflow-bundler.d.ts +6 -1
- package/dist/workflow.d.ts +1 -1
- package/package.json +5 -4
- package/dist/eject-31gjtaHF.mjs +0 -116
- package/dist/eject.d.ts +0 -60
- package/dist/templates/call-audit/workflows/temp-media.ts +0 -138
- package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
- package/dist/templates/support-line/graph.ts +0 -224
- package/dist/{_config-D_s09e7g.mjs → _config-CmJOFsAP.mjs} +1 -1
- /package/dist/templates/plan-and-execute/{graph.ts → procedure.ts} +0 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// An EVAL: does Math Buddy delegate every calculation to code? Run it with
|
|
2
|
+
// `aai eval`.
|
|
3
|
+
//
|
|
4
|
+
// This tutor's whole design is "the model does the talking, run_code does the
|
|
5
|
+
// arithmetic" — which is also why it runs on Flash-Lite. So the claim worth
|
|
6
|
+
// pinning is not that the answer is right, it is that the answer came from
|
|
7
|
+
// CODE, and that the code is the recipe the prompt gave.
|
|
8
|
+
//
|
|
9
|
+
// Two things this file has to work around, both explained at length in
|
|
10
|
+
// `../code-interpreter/agent.eval.test.ts`: `system-prompt.md` is discovered by
|
|
11
|
+
// the build rather than imported, so an eval has to apply it or it measures a
|
|
12
|
+
// different agent; and `run_code` refuses unless the EVAL supplies an executor,
|
|
13
|
+
// which this suite does — so the cases below assert the answer the code came
|
|
14
|
+
// back with as well as the code the tutor wrote.
|
|
15
|
+
import { runInNewContext } from "node:vm";
|
|
16
|
+
import { withSystemPrompt } from "@alexkroman1/aai/manifest";
|
|
17
|
+
import { errorMessage } from "@alexkroman1/aai/utils";
|
|
18
|
+
import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
|
|
19
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
20
|
+
import { expect } from "vitest";
|
|
21
|
+
import authored from "./agent.ts";
|
|
22
|
+
import systemPrompt from "./system-prompt.md?raw";
|
|
23
|
+
|
|
24
|
+
const agentDef = withSystemPrompt(authored, systemPrompt);
|
|
25
|
+
|
|
26
|
+
/** The code every `run_code` call in this turn carried, joined. */
|
|
27
|
+
const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
|
|
28
|
+
turn.toolCalls
|
|
29
|
+
.filter((c) => c.name === "run_code")
|
|
30
|
+
.map((c) => String(c.args.code ?? ""))
|
|
31
|
+
.join("\n");
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* A `run_code` executor, so these cases can assert the ANSWER.
|
|
35
|
+
*
|
|
36
|
+
* The builtin refuses without one — the Modal container is the security
|
|
37
|
+
* boundary, and off-platform there is none — so a case could assert the CALL and
|
|
38
|
+
* the code it carried, and never what the code came back with. A `node:vm`
|
|
39
|
+
* context with a capturing `console.log` is what a developer would reach for on
|
|
40
|
+
* their own machine, and it is enough here: what runs is arithmetic, not a
|
|
41
|
+
* program. It is NOT a sandbox and does not pretend to be one; a deployed agent
|
|
42
|
+
* still gets the refusal.
|
|
43
|
+
*
|
|
44
|
+
* A template eval imports from `@alexkroman1/aai-runtime/eval` and
|
|
45
|
+
* `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
|
|
46
|
+
* node-reaching module graph into this project's TypeScript program, which is
|
|
47
|
+
* three errors in runtime files no eval ever calls. `RunCodeExecutor` is
|
|
48
|
+
* re-exported from `/eval` for exactly that reason.
|
|
49
|
+
*/
|
|
50
|
+
const runCode: RunCodeExecutor = async (code) => {
|
|
51
|
+
const lines: string[] = [];
|
|
52
|
+
const log = (...args: unknown[]): void => {
|
|
53
|
+
lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
|
|
54
|
+
};
|
|
55
|
+
try {
|
|
56
|
+
runInNewContext(code, { console: { log } }, { timeout: 1000 });
|
|
57
|
+
} catch (err) {
|
|
58
|
+
return { error: errorMessage(err) };
|
|
59
|
+
}
|
|
60
|
+
return lines.join("\n");
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/** What every `run_code` call in this turn PRINTED, joined. */
|
|
64
|
+
const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
|
|
65
|
+
turn.toolCalls
|
|
66
|
+
.filter((c) => c.name === "run_code")
|
|
67
|
+
.map((c) => c.result ?? "")
|
|
68
|
+
.join("\n");
|
|
69
|
+
|
|
70
|
+
describeEval(
|
|
71
|
+
agentDef,
|
|
72
|
+
(test) => {
|
|
73
|
+
test(
|
|
74
|
+
"converts units in code, with a real conversion factor",
|
|
75
|
+
async ({ session }) => {
|
|
76
|
+
const turn = await session.say("Convert 5 miles to kilometres.");
|
|
77
|
+
|
|
78
|
+
// The prompt hands the tutor the factors; the finding it guards against
|
|
79
|
+
// is a tutor that recites a remembered figure instead. A factor in the
|
|
80
|
+
// code is the evidence that the conversion was computed, not recalled.
|
|
81
|
+
expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
|
|
82
|
+
const code = codeIn(turn);
|
|
83
|
+
expect(code).toContain("5");
|
|
84
|
+
expect(code).toMatch(/1\.60|1\.61|0\.621|8\.04/);
|
|
85
|
+
// And the factor was applied rather than merely mentioned: five miles is
|
|
86
|
+
// 8.0467 km, so whatever rounding the tutor chose the answer starts 8.0.
|
|
87
|
+
const output = outputIn(turn);
|
|
88
|
+
expect(output, `run_code printed: ${output}`).toMatch(/8\.0/);
|
|
89
|
+
},
|
|
90
|
+
{ live: true },
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
test(
|
|
94
|
+
"rolls dice with a random draw rather than inventing numbers",
|
|
95
|
+
async ({ session }) => {
|
|
96
|
+
const turn = await session.say("Roll 3 twenty-sided dice for me.");
|
|
97
|
+
|
|
98
|
+
// A model asked for dice will happily make three numbers up, and the
|
|
99
|
+
// reply is indistinguishable from a real roll. `Math.random` in the code
|
|
100
|
+
// is the only thing that tells them apart.
|
|
101
|
+
expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
|
|
102
|
+
const code = codeIn(turn);
|
|
103
|
+
expect(code).toMatch(/Math\.random/);
|
|
104
|
+
expect(code).toContain("20");
|
|
105
|
+
expect(code).toContain("3");
|
|
106
|
+
|
|
107
|
+
// And the draw really happened: three integers, every one of them a legal
|
|
108
|
+
// face of a twenty-sided die. `Math.random` in the code says the tutor
|
|
109
|
+
// asked for a roll; this says it GOT one — a `run_code` that refused
|
|
110
|
+
// prints a sentence with no dice in it at all.
|
|
111
|
+
const output = outputIn(turn);
|
|
112
|
+
const rolled = [...output.matchAll(/\d+/g)].map((m) => Number(m[0]));
|
|
113
|
+
expect(rolled.length, `run_code printed: ${output}`).toBeGreaterThanOrEqual(3);
|
|
114
|
+
for (const face of rolled) {
|
|
115
|
+
expect(face, `run_code printed: ${output}`).toBeGreaterThanOrEqual(1);
|
|
116
|
+
expect(face, `run_code printed: ${output}`).toBeLessThanOrEqual(20);
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
{ live: true },
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
test(
|
|
123
|
+
"the run_code builtin is wired to the agent's tool executor",
|
|
124
|
+
async ({ session }) => {
|
|
125
|
+
const turn = await session.say("What is 127 times 849?");
|
|
126
|
+
|
|
127
|
+
// A tool the agent does not declare produces a `tool.called` with no
|
|
128
|
+
// result, so the paired result is what says `builtinTools: ["run_code"]`
|
|
129
|
+
// still resolves to something executable. The ANSWER rather than
|
|
130
|
+
// `toBeDefined()`, which the refusal string satisfied too.
|
|
131
|
+
const [call] = turn.toolCalls;
|
|
132
|
+
expect(call?.name).toBe("run_code");
|
|
133
|
+
expect(call?.result).toBe("107823");
|
|
134
|
+
expect(turn.completed).toBe(true);
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
stubReply: [
|
|
138
|
+
{ tool: "run_code", args: { code: "console.log(127 * 849)" } },
|
|
139
|
+
"That's 107,823.",
|
|
140
|
+
],
|
|
141
|
+
},
|
|
142
|
+
);
|
|
143
|
+
},
|
|
144
|
+
// `runCode` is what makes these cases about the ANSWER and not just the call.
|
|
145
|
+
{ runCode },
|
|
146
|
+
);
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/// <reference types="vite/client" />
|
|
2
|
+
|
|
3
|
+
// An EVAL: does the companion actually pick from its own shelf?
|
|
4
|
+
//
|
|
5
|
+
// `agent.test.ts` calls `recommend` directly, which settles what the tool does
|
|
6
|
+
// with a category and a mood it is handed. What it cannot settle is whether the
|
|
7
|
+
// MODEL turns "something cozy to watch" into `{ category: "movie", mood:
|
|
8
|
+
// "cozy" }` rather than reciting three films it likes — and whether the night's
|
|
9
|
+
// log, which lives in a `sessionSlot`, is still there two turns later.
|
|
10
|
+
//
|
|
11
|
+
// Run it with `aai eval`. Without a provider key every case runs against a
|
|
12
|
+
// SCRIPTED model (its `stubReply`): the real session, the real slot, the real
|
|
13
|
+
// tool, a fake reply. That proves the wiring and nothing about the choice.
|
|
14
|
+
|
|
15
|
+
import { runInNewContext } from "node:vm";
|
|
16
|
+
import { withSystemPrompt } from "@alexkroman1/aai/manifest";
|
|
17
|
+
import type { SessionEvent } from "@alexkroman1/aai/protocol";
|
|
18
|
+
import { withDiscoveredTools } from "@alexkroman1/aai/testing";
|
|
19
|
+
import { errorMessage } from "@alexkroman1/aai/utils";
|
|
20
|
+
import {
|
|
21
|
+
customEventsIn,
|
|
22
|
+
lastStateIn,
|
|
23
|
+
type RunCodeExecutor,
|
|
24
|
+
toolResultIn,
|
|
25
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
26
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
27
|
+
import { expect } from "vitest";
|
|
28
|
+
import { z } from "zod";
|
|
29
|
+
import authoredAgent from "./agent.ts";
|
|
30
|
+
import { CATEGORIES, MOODS } from "./shared.ts";
|
|
31
|
+
import systemPrompt from "./system-prompt.md?raw";
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
35
|
+
*
|
|
36
|
+
* The glob is written HERE rather than reached for from a shared helper because
|
|
37
|
+
* this file SHIPS — a scaffolded project has no repo helper to import. Without
|
|
38
|
+
* it the eval would drive an agent with no `recommend` at all, which is the one
|
|
39
|
+
* failure a green eval must never be able to hide.
|
|
40
|
+
*
|
|
41
|
+
* And plus its PROMPT. `agent.ts` does not declare one — `system-prompt.md` is
|
|
42
|
+
* resolved by the BUILD (`aai build`/`aai deploy`), so the raw default export
|
|
43
|
+
* carries the FRAMEWORK DEFAULT prompt. An eval that drives it measures a
|
|
44
|
+
* different agent than the one that deploys, and every tool-choice claim below
|
|
45
|
+
* then passes or fails for the wrong reason.
|
|
46
|
+
*/
|
|
47
|
+
const agentDef = withSystemPrompt(
|
|
48
|
+
withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
|
|
49
|
+
systemPrompt,
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* One `recommend` answer, and the whole projection, as the wire carries them.
|
|
54
|
+
*
|
|
55
|
+
* Schemas rather than casts, which is what `toolResultIn` and `lastStateIn`
|
|
56
|
+
* take one for: a shelf or a projection that stopped matching FAILS here naming
|
|
57
|
+
* the field, where a cast hands the assertions `undefined` and fails a line
|
|
58
|
+
* later on something unrelated.
|
|
59
|
+
*/
|
|
60
|
+
const RecSchema = z.object({
|
|
61
|
+
category: z.enum(CATEGORIES),
|
|
62
|
+
mood: z.enum(MOODS),
|
|
63
|
+
picks: z.array(z.string()),
|
|
64
|
+
});
|
|
65
|
+
const ProjectedNight = z.object({ recs: z.array(RecSchema) });
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The night's log as the PAGE has it: the last `syncState` frame pushed.
|
|
69
|
+
*
|
|
70
|
+
* This agent declares one projection, so the frame IS `nightProjection`'s
|
|
71
|
+
* result — the same `{ recs }` value `useAgentState(nightProjection)` reads in
|
|
72
|
+
* `client.tsx`. Asserting on it is asserting on what the sidebar shows.
|
|
73
|
+
*/
|
|
74
|
+
const pushedRecs = (events: readonly SessionEvent[]) =>
|
|
75
|
+
lastStateIn(events, ProjectedNight)?.recs ?? [];
|
|
76
|
+
|
|
77
|
+
/** The `wind_down` nudges in `events` — `customEventsIn` filters by name. */
|
|
78
|
+
const nudges = (events: readonly SessionEvent[]) => customEventsIn(events, "wind_down");
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* A `run_code` executor, so the sleep-cycle case can assert the ANSWER.
|
|
82
|
+
*
|
|
83
|
+
* The builtin refuses without one — the Modal container is the security
|
|
84
|
+
* boundary, and off-platform there is none — which left this template's
|
|
85
|
+
* headline feature assertable as a CALL and never as a number. A `node:vm`
|
|
86
|
+
* context with a capturing `console.log` is what a developer would reach for on
|
|
87
|
+
* their own machine, and it is enough: the code under test is arithmetic the
|
|
88
|
+
* model wrote, not a program.
|
|
89
|
+
*
|
|
90
|
+
* A template eval imports from `@alexkroman1/aai-runtime/eval` and
|
|
91
|
+
* `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
|
|
92
|
+
* node-reaching module graph into this project's TypeScript program, which is
|
|
93
|
+
* three errors in runtime files no eval ever calls. `RunCodeExecutor` is
|
|
94
|
+
* re-exported from `/eval` for exactly that reason.
|
|
95
|
+
*/
|
|
96
|
+
const runCode: RunCodeExecutor = async (code) => {
|
|
97
|
+
const lines: string[] = [];
|
|
98
|
+
const log = (...args: unknown[]): void => {
|
|
99
|
+
lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
|
|
100
|
+
};
|
|
101
|
+
try {
|
|
102
|
+
runInNewContext(code, { console: { log } }, { timeout: 1000 });
|
|
103
|
+
} catch (err) {
|
|
104
|
+
return { error: errorMessage(err) };
|
|
105
|
+
}
|
|
106
|
+
return lines.join("\n");
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
/** Two-digit, for the clock arithmetic below. */
|
|
110
|
+
const pad = (n: number): string => String(n).padStart(2, "0");
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Every bedtime `system-prompt.md`'s recipe admits for a given wake-up hour,
|
|
114
|
+
* in each of the spellings a tutor might print it in.
|
|
115
|
+
*
|
|
116
|
+
* Derived from the recipe rather than typed out — 90 minutes a cycle plus 15 to
|
|
117
|
+
* fall asleep, wrapped into the previous day — so this and the prompt cannot
|
|
118
|
+
* disagree about the arithmetic the case is checking.
|
|
119
|
+
*/
|
|
120
|
+
const bedtimesFor = (wakeHour: number): string[] =>
|
|
121
|
+
[3, 4, 5, 6].flatMap((cycles) => {
|
|
122
|
+
const at = (wakeHour * 60 - (cycles * 90 + 15) + 1440) % 1440;
|
|
123
|
+
const hour = Math.floor(at / 60);
|
|
124
|
+
const minute = pad(at % 60);
|
|
125
|
+
// Three spellings per time, because the prompt asks for HH:MM and a tutor
|
|
126
|
+
// reasonably prints any of them: padded, unpadded (measured — a live run
|
|
127
|
+
// printed "0:45" for four cycles), and the 12-hour clock.
|
|
128
|
+
return [
|
|
129
|
+
`${pad(hour)}:${minute}`,
|
|
130
|
+
`${hour}:${minute}`,
|
|
131
|
+
`${hour % 12 === 0 ? 12 : hour % 12}:${minute}`,
|
|
132
|
+
];
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
describeEval(
|
|
136
|
+
agentDef,
|
|
137
|
+
(test) => {
|
|
138
|
+
test(
|
|
139
|
+
"turns a mood into the category and mood the tool takes",
|
|
140
|
+
async ({ session }) => {
|
|
141
|
+
const turn = await session.say("I want something cozy to watch tonight.");
|
|
142
|
+
|
|
143
|
+
// "to watch" is the category and "cozy" is the mood; the shelf is the
|
|
144
|
+
// tool's, so answering from the model's own taste is the regression.
|
|
145
|
+
expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
|
|
146
|
+
const call = turn.toolCalls[0]!;
|
|
147
|
+
expect(call.args).toEqual({ category: "movie", mood: "cozy" });
|
|
148
|
+
|
|
149
|
+
// And what it read out came back from the shelf: the tool answers with
|
|
150
|
+
// the picks for exactly the pair it was asked for.
|
|
151
|
+
const rec = toolResultIn(turn.toolCalls, "recommend", RecSchema);
|
|
152
|
+
expect(rec).toMatchObject({ category: "movie", mood: "cozy" });
|
|
153
|
+
expect(rec.picks.length).toBeGreaterThan(0);
|
|
154
|
+
// It read out what the shelf handed back rather than a title of its own.
|
|
155
|
+
expect(rec.picks.some((pick) => turn.text.includes(pick))).toBe(true);
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
stubReply: [
|
|
159
|
+
{ tool: "recommend", args: { category: "movie", mood: "cozy" } },
|
|
160
|
+
"Paddington 2 is the coziest thing I own.",
|
|
161
|
+
],
|
|
162
|
+
},
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
test(
|
|
166
|
+
"the night's log keeps what an earlier turn picked, newest first",
|
|
167
|
+
async ({ session }) => {
|
|
168
|
+
await session.say("I want something cozy to watch tonight.");
|
|
169
|
+
const turn = await session.say("Now give me something spooky to read.");
|
|
170
|
+
|
|
171
|
+
expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
|
|
172
|
+
expect(turn.toolCalls[0]!.args).toEqual({ category: "book", mood: "spooky" });
|
|
173
|
+
|
|
174
|
+
// The slot survived the turn boundary: the frame the page renders after
|
|
175
|
+
// the second answer still carries the first, and the newest is first —
|
|
176
|
+
// which is the order the sidebar lists them in.
|
|
177
|
+
const recs = pushedRecs(session.events());
|
|
178
|
+
expect(recs.map((r) => `${r.category}/${r.mood}`)).toEqual(["book/spooky", "movie/cozy"]);
|
|
179
|
+
},
|
|
180
|
+
{
|
|
181
|
+
stubReply: [
|
|
182
|
+
{ tool: "recommend", args: { category: "movie", mood: "cozy" } },
|
|
183
|
+
"Paddington 2 it is.",
|
|
184
|
+
{ tool: "recommend", args: { category: "book", mood: "spooky" } },
|
|
185
|
+
"Mexican Gothic, then.",
|
|
186
|
+
],
|
|
187
|
+
},
|
|
188
|
+
);
|
|
189
|
+
|
|
190
|
+
test(
|
|
191
|
+
"the wind-down nudge arrives once, on the third pick",
|
|
192
|
+
async ({ session }) => {
|
|
193
|
+
const first = await session.say("I want something cozy to watch tonight.");
|
|
194
|
+
expect(nudges(first.events)).toEqual([]);
|
|
195
|
+
const second = await session.say("Now give me something spooky to read.");
|
|
196
|
+
expect(nudges(second.events)).toEqual([]);
|
|
197
|
+
|
|
198
|
+
// Counted in the SLOT, so the third pick is only the third if the two
|
|
199
|
+
// before it were still there — and it is a `ctx.send`, not a field on the
|
|
200
|
+
// projection, so it must arrive exactly once and never be replayed.
|
|
201
|
+
const third = await session.say("And some chill music too.");
|
|
202
|
+
expect(nudges(third.events)).toHaveLength(1);
|
|
203
|
+
|
|
204
|
+
const fourth = await session.say("One funny book as well, please.");
|
|
205
|
+
expect(nudges(fourth.events)).toEqual([]);
|
|
206
|
+
expect(nudges(session.events())).toHaveLength(1);
|
|
207
|
+
expect(pushedRecs(session.events())).toHaveLength(4);
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
stubReply: [
|
|
211
|
+
{ tool: "recommend", args: { category: "movie", mood: "cozy" } },
|
|
212
|
+
"Paddington 2 it is.",
|
|
213
|
+
{ tool: "recommend", args: { category: "book", mood: "spooky" } },
|
|
214
|
+
"Mexican Gothic, then.",
|
|
215
|
+
{ tool: "recommend", args: { category: "music", mood: "chill" } },
|
|
216
|
+
"Tycho, Dive.",
|
|
217
|
+
{ tool: "recommend", args: { category: "book", mood: "funny" } },
|
|
218
|
+
"Good Omens.",
|
|
219
|
+
],
|
|
220
|
+
},
|
|
221
|
+
);
|
|
222
|
+
test(
|
|
223
|
+
"works the bedtime out in CODE, and the number is right",
|
|
224
|
+
async ({ session }) => {
|
|
225
|
+
const turn = await session.say(
|
|
226
|
+
"I need to be up at 7 in the morning. When should I fall asleep?",
|
|
227
|
+
);
|
|
228
|
+
|
|
229
|
+
const ran = turn.toolCalls.filter((c) => c.name === "run_code");
|
|
230
|
+
expect(
|
|
231
|
+
ran,
|
|
232
|
+
`tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
|
|
233
|
+
).not.toEqual([]);
|
|
234
|
+
// The recipe is the prompt's, and it is two constants: a 90-minute cycle
|
|
235
|
+
// plus the 15 minutes it takes to fall asleep. Arithmetic done in the
|
|
236
|
+
// model's head has neither of them anywhere in the code.
|
|
237
|
+
const code = ran.map((c) => String(c.args.code ?? "")).join("\n");
|
|
238
|
+
expect(code).toContain("90");
|
|
239
|
+
expect(code).toContain("15");
|
|
240
|
+
|
|
241
|
+
const output = ran.map((c) => c.result ?? "").join("\n");
|
|
242
|
+
// The builtin really EXECUTED. With no `runCode` executor this string is
|
|
243
|
+
// "run_code is only available in the sandboxed runtime", which every
|
|
244
|
+
// assertion about a CALL sails past — so this template's headline
|
|
245
|
+
// feature could be checked as a call and never as an answer.
|
|
246
|
+
expect(output).not.toMatch(/only available in the sandboxed runtime/);
|
|
247
|
+
// And the answer is a whole number of cycles back from 07:00 with the
|
|
248
|
+
// quarter hour added. A tutor that dropped the 15 lands on :00 and a
|
|
249
|
+
// tutor that guessed lands anywhere; both fail here.
|
|
250
|
+
expect(output, `run_code printed: ${output}`).toMatch(new RegExp(bedtimesFor(7).join("|")));
|
|
251
|
+
},
|
|
252
|
+
{
|
|
253
|
+
stubReply: [
|
|
254
|
+
{
|
|
255
|
+
tool: "run_code",
|
|
256
|
+
args: {
|
|
257
|
+
code: [
|
|
258
|
+
"const wake = 7 * 60;",
|
|
259
|
+
"const at = (wake - (6 * 90 + 15) + 1440) % 1440;",
|
|
260
|
+
"const two = (n) => String(n).padStart(2, '0');",
|
|
261
|
+
"console.log(two(Math.floor(at / 60)) + ':' + two(at % 60));",
|
|
262
|
+
].join("\n"),
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
"Aim for nine forty-five tonight — that's six full cycles before seven.",
|
|
266
|
+
],
|
|
267
|
+
},
|
|
268
|
+
);
|
|
269
|
+
},
|
|
270
|
+
// `runCode` is what makes the case above about an ANSWER rather than a call.
|
|
271
|
+
{ runCode },
|
|
272
|
+
);
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
/// <reference types="vite/client" />
|
|
2
2
|
|
|
3
3
|
import { toAgentConfig } from "@alexkroman1/aai/manifest";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
createToolContext,
|
|
6
|
+
parseToolInput,
|
|
7
|
+
toolInputIssues,
|
|
8
|
+
toolRunner,
|
|
9
|
+
withDiscoveredTools,
|
|
10
|
+
} from "@alexkroman1/aai/testing";
|
|
5
11
|
import { describe, expect, test } from "vitest";
|
|
6
12
|
import authoredAgent from "./agent.ts";
|
|
7
|
-
import { CATEGORIES, MOODS } from "./shared.ts";
|
|
13
|
+
import { CATEGORIES, MOODS, nightProjection, nightSlot } from "./shared.ts";
|
|
8
14
|
|
|
9
15
|
/**
|
|
10
16
|
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
@@ -17,6 +23,14 @@ const agentDef = withDiscoveredTools(
|
|
|
17
23
|
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
18
24
|
);
|
|
19
25
|
|
|
26
|
+
/**
|
|
27
|
+
* `runTool` takes the context in the ARGUMENTS' place when a tool needs none,
|
|
28
|
+
* so `toolRunner`'s second parameter takes either — which is why it is one
|
|
29
|
+
* signature rather than an overload pair. An omitted context is a fresh one,
|
|
30
|
+
* i.e. a distinct session with an empty slot.
|
|
31
|
+
*/
|
|
32
|
+
const run = toolRunner(agentDef);
|
|
33
|
+
|
|
20
34
|
describe("night-owl template", () => {
|
|
21
35
|
test("config passes manifest validation", () => {
|
|
22
36
|
// Same conversion `aai build`/`aai deploy` run.
|
|
@@ -33,22 +47,53 @@ describe("night-owl template", () => {
|
|
|
33
47
|
test("recommend is discovered from tools/", () => {
|
|
34
48
|
expect(Object.keys(agentDef.tools ?? {})).toEqual(["recommend"]);
|
|
35
49
|
});
|
|
50
|
+
|
|
51
|
+
test("the projection an untouched session pushes is an empty log", () => {
|
|
52
|
+
// What `useAgentState(nightProjection)` reads before the first tool call —
|
|
53
|
+
// derived from the slot's own default rather than guessed at in the page.
|
|
54
|
+
expect(nightProjection()).toEqual({ recs: [] });
|
|
55
|
+
});
|
|
36
56
|
});
|
|
37
57
|
|
|
38
58
|
describe("recommend", () => {
|
|
39
59
|
test("answers with picks for the category and mood asked for", async () => {
|
|
40
|
-
const
|
|
41
|
-
const result = await runTool(agentDef, "recommend", { category: "movie", mood: "cozy" }, ctx);
|
|
60
|
+
const result = await run("recommend", { category: "movie", mood: "cozy" });
|
|
42
61
|
expect(result).toMatchObject({ category: "movie", mood: "cozy" });
|
|
43
62
|
expect((result as { picks: string[] }).picks.length).toBeGreaterThan(0);
|
|
44
63
|
});
|
|
45
64
|
|
|
46
|
-
test("
|
|
47
|
-
//
|
|
48
|
-
//
|
|
65
|
+
test("the picks land in the session's own log, newest first", async () => {
|
|
66
|
+
// The log is STATE, not an event stream: `syncState` pushes this projection
|
|
67
|
+
// after every tool call, so a page that reloads mid-session resumes with it
|
|
68
|
+
// rather than starting empty.
|
|
69
|
+
const ctx = createToolContext();
|
|
70
|
+
const first = await run("recommend", { category: "book", mood: "spooky" }, ctx);
|
|
71
|
+
const second = await run("recommend", { category: "music", mood: "chill" }, ctx);
|
|
72
|
+
expect(nightProjection(nightSlot.get(ctx))).toEqual({ recs: [second, first] });
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("two calls with no shared context are two sessions", async () => {
|
|
76
|
+
// The other half of the same rule, and the one that bites: an omitted
|
|
77
|
+
// context is a FRESH session, so nothing accumulates across these calls.
|
|
49
78
|
const ctx = createToolContext();
|
|
50
|
-
|
|
51
|
-
expect(ctx.
|
|
79
|
+
await run("recommend", { category: "book", mood: "cozy" });
|
|
80
|
+
expect(nightSlot.get(ctx).recs).toHaveLength(0);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("the wind-down nudge is sent once, on the third pick", async () => {
|
|
84
|
+
// A moment rather than state, which is why it is a `ctx.send` the page
|
|
85
|
+
// consumes with `useEvent` and not a field on the projection: re-delivering
|
|
86
|
+
// it on every reconnect would be nagging.
|
|
87
|
+
const ctx = createToolContext();
|
|
88
|
+
await run("recommend", { category: "movie", mood: "cozy" }, ctx);
|
|
89
|
+
await run("recommend", { category: "music", mood: "cozy" }, ctx);
|
|
90
|
+
expect(ctx.sent).toEqual([]);
|
|
91
|
+
await run("recommend", { category: "book", mood: "cozy" }, ctx);
|
|
92
|
+
expect(ctx.sent).toEqual([
|
|
93
|
+
{ event: "wind_down", data: "Three picks in. Want me to work out your bedtime?" },
|
|
94
|
+
]);
|
|
95
|
+
await run("recommend", { category: "movie", mood: "chill" }, ctx);
|
|
96
|
+
expect(ctx.sent).toHaveLength(1);
|
|
52
97
|
});
|
|
53
98
|
|
|
54
99
|
test("every category/mood pair the schema admits has picks behind it", async () => {
|
|
@@ -58,23 +103,29 @@ describe("recommend", () => {
|
|
|
58
103
|
// package's guide records three shipped tools having.
|
|
59
104
|
for (const category of CATEGORIES) {
|
|
60
105
|
for (const mood of MOODS) {
|
|
61
|
-
const result = await
|
|
62
|
-
agentDef,
|
|
63
|
-
"recommend",
|
|
64
|
-
{ category, mood },
|
|
65
|
-
createToolContext(),
|
|
66
|
-
);
|
|
106
|
+
const result = await run("recommend", { category, mood });
|
|
67
107
|
expect((result as { picks: string[] }).picks, `${category}/${mood}`).not.toHaveLength(0);
|
|
68
108
|
}
|
|
69
109
|
}
|
|
70
110
|
});
|
|
71
111
|
|
|
112
|
+
test("the schema accepts a category/mood pair from the enums", async () => {
|
|
113
|
+
const parsed = await parseToolInput<{ category: string; mood: string }>(agentDef, "recommend", {
|
|
114
|
+
category: "movie",
|
|
115
|
+
mood: "cozy",
|
|
116
|
+
});
|
|
117
|
+
expect(parsed).toEqual({ category: "movie", mood: "cozy" });
|
|
118
|
+
});
|
|
119
|
+
|
|
72
120
|
test("a mood outside the enum is refused by the schema", async () => {
|
|
73
121
|
// The wire boundary: an LLM tool call is untyped, so the schema is the only
|
|
74
122
|
// thing between a hallucinated mood and an index into `undefined`.
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
123
|
+
// `toolInputIssues` is the SDK's own ask — `~standard` is a vendor wire
|
|
124
|
+
// contract, and the detail a hand-rolled version gets wrong first is that
|
|
125
|
+
// `.validate` may be sync or async, so a missing `await` leaves `.issues`
|
|
126
|
+
// undefined and the negative test passes for the wrong reason.
|
|
127
|
+
expect(
|
|
128
|
+
await toolInputIssues(agentDef, "recommend", { category: "movie", mood: "melancholy" }),
|
|
129
|
+
).toBeDefined();
|
|
79
130
|
});
|
|
80
131
|
});
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { agent } from "@alexkroman1/aai";
|
|
2
|
+
import { nightProjection } from "./shared.ts";
|
|
2
3
|
|
|
3
4
|
export default agent({
|
|
4
5
|
name: "Night Owl",
|
|
6
|
+
// The night's recommendation log, pushed to the client after every tool
|
|
7
|
+
// call. The page renders `useAgentState(nightProjection)` and keeps no copy
|
|
8
|
+
// of its own, so a reload resumes with the list intact.
|
|
9
|
+
syncState: nightProjection,
|
|
5
10
|
greeting:
|
|
6
11
|
"Hey there, night owl. Try asking me for a cozy movie recommendation, or tell me what time you need to wake up and I'll calculate the best time to fall asleep.",
|
|
7
12
|
builtinTools: ["run_code"],
|