@alexkroman1/aai 0.7.11 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/aai.js +1 -1
- package/dist/cli.js +693 -349
- package/dist/sdk/_internal_types.d.ts +0 -1
- package/dist/sdk/_internal_types.d.ts.map +1 -1
- package/dist/sdk/_internal_types.js.map +1 -1
- package/dist/sdk/_render_check.d.ts +7 -0
- package/dist/sdk/_render_check.d.ts.map +1 -0
- package/dist/sdk/_render_check.js +38 -0
- package/dist/sdk/_render_check.js.map +1 -0
- package/dist/sdk/builtin_tools.d.ts.map +1 -1
- package/dist/sdk/builtin_tools.js +21 -0
- package/dist/sdk/builtin_tools.js.map +1 -1
- package/dist/sdk/define_agent.d.ts.map +1 -1
- package/dist/sdk/define_agent.js +0 -1
- package/dist/sdk/define_agent.js.map +1 -1
- package/dist/sdk/direct_executor.d.ts.map +1 -1
- package/dist/sdk/direct_executor.js +0 -1
- package/dist/sdk/direct_executor.js.map +1 -1
- package/dist/sdk/memory_tools.d.ts +38 -0
- package/dist/sdk/memory_tools.d.ts.map +1 -0
- package/dist/sdk/memory_tools.js +77 -0
- package/dist/sdk/memory_tools.js.map +1 -0
- package/dist/sdk/mod.d.ts +3 -1
- package/dist/sdk/mod.d.ts.map +1 -1
- package/dist/sdk/mod.js +2 -0
- package/dist/sdk/mod.js.map +1 -1
- package/dist/sdk/protocol.d.ts +9 -3
- package/dist/sdk/protocol.d.ts.map +1 -1
- package/dist/sdk/protocol.js +2 -1
- package/dist/sdk/protocol.js.map +1 -1
- package/dist/sdk/s2s.d.ts.map +1 -1
- package/dist/sdk/s2s.js +9 -3
- package/dist/sdk/s2s.js.map +1 -1
- package/dist/sdk/session.d.ts.map +1 -1
- package/dist/sdk/session.js +5 -0
- package/dist/sdk/session.js.map +1 -1
- package/dist/sdk/types.d.ts +23 -14
- package/dist/sdk/types.d.ts.map +1 -1
- package/dist/sdk/types.js +29 -0
- package/dist/sdk/types.js.map +1 -1
- package/dist/ui/_components/app.d.ts.map +1 -1
- package/dist/ui/_components/app.js +6 -7
- package/dist/ui/_components/app.js.map +1 -1
- package/dist/ui/_components/message_list.d.ts.map +1 -1
- package/dist/ui/_components/message_list.js +2 -1
- package/dist/ui/_components/message_list.js.map +1 -1
- package/dist/ui/_components/sidebar_layout.d.ts +19 -0
- package/dist/ui/_components/sidebar_layout.d.ts.map +1 -0
- package/dist/ui/_components/sidebar_layout.js +21 -0
- package/dist/ui/_components/sidebar_layout.js.map +1 -0
- package/dist/ui/_components/start_screen.d.ts +24 -0
- package/dist/ui/_components/start_screen.d.ts.map +1 -0
- package/dist/ui/_components/start_screen.js +25 -0
- package/dist/ui/_components/start_screen.js.map +1 -0
- package/dist/ui/_hooks.d.ts +21 -0
- package/dist/ui/_hooks.d.ts.map +1 -0
- package/dist/ui/_hooks.js +35 -0
- package/dist/ui/_hooks.js.map +1 -0
- package/dist/ui/components.d.ts +20 -0
- package/dist/ui/components.d.ts.map +1 -1
- package/dist/ui/components.js +12 -0
- package/dist/ui/components.js.map +1 -1
- package/dist/ui/components_mod.d.ts +3 -2
- package/dist/ui/components_mod.d.ts.map +1 -1
- package/dist/ui/components_mod.js +3 -2
- package/dist/ui/components_mod.js.map +1 -1
- package/dist/ui/mod.d.ts +4 -3
- package/dist/ui/mod.d.ts.map +1 -1
- package/dist/ui/mod.js +3 -2
- package/dist/ui/mod.js.map +1 -1
- package/dist/ui/session.d.ts +7 -0
- package/dist/ui/session.d.ts.map +1 -1
- package/dist/ui/session.js +21 -3
- package/dist/ui/session.js.map +1 -1
- package/dist/ui/signals.d.ts +14 -0
- package/dist/ui/signals.d.ts.map +1 -1
- package/dist/ui/signals.js +55 -3
- package/dist/ui/signals.js.map +1 -1
- package/package.json +19 -2
- package/templates/_shared/CLAUDE.md +305 -116
- package/templates/_shared/package.json +4 -1
- package/templates/dispatch-center/agent.ts +43 -72
- package/templates/embedded-assets/agent.ts +4 -5
- package/templates/health-assistant/agent.ts +7 -7
- package/templates/infocom-adventure/agent.ts +20 -20
- package/templates/memory-agent/agent.ts +1 -55
- package/templates/night-owl/agent.ts +4 -4
- package/templates/night-owl/client.tsx +6 -23
- package/templates/pizza-ordering/agent.ts +214 -0
- package/templates/pizza-ordering/client.tsx +264 -0
- package/templates/smart-research/agent.ts +10 -10
|
@@ -31,6 +31,7 @@ You are helping a user build a voice agent using the **aai** framework.
|
|
|
31
31
|
aai init # Scaffold a new agent (uses simple template)
|
|
32
32
|
aai init -t <template> # Scaffold from a specific template
|
|
33
33
|
aai dev # Start local dev server
|
|
34
|
+
aai build # Bundle and validate (no server or deploy)
|
|
34
35
|
aai deploy # Bundle and deploy to production
|
|
35
36
|
aai deploy -y # Deploy without prompts
|
|
36
37
|
aai deploy --dry-run # Validate and bundle without deploying
|
|
@@ -57,6 +58,7 @@ with `aai init -t <template_name>`.
|
|
|
57
58
|
| `personal-finance` | Currency conversion, crypto prices, loan calculations, savings projections |
|
|
58
59
|
| `travel-concierge` | Trip planning, weather, flights, hotels, currency conversion |
|
|
59
60
|
| `night-owl` | Movie/music/book recs by mood, sleep calculator. **Has custom UI.** |
|
|
61
|
+
| `pizza-ordering` | Pizza order-taker with dynamic cart sidebar. **Has custom UI.** |
|
|
60
62
|
| `dispatch-center` | 911 dispatch with incident triage and resource assignment. **Has custom UI.** |
|
|
61
63
|
| `infocom-adventure` | Zork-style text adventure with state, puzzles, inventory. **Has custom UI.** |
|
|
62
64
|
| `embedded-assets` | FAQ bot using embedded JSON knowledge (no web search) |
|
|
@@ -74,15 +76,14 @@ export default defineAgent({
|
|
|
74
76
|
name: "My Agent",
|
|
75
77
|
instructions: "You are a helpful assistant that...",
|
|
76
78
|
greeting: "Hey there. What can I help you with?",
|
|
77
|
-
voice: "694f9389-aac1-45b6-b726-9d9369183238", // Sarah
|
|
78
79
|
});
|
|
79
80
|
```
|
|
80
81
|
|
|
81
82
|
### Imports
|
|
82
83
|
|
|
83
84
|
```ts
|
|
84
|
-
import { defineAgent } from "aai"; //
|
|
85
|
-
import type { BeforeStepResult, HookContext, ToolContext } from "aai";
|
|
85
|
+
import { defineAgent, memoryTools, tool } from "aai"; // defineAgent + helpers
|
|
86
|
+
import type { BeforeStepResult, BuiltinTool, HookContext, StepInfo, ToolContext } from "aai";
|
|
86
87
|
import { z } from "zod"; // Tools with typed params (included in package.json)
|
|
87
88
|
```
|
|
88
89
|
|
|
@@ -94,8 +95,6 @@ defineAgent({
|
|
|
94
95
|
name: string; // Required: display name
|
|
95
96
|
instructions?: string; // System prompt (default: general voice assistant)
|
|
96
97
|
greeting?: string; // Spoken on connect (default: "Hey, how can I help you?")
|
|
97
|
-
voice?: Voice; // Cartesia voice UUID (default: Sarah 694f9389...)
|
|
98
|
-
|
|
99
98
|
// Speech
|
|
100
99
|
sttPrompt?: string; // STT guidance for jargon, names, acronyms
|
|
101
100
|
|
|
@@ -120,42 +119,11 @@ defineAgent({
|
|
|
120
119
|
});
|
|
121
120
|
```
|
|
122
121
|
|
|
123
|
-
### Voices
|
|
124
|
-
|
|
125
|
-
Voices use Cartesia voice UUIDs. Browse all voices at
|
|
126
|
-
[play.cartesia.ai](https://play.cartesia.ai).
|
|
127
|
-
|
|
128
|
-
Common voices:
|
|
129
|
-
|
|
130
|
-
| Name | Voice ID |
|
|
131
|
-
| --------------------- | -------------------------------------- |
|
|
132
|
-
| Sarah (default) | `694f9389-aac1-45b6-b726-9d9369183238` |
|
|
133
|
-
| Customer Support Man | `a167e0f3-df7e-4d52-a9c3-f949145efdab` |
|
|
134
|
-
| Customer Support Lady | `829ccd10-f8b3-43cd-b8a0-4aeaa81f3b30` |
|
|
135
|
-
| Helpful Woman | `156fb8d2-335b-4950-9cb3-a2d33befec77` |
|
|
136
|
-
| Professional Woman | `248be419-c632-4f23-adf1-5324ed7dbf1d` |
|
|
137
|
-
| Sweet Lady | `e3827ec5-697a-4b7c-9704-1a23041bbc51` |
|
|
138
|
-
| British Lady | `79a125e8-cd45-4c13-8a67-188112f4dd22` |
|
|
139
|
-
| Calm Lady | `00a77add-48d5-4ef6-8157-71e5437b282d` |
|
|
140
|
-
| Laidback Woman | `21b81c14-f85b-436d-aff5-43f2e788ecf8` |
|
|
141
|
-
| Storyteller Lady | `996a8b96-4804-46f0-8e05-3fd4ef1a87cd` |
|
|
142
|
-
| Newslady | `bf991597-6c13-47e4-8411-91ec2de5c466` |
|
|
143
|
-
| Friendly Reading Man | `69267136-1bdc-412f-ad78-0caad210fb40` |
|
|
144
|
-
| Confident British Man | `63ff761f-c1e8-414b-b969-d1833d1c870c` |
|
|
145
|
-
| New York Man | `34575e71-908f-4ab6-ab54-b08c95d6597d` |
|
|
146
|
-
| California Girl | `b7d50908-b17c-442d-ad8d-810c63997ed9` |
|
|
147
|
-
| Newsman | `d46abd1d-2d02-43e8-819f-51fb652c1c61` |
|
|
148
|
-
| Salesman | `820a3788-2b37-4d21-847a-b65d8a68c99a` |
|
|
149
|
-
| Wise Man | `b043dea0-a007-4bbe-a708-769dc0d0c569` |
|
|
150
|
-
| Child | `2ee87190-8f84-4925-97da-e52547f9462c` |
|
|
151
|
-
|
|
152
|
-
Any Cartesia voice UUID works — the list above is just a starting point.
|
|
153
|
-
|
|
154
122
|
Use `sttPrompt` for domain-specific vocabulary:
|
|
155
123
|
|
|
156
124
|
```ts
|
|
157
125
|
export default defineAgent({
|
|
158
|
-
|
|
126
|
+
name: "Tech Support",
|
|
159
127
|
sttPrompt: "Transcribe technical terms: Kubernetes, gRPC, PostgreSQL",
|
|
160
128
|
});
|
|
161
129
|
```
|
|
@@ -195,17 +163,6 @@ aai env pull
|
|
|
195
163
|
aai env rm MY_API_KEY
|
|
196
164
|
```
|
|
197
165
|
|
|
198
|
-
Declare required env vars in the agent config so the CLI validates them at
|
|
199
|
-
deploy time:
|
|
200
|
-
|
|
201
|
-
```ts
|
|
202
|
-
export default defineAgent({
|
|
203
|
-
name: "API Agent",
|
|
204
|
-
env: ["ASSEMBLYAI_API_KEY", "MY_API_KEY"],
|
|
205
|
-
// ...
|
|
206
|
-
});
|
|
207
|
-
```
|
|
208
|
-
|
|
209
166
|
Access secrets in tool code via `ctx.env`:
|
|
210
167
|
|
|
211
168
|
```ts
|
|
@@ -214,7 +171,6 @@ import { z } from "zod";
|
|
|
214
171
|
|
|
215
172
|
export default defineAgent({
|
|
216
173
|
name: "API Agent",
|
|
217
|
-
env: ["ASSEMBLYAI_API_KEY", "MY_API_KEY"],
|
|
218
174
|
tools: {
|
|
219
175
|
call_api: {
|
|
220
176
|
description: "Call an external API",
|
|
@@ -238,26 +194,27 @@ Define tools as plain objects in the `tools` record. The `parameters` field
|
|
|
238
194
|
takes a Zod schema for type-safe argument inference:
|
|
239
195
|
|
|
240
196
|
```ts
|
|
241
|
-
import { defineAgent } from "aai";
|
|
197
|
+
import { defineAgent, tool } from "aai";
|
|
242
198
|
import { z } from "zod";
|
|
243
199
|
|
|
244
200
|
export default defineAgent({
|
|
245
201
|
name: "Weather Agent",
|
|
246
202
|
tools: {
|
|
247
|
-
get_weather: {
|
|
203
|
+
get_weather: tool({
|
|
248
204
|
description: "Get current weather for a city",
|
|
249
205
|
parameters: z.object({
|
|
250
206
|
city: z.string().describe("City name"),
|
|
251
207
|
}),
|
|
252
|
-
execute: async (
|
|
208
|
+
execute: async ({ city }, ctx) => {
|
|
209
|
+
// city is typed as string (inferred from Zod schema)
|
|
253
210
|
const data = await fetch(
|
|
254
|
-
`https://api.example.com/weather?q=${
|
|
211
|
+
`https://api.example.com/weather?q=${city}`,
|
|
255
212
|
);
|
|
256
213
|
return data.json();
|
|
257
214
|
},
|
|
258
|
-
},
|
|
215
|
+
}),
|
|
259
216
|
|
|
260
|
-
// No-parameter tools — omit `parameters`
|
|
217
|
+
// No-parameter tools — omit `parameters` and `tool()` wrapper
|
|
261
218
|
list_items: {
|
|
262
219
|
description: "List all items",
|
|
263
220
|
execute: () => items,
|
|
@@ -266,6 +223,9 @@ export default defineAgent({
|
|
|
266
223
|
});
|
|
267
224
|
```
|
|
268
225
|
|
|
226
|
+
**Important:** Wrap tool definitions in `tool()` to get typed `args` inferred
|
|
227
|
+
from the Zod `parameters` schema. Without `tool()`, args are untyped.
|
|
228
|
+
|
|
269
229
|
Zod schema patterns:
|
|
270
230
|
|
|
271
231
|
```ts
|
|
@@ -288,6 +248,7 @@ Enable via `builtinTools`.
|
|
|
288
248
|
| `fetch_json` | HTTP GET a JSON API | `url`, `headers?` |
|
|
289
249
|
| `run_code` | Execute JS in sandbox (no net/fs, 30s timeout) | `code` |
|
|
290
250
|
| `vector_search` | Search the agent's RAG knowledge base | `query`, `topK?` (default 5) |
|
|
251
|
+
| `memory` | Persistent KV memory (4 tools, see below) | — |
|
|
291
252
|
|
|
292
253
|
The agentic loop runs up to `maxSteps` iterations (default 5) and stops when the
|
|
293
254
|
LLM produces a text response.
|
|
@@ -302,10 +263,17 @@ ctx.env; // Record<string, string> — secrets from `aai env add`
|
|
|
302
263
|
ctx.abortSignal; // AbortSignal — cancelled on interruption (tools only)
|
|
303
264
|
ctx.state; // per-session state
|
|
304
265
|
ctx.kv; // persistent KV store
|
|
266
|
+
ctx.vector; // VectorStore — vector store for RAG (tools only)
|
|
305
267
|
ctx.messages; // readonly Message[] — conversation history (tools only)
|
|
306
268
|
```
|
|
307
269
|
|
|
308
|
-
Hooks get `HookContext` (same but without `abortSignal` and
|
|
270
|
+
Hooks get `HookContext` (same but without `abortSignal`, `vector`, and
|
|
271
|
+
`messages`).
|
|
272
|
+
|
|
273
|
+
**Timeouts:** Tool execution times out after **30 seconds** (`abortSignal`
|
|
274
|
+
fires). Lifecycle hooks (`onConnect`, `onTurn`, etc.) time out after **5
|
|
275
|
+
seconds**. Long-running tools should pass `ctx.abortSignal` to `fetch` and
|
|
276
|
+
check `ctx.abortSignal.aborted` in loops.
|
|
309
277
|
|
|
310
278
|
### Fetching external APIs
|
|
311
279
|
|
|
@@ -368,6 +336,37 @@ Keys are strings; use colon-separated prefixes (`"user:123"`). Max value: 64 KB.
|
|
|
368
336
|
`kv.list()` returns `KvEntry[]` where each entry has
|
|
369
337
|
`{ key: string, value: T }`.
|
|
370
338
|
|
|
339
|
+
### Memory tools (pre-built KV tools)
|
|
340
|
+
|
|
341
|
+
Add `"memory"` to `builtinTools` to give the agent four persistent KV tools:
|
|
342
|
+
`save_memory`, `recall_memory`, `list_memories`, and `forget_memory`.
|
|
343
|
+
|
|
344
|
+
```ts
|
|
345
|
+
import { defineAgent } from "aai";
|
|
346
|
+
|
|
347
|
+
export default defineAgent({
|
|
348
|
+
name: "My Agent",
|
|
349
|
+
builtinTools: ["memory"],
|
|
350
|
+
});
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
Keys use colon-separated prefixes (`"user:name"`, `"preference:color"`).
|
|
354
|
+
|
|
355
|
+
You can also spread `memoryTools()` into `tools` if you want to combine them
|
|
356
|
+
with custom tools or override individual tools:
|
|
357
|
+
|
|
358
|
+
```ts
|
|
359
|
+
import { defineAgent, memoryTools } from "aai";
|
|
360
|
+
|
|
361
|
+
export default defineAgent({
|
|
362
|
+
name: "My Agent",
|
|
363
|
+
tools: {
|
|
364
|
+
...memoryTools(),
|
|
365
|
+
// your other tools...
|
|
366
|
+
},
|
|
367
|
+
});
|
|
368
|
+
```
|
|
369
|
+
|
|
371
370
|
## Advanced patterns
|
|
372
371
|
|
|
373
372
|
### Step hooks
|
|
@@ -551,18 +550,41 @@ mount(App, {
|
|
|
551
550
|
|
|
552
551
|
Import from `aai/ui`:
|
|
553
552
|
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
|
557
|
-
|
|
|
558
|
-
| `
|
|
559
|
-
| `
|
|
560
|
-
| `
|
|
561
|
-
| `
|
|
562
|
-
| `
|
|
563
|
-
| `
|
|
553
|
+
**Layout components:**
|
|
554
|
+
|
|
555
|
+
| Component | Description |
|
|
556
|
+
| --------------- | ---------------------------------------------------- |
|
|
557
|
+
| `App` | Default full UI (StartScreen + ChatView) |
|
|
558
|
+
| `StartScreen` | Centered start card; renders children after start |
|
|
559
|
+
| `ChatView` | Chat interface (header + messages + controls) |
|
|
560
|
+
| `SidebarLayout` | Two-column layout with sidebar + main area |
|
|
561
|
+
| `Controls` | Stop/Resume + New Conversation buttons |
|
|
562
|
+
| `MessageList` | Messages with auto-scroll, tool calls, transcript |
|
|
563
|
+
|
|
564
|
+
`StartScreen` props: `{ children, icon?, title?, subtitle?, buttonText? }`
|
|
565
|
+
`SidebarLayout` props: `{ sidebar, children, width?, side? }`
|
|
566
|
+
|
|
567
|
+
**Atomic components:**
|
|
564
568
|
|
|
565
|
-
|
|
569
|
+
| Component | Props | Description |
|
|
570
|
+
| ------------------- | --------------------------------------- | ------------------------------- |
|
|
571
|
+
| `MessageBubble` | `{ message: Message }` | Single message bubble |
|
|
572
|
+
| `Transcript` | `{ userUtterance: Signal<str\|null> }` | Live STT text display |
|
|
573
|
+
| `StateIndicator` | `{ state: Signal<AgentState> }` | Colored dot + state label |
|
|
574
|
+
| `ErrorBanner` | `{ error: Signal<SessionError\|null> }` | Red error box with message |
|
|
575
|
+
| `ThinkingIndicator` | none | Animated dots during processing |
|
|
576
|
+
| `ToolCallBlock` | `{ toolCall: ToolCallInfo }` | Collapsible tool call display |
|
|
577
|
+
|
|
578
|
+
**Hooks:**
|
|
579
|
+
|
|
580
|
+
- `useAutoScroll()` — returns a `RefObject<HTMLDivElement>` to attach to a
|
|
581
|
+
sentinel div. Auto-scrolls when messages or utterances change.
|
|
582
|
+
- `useMountConfig()` — returns the `title` and `theme` passed to `mount()`.
|
|
583
|
+
|
|
584
|
+
**Important:** Components that accept `Signal<T>` props (like `StateIndicator`,
|
|
585
|
+
`Transcript`, `ErrorBanner`) expect the Signal object itself, NOT `.value`. Pass
|
|
586
|
+
`session.state`, not `session.state.value`. Passing `.value` compiles but breaks
|
|
587
|
+
reactivity silently.
|
|
566
588
|
|
|
567
589
|
### Session signals (`useSession()`)
|
|
568
590
|
|
|
@@ -570,19 +592,40 @@ Use `useMountConfig()` to access the `title` and `theme` passed to `mount()`.
|
|
|
570
592
|
`{ session, started, running, start, toggle, reset, dispose }`. Reactive agent
|
|
571
593
|
data lives on `session` (a `VoiceSession`); UI-only controls are top-level.
|
|
572
594
|
|
|
573
|
-
| Signal / field
|
|
574
|
-
|
|
|
575
|
-
| `session.state.value`
|
|
576
|
-
| `session.messages.value`
|
|
577
|
-
| `session.toolCalls.value`
|
|
578
|
-
| `session.userUtterance.value`
|
|
579
|
-
| `session.
|
|
580
|
-
| `session.
|
|
581
|
-
| `
|
|
582
|
-
| `
|
|
595
|
+
| Signal / field | Type | Description |
|
|
596
|
+
| ------------------------------ | ---------------------- | --------------------------------------------------------------- |
|
|
597
|
+
| `session.state.value` | `AgentState` | "disconnected", "connecting", "ready", "listening", etc. |
|
|
598
|
+
| `session.messages.value` | `Message[]` | `{ role, text }` objects |
|
|
599
|
+
| `session.toolCalls.value` | `ToolCallInfo[]` | `{ toolCallId, toolName, args, status, result? }` — tool calls |
|
|
600
|
+
| `session.userUtterance.value` | `string \| null` | `null` = not speaking, `""` = speech detected, string = text |
|
|
601
|
+
| `session.agentUtterance.value` | `string \| null` | `null` = not speaking, string = streaming agent response text |
|
|
602
|
+
| `session.error.value` | `SessionError \| null` | `{ code, message }` |
|
|
603
|
+
| `session.disconnected.value` | `object \| null` | `{ intentional: boolean }` when disconnected, `null` otherwise |
|
|
604
|
+
| `started.value` | `boolean` | Whether session has been started |
|
|
605
|
+
| `running.value` | `boolean` | Whether session is active |
|
|
583
606
|
|
|
584
607
|
**Methods:** `start()`, `toggle()`, `reset()`, `dispose()`
|
|
585
608
|
|
|
609
|
+
**Hooks:**
|
|
610
|
+
|
|
611
|
+
| Hook | Description |
|
|
612
|
+
| --------------------------------------------- | -------------------------------------------------------------------------------------------- |
|
|
613
|
+
| `useToolResult((toolName, result, tc) => {})` | Fires once per completed tool call with parsed JSON result. Use for carts, scoreboards, etc. |
|
|
614
|
+
|
|
615
|
+
**Signal semantics for utterances:**
|
|
616
|
+
|
|
617
|
+
- `userUtterance`: `null` = user is not speaking, `""` = speech detected but
|
|
618
|
+
no text yet (show "..."), non-empty string = partial/final transcript
|
|
619
|
+
- `agentUtterance`: `null` = agent is not speaking, non-empty string =
|
|
620
|
+
streaming response text (cleared when final `chat` message arrives)
|
|
621
|
+
- `disconnected`: `null` = connected, `{ intentional: true }` = user
|
|
622
|
+
disconnected, `{ intentional: false }` = unexpected disconnect (show
|
|
623
|
+
reconnect UI)
|
|
624
|
+
|
|
625
|
+
**UI Message type:** `{ role: "user" | "assistant"; text: string }`. Note: UI
|
|
626
|
+
messages use `text` (not `content`). The SDK `Message` type uses `content` — do
|
|
627
|
+
not mix them up in custom UIs.
|
|
628
|
+
|
|
586
629
|
### Showing tool calls in custom UI
|
|
587
630
|
|
|
588
631
|
```tsx
|
|
@@ -613,6 +656,60 @@ function App() {
|
|
|
613
656
|
mount(App);
|
|
614
657
|
```
|
|
615
658
|
|
|
659
|
+
### Building dynamic UI from tool results
|
|
660
|
+
|
|
661
|
+
Use `useToolResult` to update local state (carts, scoreboards, dashboards)
|
|
662
|
+
whenever a tool completes. It fires exactly once per completed tool call with
|
|
663
|
+
the parsed JSON result, handling deduplication internally.
|
|
664
|
+
|
|
665
|
+
```tsx
|
|
666
|
+
import "aai/ui/styles.css";
|
|
667
|
+
import { useState } from "preact/hooks";
|
|
668
|
+
import { ChatView, SidebarLayout, StartScreen, mount, useToolResult } from "aai/ui";
|
|
669
|
+
|
|
670
|
+
interface CartItem { id: number; name: string; price: number }
|
|
671
|
+
|
|
672
|
+
function ShopAgent() {
|
|
673
|
+
const [cart, setCart] = useState<CartItem[]>([]);
|
|
674
|
+
|
|
675
|
+
useToolResult((toolName, result: any) => {
|
|
676
|
+
switch (toolName) {
|
|
677
|
+
case "add_item":
|
|
678
|
+
setCart((prev) => [...prev, result.item]);
|
|
679
|
+
break;
|
|
680
|
+
case "remove_item":
|
|
681
|
+
setCart((prev) => prev.filter((i) => i.id !== result.removedId));
|
|
682
|
+
break;
|
|
683
|
+
case "clear_cart":
|
|
684
|
+
setCart([]);
|
|
685
|
+
break;
|
|
686
|
+
}
|
|
687
|
+
});
|
|
688
|
+
|
|
689
|
+
const sidebar = (
|
|
690
|
+
<div class="p-4">
|
|
691
|
+
<h3 class="text-aai-text font-bold">Cart ({cart.length})</h3>
|
|
692
|
+
{cart.map((i) => <p key={i.id} class="text-aai-text text-sm">{i.name} — ${i.price}</p>)}
|
|
693
|
+
</div>
|
|
694
|
+
);
|
|
695
|
+
|
|
696
|
+
return (
|
|
697
|
+
<StartScreen title="Shop" buttonText="Start Shopping">
|
|
698
|
+
<SidebarLayout sidebar={sidebar}>
|
|
699
|
+
<ChatView />
|
|
700
|
+
</SidebarLayout>
|
|
701
|
+
</StartScreen>
|
|
702
|
+
);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
mount(ShopAgent);
|
|
706
|
+
```
|
|
707
|
+
|
|
708
|
+
**Do NOT use `useEffect` + `session.toolCalls.value` to build derived state.**
|
|
709
|
+
That pattern re-processes every tool call on every signal change, causing
|
|
710
|
+
duplicates (e.g. items added to the cart multiple times). `useToolResult`
|
|
711
|
+
handles this correctly.
|
|
712
|
+
|
|
616
713
|
### Reacting to agent state
|
|
617
714
|
|
|
618
715
|
```tsx
|
|
@@ -643,11 +740,15 @@ mount(App);
|
|
|
643
740
|
|
|
644
741
|
### Styling custom UIs
|
|
645
742
|
|
|
646
|
-
The framework uses **Tailwind CSS v4** (compiled at bundle time).
|
|
647
|
-
|
|
743
|
+
The framework uses **Tailwind CSS v4** (compiled at bundle time). Prefer
|
|
744
|
+
Tailwind classes over inline styles — all design tokens work as classes:
|
|
745
|
+
`bg-aai-surface` not `style={{ background: "var(--color-aai-surface)" }}`,
|
|
746
|
+
`border-t border-aai-border` not `style={{ borderTop: "1px solid var(--)" }}`.
|
|
648
747
|
|
|
649
|
-
|
|
650
|
-
|
|
748
|
+
Three approaches:
|
|
749
|
+
|
|
750
|
+
1. **Tailwind classes** — `class="flex items-center gap-2 bg-aai-surface"`
|
|
751
|
+
2. **Inline styles** — only for dynamic values (`style={{ width: pixels }}`)
|
|
651
752
|
3. **Injected `<style>` tags** — for keyframes, selectors, media queries:
|
|
652
753
|
|
|
653
754
|
```tsx
|
|
@@ -667,11 +768,65 @@ function App() {
|
|
|
667
768
|
}
|
|
668
769
|
```
|
|
669
770
|
|
|
670
|
-
**CSS custom properties
|
|
771
|
+
**Design tokens** — available as CSS custom properties and Tailwind classes
|
|
772
|
+
(e.g. `bg-aai-bg`, `text-aai-text-muted`, `rounded-aai`, `font-aai`):
|
|
773
|
+
|
|
774
|
+
| Token | Tailwind class | Default |
|
|
775
|
+
| ---------------------------- | ------------------------- | ------------------------- |
|
|
776
|
+
| `--color-aai-bg` | `bg-aai-bg` | `#101010` |
|
|
777
|
+
| `--color-aai-surface` | `bg-aai-surface` | `#151515` |
|
|
778
|
+
| `--color-aai-surface-faint` | `bg-aai-surface-faint` | `rgba(255,255,255,0.031)` |
|
|
779
|
+
| `--color-aai-surface-hover` | `bg-aai-surface-hover` | `rgba(255,255,255,0.059)` |
|
|
780
|
+
| `--color-aai-border` | `border-aai-border` | `#282828` |
|
|
781
|
+
| `--color-aai-primary` | `text-aai-primary` | `#fab283` |
|
|
782
|
+
| `--color-aai-text` | `text-aai-text` | `rgba(255,255,255,0.936)` |
|
|
783
|
+
| `--color-aai-text-secondary` | `text-aai-text-secondary` | `rgba(255,255,255,0.618)` |
|
|
784
|
+
| `--color-aai-text-muted` | `text-aai-text-muted` | `rgba(255,255,255,0.284)` |
|
|
785
|
+
| `--color-aai-text-dim` | `text-aai-text-dim` | `rgba(255,255,255,0.422)` |
|
|
786
|
+
| `--color-aai-error` | `text-aai-error` | `#e06c75` |
|
|
787
|
+
| `--color-aai-ring` | `ring-aai-ring` | `#56b6c2` |
|
|
788
|
+
| `--color-aai-state-{state}` | `text-aai-state-{state}` | per-state colors |
|
|
789
|
+
| `--radius-aai` | `rounded-aai` | `6px` |
|
|
790
|
+
| `--font-aai` | `font-aai` | Inter, sans-serif |
|
|
791
|
+
| `--font-aai-mono` | `font-aai-mono` | IBM Plex Mono, mono |
|
|
792
|
+
|
|
793
|
+
The 5 core colors (`bg`, `primary`, `text`, `surface`, `border`) can be
|
|
794
|
+
overridden via `mount()` theme options. All other tokens use fixed defaults.
|
|
795
|
+
|
|
796
|
+
### Common UI patterns
|
|
797
|
+
|
|
798
|
+
**Auto-scrolling messages** — use `useAutoScroll` for custom message lists:
|
|
799
|
+
|
|
800
|
+
```tsx
|
|
801
|
+
import { useAutoScroll, useSession } from "aai/ui";
|
|
802
|
+
|
|
803
|
+
function MyChat() {
|
|
804
|
+
const { session } = useSession();
|
|
805
|
+
const bottomRef = useAutoScroll();
|
|
806
|
+
|
|
807
|
+
return (
|
|
808
|
+
<div class="overflow-y-auto">
|
|
809
|
+
{session.messages.value.map((m, i) => <p key={i}>{m.text}</p>)}
|
|
810
|
+
<div ref={bottomRef} />
|
|
811
|
+
</div>
|
|
812
|
+
);
|
|
813
|
+
}
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
Note: `MessageList` and `ChatView` already include auto-scroll. Only use
|
|
817
|
+
`useAutoScroll` when building a fully custom message list.
|
|
671
818
|
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
819
|
+
**Reading signal values in render:** Extract `.value` once at the top of the
|
|
820
|
+
component to avoid redundant signal subscriptions:
|
|
821
|
+
|
|
822
|
+
```tsx
|
|
823
|
+
function MyComponent() {
|
|
824
|
+
const { session } = useSession();
|
|
825
|
+
const state = session.state.value;
|
|
826
|
+
const msgs = session.messages.value;
|
|
827
|
+
// Use `state` and `msgs` as plain values throughout the render
|
|
828
|
+
}
|
|
829
|
+
```
|
|
675
830
|
|
|
676
831
|
## Self-hosting with `createServer()`
|
|
677
832
|
|
|
@@ -723,41 +878,65 @@ Drug interactions (RxNorm):
|
|
|
723
878
|
|
|
724
879
|
Use `fetch_json` builtin tool or `fetch` in custom tools to call these.
|
|
725
880
|
|
|
726
|
-
##
|
|
881
|
+
## Custom start screen with `StartScreen`
|
|
727
882
|
|
|
728
|
-
|
|
883
|
+
Use `StartScreen` for a branded start card that transitions to `ChatView`:
|
|
729
884
|
|
|
730
885
|
```tsx
|
|
731
886
|
import "aai/ui/styles.css";
|
|
732
|
-
import { ChatView,
|
|
887
|
+
import { ChatView, StartScreen, mount } from "aai/ui";
|
|
733
888
|
|
|
734
889
|
function MyAgent() {
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
<div class="flex flex-col items-center gap-6">
|
|
741
|
-
<h1 class="text-xl text-aai-text">My Agent</h1>
|
|
742
|
-
<button
|
|
743
|
-
class="px-8 py-3 rounded-aai bg-aai-primary text-white border-none cursor-pointer"
|
|
744
|
-
onClick={start}
|
|
745
|
-
>
|
|
746
|
-
Start
|
|
747
|
-
</button>
|
|
748
|
-
</div>
|
|
749
|
-
</div>
|
|
750
|
-
);
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
return <ChatView />;
|
|
890
|
+
return (
|
|
891
|
+
<StartScreen icon={<span>🎤</span>} title="My Agent" subtitle="Ask me anything">
|
|
892
|
+
<ChatView />
|
|
893
|
+
</StartScreen>
|
|
894
|
+
);
|
|
754
895
|
}
|
|
755
896
|
|
|
756
897
|
mount(MyAgent);
|
|
757
898
|
```
|
|
758
899
|
|
|
759
|
-
|
|
760
|
-
|
|
900
|
+
`StartScreen` handles the started/not-started transition automatically. Pass
|
|
901
|
+
`icon`, `title`, `subtitle`, and `buttonText` to customize the card.
|
|
902
|
+
|
|
903
|
+
## Sidebar layout with `SidebarLayout`
|
|
904
|
+
|
|
905
|
+
Use `SidebarLayout` for apps with a persistent side panel (cart, dashboard):
|
|
906
|
+
|
|
907
|
+
```tsx
|
|
908
|
+
import "aai/ui/styles.css";
|
|
909
|
+
import { useState } from "preact/hooks";
|
|
910
|
+
import { ChatView, SidebarLayout, StartScreen, mount, useToolResult } from "aai/ui";
|
|
911
|
+
|
|
912
|
+
function ShopAgent() {
|
|
913
|
+
const [cart, setCart] = useState<{ id: number; name: string }[]>([]);
|
|
914
|
+
|
|
915
|
+
useToolResult((toolName, result: any) => {
|
|
916
|
+
if (toolName === "add_item") setCart((prev) => [...prev, result.item]);
|
|
917
|
+
});
|
|
918
|
+
|
|
919
|
+
const sidebar = (
|
|
920
|
+
<div class="p-4">
|
|
921
|
+
<h3 class="text-aai-text font-bold">Cart ({cart.length})</h3>
|
|
922
|
+
{cart.map((i) => <p key={i.id} class="text-aai-text text-sm">{i.name}</p>)}
|
|
923
|
+
</div>
|
|
924
|
+
);
|
|
925
|
+
|
|
926
|
+
return (
|
|
927
|
+
<StartScreen title="Shop" buttonText="Start Shopping">
|
|
928
|
+
<SidebarLayout sidebar={sidebar}>
|
|
929
|
+
<ChatView />
|
|
930
|
+
</SidebarLayout>
|
|
931
|
+
</StartScreen>
|
|
932
|
+
);
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
mount(ShopAgent);
|
|
936
|
+
```
|
|
937
|
+
|
|
938
|
+
`SidebarLayout` accepts `width` (default `"20rem"`) and `side` (`"left"` or
|
|
939
|
+
`"right"`).
|
|
761
940
|
|
|
762
941
|
## Project structure
|
|
763
942
|
|
|
@@ -786,28 +965,32 @@ Good instructions tell the LLM what it is, how to behave, and when to use each
|
|
|
786
965
|
tool. Study these patterns:
|
|
787
966
|
|
|
788
967
|
**Code execution agent** — force tool use for anything computational:
|
|
789
|
-
|
|
968
|
+
|
|
969
|
+
```text
|
|
790
970
|
You MUST use the run_code tool for ANY question involving math, counting,
|
|
791
971
|
string manipulation, or data processing. NEVER do mental math or estimate.
|
|
792
972
|
Use console.log() to output intermediate steps.
|
|
793
973
|
```
|
|
794
974
|
|
|
795
975
|
**Research agent** — search before answering:
|
|
796
|
-
|
|
976
|
+
|
|
977
|
+
```text
|
|
797
978
|
Search first. Never guess or rely on memory for factual questions.
|
|
798
979
|
Use visit_webpage when search snippets aren't detailed enough.
|
|
799
980
|
For complex questions, search multiple times with different queries.
|
|
800
981
|
```
|
|
801
982
|
|
|
802
983
|
**FAQ/support agent** — stay grounded in knowledge:
|
|
803
|
-
|
|
984
|
+
|
|
985
|
+
```text
|
|
804
986
|
Always use vector_search to find relevant documentation before answering.
|
|
805
987
|
Base your answers strictly on the retrieved documentation — don't guess.
|
|
806
988
|
If search results aren't relevant, say the docs don't cover that topic.
|
|
807
989
|
```
|
|
808
990
|
|
|
809
991
|
**API-calling agent** — tell the LLM which endpoints to use:
|
|
810
|
-
|
|
992
|
+
|
|
993
|
+
```text
|
|
811
994
|
API endpoints (use fetch_json):
|
|
812
995
|
- Currency rates: https://open.er-api.com/v6/latest/{CODE}
|
|
813
996
|
Returns { rates: { USD: 1.0, EUR: 0.85, ... } }
|
|
@@ -815,7 +998,8 @@ API endpoints (use fetch_json):
|
|
|
815
998
|
```
|
|
816
999
|
|
|
817
1000
|
**Game/interactive agent** — establish world rules and voice style:
|
|
818
|
-
|
|
1001
|
+
|
|
1002
|
+
```text
|
|
819
1003
|
You ARE the game. Maintain world state, describe rooms, handle puzzles.
|
|
820
1004
|
Keep descriptions to two to four sentences. No visual formatting.
|
|
821
1005
|
Use directional words naturally: "To the north you see..." not "N: forest"
|
|
@@ -823,6 +1007,11 @@ Use directional words naturally: "To the north you see..." not "N: forest"
|
|
|
823
1007
|
|
|
824
1008
|
## Common pitfalls
|
|
825
1009
|
|
|
1010
|
+
- **Using `useEffect` to build state from tool calls** — Iterating
|
|
1011
|
+
`session.toolCalls.value` in a `useEffect` re-processes every tool call on
|
|
1012
|
+
every signal change, causing duplicates (e.g. cart items added multiple
|
|
1013
|
+
times). Use the `useToolResult` hook instead — it fires exactly once per
|
|
1014
|
+
completed tool call with proper deduplication.
|
|
826
1015
|
- **Writing `instructions` with visual formatting** — Bullets, bold, numbered
|
|
827
1016
|
lists sound terrible when spoken. Use natural transitions: "First", "Next",
|
|
828
1017
|
"Finally". Write instructions as if you're coaching a human phone operator.
|
|
@@ -843,8 +1032,8 @@ Use directional words naturally: "To the north you see..." not "N: forest"
|
|
|
843
1032
|
- **Telling the agent to be verbose** — Voice responses should be 1-3 sentences.
|
|
844
1033
|
If your `instructions` say "provide detailed explanations", the agent will
|
|
845
1034
|
monologue. Instruct it to be brief and let the user ask follow-ups.
|
|
846
|
-
- **Not
|
|
847
|
-
|
|
1035
|
+
- **Not setting env vars before deploying** — If your agent needs custom env
|
|
1036
|
+
vars, set them with `aai env add MY_KEY` before deploying.
|
|
848
1037
|
- **Forgetting SSRF restrictions on `fetch`** — The host validates all proxied
|
|
849
1038
|
fetch URLs. Requests to private/internal IP addresses (localhost, 10.x,
|
|
850
1039
|
192.168.x, etc.) are blocked.
|
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
"lint:fix": "biome check --write ."
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"@alexkroman1/aai": "*"
|
|
11
|
+
"@alexkroman1/aai": "*",
|
|
12
|
+
"@preact/signals": "^2.8.2",
|
|
13
|
+
"preact": "^10.29.0",
|
|
14
|
+
"tailwindcss": "^4.2.1"
|
|
12
15
|
},
|
|
13
16
|
"devDependencies": {
|
|
14
17
|
"@biomejs/biome": "^2",
|