@alexkroman1/aai-cli 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/{_agent-D8zBb3M5.mjs → _agent-DMyOab9_.mjs} +2 -2
- package/dist/{_bundler-DFS4xxqE.mjs → _bundler-Cjaxa2wi.mjs} +1 -1
- package/dist/{_config-D3F9km8X.mjs → _config-5AEqhh-O.mjs} +2 -2
- package/dist/{_dev-server-BBUWo0sb.mjs → _dev-server-vV05Fnki.mjs} +3 -3
- package/dist/{_init-DU-sXH6S.mjs → _init-BZ9t_Kz-.mjs} +3 -56
- package/dist/{_slug-api-19R5kZ0U.mjs → _slug-api-DaqQJHk8.mjs} +1 -1
- package/dist/_templates-Bv8CR800.mjs +69 -0
- package/dist/_templates.d.ts +6 -0
- package/dist/{build-Xq1xxoSg.mjs → build-D_PgQOD4.mjs} +1 -1
- package/dist/cli.mjs +59 -17
- package/dist/{client-bundler-BWDkUeEP.mjs → client-bundler-yiWoXrgb.mjs} +3 -0
- package/dist/client-bundler.d.ts +9 -0
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-kSXFLxek.mjs → delete-DXilFBb1.mjs} +1 -1
- package/dist/{deploy-Ci0X1gp2.mjs → deploy-1eaXcfUw.mjs} +3 -3
- package/dist/{dev-g7sGt8qX.mjs → dev-gVNdGFYY.mjs} +1 -1
- package/dist/{init-BjeK8crW.mjs → init-DoU4_txp.mjs} +3 -3
- package/dist/login-C59ZHzuO.mjs +109 -0
- package/dist/login.d.ts +34 -0
- package/dist/scaffold/CLAUDE.md +190 -120
- package/dist/scaffold/package.json +3 -3
- package/dist/{secret-DOva9OGk.mjs → secret-CGAIAbUx.mjs} +1 -1
- package/dist/{storage-vM6HjDZZ.mjs → storage-CnhOayhm.mjs} +1 -1
- package/dist/templates/code-interpreter/agent.ts +1 -2
- package/dist/templates/dispatch-center/agent.test.ts +3 -3
- package/dist/templates/dispatch-center/agent.ts +7 -7
- package/dist/templates/dispatch-center/client.tsx +7 -2
- package/dist/templates/dispatch-center/tools/incident_add_note.ts +1 -1
- package/dist/templates/dispatch-center/tools/incident_create.ts +1 -1
- package/dist/templates/dispatch-center/tools/incident_escalate.ts +1 -1
- package/dist/templates/dispatch-center/tools/incident_get.ts +1 -1
- package/dist/templates/dispatch-center/tools/incident_triage.ts +1 -1
- package/dist/templates/dispatch-center/tools/incident_update_status.ts +1 -1
- package/dist/templates/dispatch-center/tools/ops_protocols.ts +1 -1
- package/dist/templates/dispatch-center/tools/ops_run_scenario.ts +1 -1
- package/dist/templates/dispatch-center/tools/resources_dispatch.ts +1 -1
- package/dist/templates/dispatch-center/tools/resources_get_available.ts +1 -1
- package/dist/templates/dispatch-center/tools/resources_update_status.ts +1 -1
- package/dist/templates/embedded-assets/agent.ts +2 -3
- package/dist/templates/health-assistant/agent.ts +3 -4
- package/dist/templates/infocom-adventure/agent.ts +10 -12
- package/dist/templates/infocom-adventure/client.tsx +5 -5
- package/dist/templates/math-buddy/agent.ts +7 -6
- package/dist/templates/night-owl/agent.ts +2 -3
- package/dist/templates/personal-finance/agent.ts +1 -2
- package/dist/templates/pipeline-simple/agent.test.ts +13 -13
- package/dist/templates/pipeline-simple/agent.ts +5 -6
- package/dist/templates/pizza-ordering/agent.test.ts +13 -11
- package/dist/templates/pizza-ordering/agent.ts +5 -6
- package/dist/templates/retail/address.ts +29 -0
- package/dist/templates/retail/agent.test.ts +1030 -0
- package/dist/templates/retail/agent.ts +71 -0
- package/dist/templates/retail/authenticate.ts +34 -0
- package/dist/templates/retail/client.tsx +459 -0
- package/dist/templates/retail/refund.ts +19 -0
- package/dist/templates/retail/registry.test.ts +182 -0
- package/dist/templates/retail/resolve.test.ts +158 -0
- package/dist/templates/retail/resolve.ts +158 -0
- package/dist/templates/retail/seed.json +4260 -0
- package/dist/templates/retail/seed.test.ts +224 -0
- package/dist/templates/retail/shared.test.ts +192 -0
- package/dist/templates/retail/shared.ts +430 -0
- package/dist/templates/retail/store.test.ts +256 -0
- package/dist/templates/retail/store.ts +243 -0
- package/dist/templates/retail/swap.test.ts +186 -0
- package/dist/templates/retail/swap.ts +138 -0
- package/dist/templates/retail/system-prompt.md +111 -0
- package/dist/templates/retail/tools/cancel_pending_order.ts +82 -0
- package/dist/templates/retail/tools/exchange_delivered_order_items.ts +83 -0
- package/dist/templates/retail/tools/find_user_id_by_email.ts +32 -0
- package/dist/templates/retail/tools/find_user_id_by_name_zip.ts +38 -0
- package/dist/templates/retail/tools/get_item_details.ts +29 -0
- package/dist/templates/retail/tools/get_order_details.ts +41 -0
- package/dist/templates/retail/tools/get_product_details.ts +33 -0
- package/dist/templates/retail/tools/get_user_details.ts +44 -0
- package/dist/templates/retail/tools/list_all_product_types.ts +24 -0
- package/dist/templates/retail/tools/modify_pending_order_address.ts +46 -0
- package/dist/templates/retail/tools/modify_pending_order_items.ts +102 -0
- package/dist/templates/retail/tools/modify_pending_order_payment.ts +103 -0
- package/dist/templates/retail/tools/modify_user_address.ts +38 -0
- package/dist/templates/retail/tools/return_delivered_order_items.ts +101 -0
- package/dist/templates/retail/tools/transfer_to_human_agents.ts +22 -0
- package/dist/templates/simple/agent.test.ts +18 -0
- package/dist/templates/simple/agent.ts +3 -0
- package/dist/templates/solo-rpg/agent.test.ts +2 -2
- package/dist/templates/solo-rpg/agent.ts +1 -2
- package/dist/templates/solo-rpg/tools/action_roll.ts +1 -1
- package/dist/templates/solo-rpg/tools/load_game.ts +1 -1
- package/dist/templates/solo-rpg/tools/oracle.ts +1 -1
- package/dist/templates/solo-rpg/tools/save_game.ts +1 -1
- package/dist/templates/solo-rpg/tools/setup_character.ts +1 -1
- package/dist/templates/solo-rpg/tools/update_state.ts +1 -1
- package/dist/templates/web-researcher/agent.ts +1 -2
- package/dist/typecheck.d.ts +18 -0
- package/dist/typecheck.mjs +8 -0
- package/dist/worker-bundler.d.ts +9 -1
- package/dist/worker-bundler.mjs +8 -0
- package/package.json +3 -3
package/dist/scaffold/CLAUDE.md
CHANGED
|
@@ -11,8 +11,9 @@ The fast loop: edit → `pnpm dev` (browser, talk to it) →
|
|
|
11
11
|
agent to verify behavior end-to-end. This is the primary feedback loop.
|
|
12
12
|
2. **Run `pnpm test` after logic changes** — vitest. Co-locate tests as
|
|
13
13
|
`agent.test.ts` (see `pipeline-simple` template for a reference).
|
|
14
|
-
**
|
|
15
|
-
|
|
14
|
+
**When the project has an `agent.test.ts` (the default `simple`
|
|
15
|
+
template and several others ship one), it is yours to maintain.** It
|
|
16
|
+
asserts the agent's shape — name, providers, tool names —
|
|
16
17
|
so rewriting the agent without updating it leaves a test asserting an
|
|
17
18
|
agent that no longer exists. When a test fails after your change, decide
|
|
18
19
|
which side is stale: updating the test to match the new agent is a normal
|
|
@@ -26,17 +27,23 @@ The fast loop: edit → `pnpm dev` (browser, talk to it) →
|
|
|
26
27
|
`node_modules/@alexkroman1/aai-cli/dist/templates/`. Read them directly;
|
|
27
28
|
`aai init --template <name>` scaffolds a fresh project from one. Closest
|
|
28
29
|
matches: `simple`, `pipeline-simple`, `web-researcher`, `solo-rpg`,
|
|
29
|
-
`pizza-ordering
|
|
30
|
+
`pizza-ordering`, `retail` (the most complex — 15 tools over a
|
|
31
|
+
relational store, with a `syncState`-driven UI). When reading SDK
|
|
32
|
+
types under
|
|
33
|
+
`node_modules/@alexkroman1/aai*/dist/`, note the built entry points
|
|
34
|
+
re-export with source specifiers (`"./sdk/constants.ts"`,
|
|
35
|
+
`"./components/button.tsx"`) — rewrite `.ts`/`.tsx` to `.d.ts` to find
|
|
36
|
+
the shipped file.
|
|
30
37
|
|
|
31
38
|
## CLI
|
|
32
39
|
|
|
33
40
|
```sh
|
|
34
41
|
npx @alexkroman1/aai-cli init # Scaffold a new agent
|
|
42
|
+
npx @alexkroman1/aai-cli templates # List available templates
|
|
35
43
|
npx @alexkroman1/aai-cli dev # Start local dev server
|
|
36
44
|
npx @alexkroman1/aai-cli test # Run agent.test.ts via vitest
|
|
37
45
|
npx @alexkroman1/aai-cli build # Bundle and validate
|
|
38
46
|
npx @alexkroman1/aai-cli deploy # Deploy to production
|
|
39
|
-
npx @alexkroman1/aai-cli deploy -y # Deploy without prompts
|
|
40
47
|
npx @alexkroman1/aai-cli delete # Remove deployed agent
|
|
41
48
|
npx @alexkroman1/aai-cli secret put NAME # Set a secret
|
|
42
49
|
npx @alexkroman1/aai-cli secret delete NAME
|
|
@@ -44,7 +51,7 @@ npx @alexkroman1/aai-cli secret list
|
|
|
44
51
|
```
|
|
45
52
|
|
|
46
53
|
The scaffold's `package.json` exposes `dev`, `build`, `test`, and `deploy`
|
|
47
|
-
as `pnpm <name>` shortcuts. Other commands (`init`, `delete`, `secret`)
|
|
54
|
+
as `pnpm <name>` shortcuts. Other commands (`init`, `templates`, `delete`, `secret`)
|
|
48
55
|
are CLI-only.
|
|
49
56
|
|
|
50
57
|
## Project structure
|
|
@@ -64,23 +71,32 @@ my-agent/
|
|
|
64
71
|
|
|
65
72
|
## `agent()` API
|
|
66
73
|
|
|
67
|
-
```ts
|
|
74
|
+
```ts no-check
|
|
68
75
|
import { agent } from "@alexkroman1/aai";
|
|
69
76
|
|
|
70
77
|
export default agent({
|
|
71
78
|
name: string; // required — display name
|
|
72
79
|
systemPrompt?: string; // default: general voice assistant
|
|
80
|
+
// (`system` is an accepted alias)
|
|
73
81
|
greeting?: string; // default: "Hey there..."
|
|
82
|
+
voice?: string; // TTS voice for the default pipeline, e.g. "michael"
|
|
83
|
+
// (shorthand for tts: assemblyAITts({ voice });
|
|
84
|
+
// invalid with an explicit `tts` or with `s2s`)
|
|
85
|
+
stt?: SttProvider; // pipeline stage overrides — set any subset;
|
|
86
|
+
llm?: LlmProvider | string; // unset stages default to AssemblyAI
|
|
87
|
+
tts?: TtsProvider; // (llm also takes a model-id string)
|
|
88
|
+
s2s?: S2sProvider; // explicit opt-in to speech-to-speech mode
|
|
74
89
|
sttPrompt?: string; // STT guidance for jargon/acronyms
|
|
75
90
|
builtinTools?: BuiltinTool[]; // see built-in tools table
|
|
76
91
|
tools?: Record<string, ToolDef>;
|
|
77
92
|
maxSteps?: number; // default: 10 — max tool calls per turn
|
|
78
|
-
toolChoice?: "auto" | "required"
|
|
93
|
+
toolChoice?: ToolChoice; // "auto" (default) | "required" | "none"
|
|
94
|
+
// | { type: "tool", toolName }
|
|
79
95
|
idleTimeoutMs?: number; // disconnect after inactivity (ms)
|
|
80
96
|
silenceTimeoutMs?: number; // pipeline only — assistant speaks up after this much user silence (ms)
|
|
81
97
|
silencePrompt?: string; // instruction injected on silence timeout (requires silenceTimeoutMs)
|
|
82
98
|
minBargeInWords?: number; // pipeline only — words before user speech interrupts the reply (default 2)
|
|
83
|
-
interruptionMinDurationMs?: number; // pipeline only — sustained speech (ms) before an interim barge-in interrupts (default 0
|
|
99
|
+
interruptionMinDurationMs?: number; // pipeline only — sustained speech (ms) before an interim barge-in interrupts (default 500; 0 disables)
|
|
84
100
|
holdPhrase?: string; // pipeline only — spoken before a silent tool-call turn (default "One moment."; "" disables)
|
|
85
101
|
falseInterruptionTimeoutMs?: number; // pipeline only — resume an interrupted reply if no user turn commits (default 2000; 0 disables)
|
|
86
102
|
state?: () => S; // per-session mutable state, exposed as ctx.state
|
|
@@ -90,41 +106,60 @@ export default agent({
|
|
|
90
106
|
});
|
|
91
107
|
```
|
|
92
108
|
|
|
93
|
-
>
|
|
94
|
-
>
|
|
109
|
+
> Unless `s2s` is set, the agent runs in **Pipeline mode** — see the section
|
|
110
|
+
> below. Declare any subset of `stt`/`llm`/`tts`; unset stages default to
|
|
111
|
+
> AssemblyAI. `llm` also accepts a model-id string: `"creator/model"` routes
|
|
112
|
+
> through the Vercel AI Gateway (`AI_GATEWAY_API_KEY`), a bare id through
|
|
113
|
+
> the AssemblyAI LLM Gateway (`ASSEMBLYAI_API_KEY`).
|
|
95
114
|
|
|
96
115
|
Minimal agent — a cascaded pipeline, which is what you should build unless
|
|
97
116
|
the user asks for the speech-to-speech API:
|
|
98
117
|
|
|
99
118
|
```ts
|
|
100
|
-
import { agent
|
|
119
|
+
import { agent } from "@alexkroman1/aai";
|
|
101
120
|
|
|
102
121
|
export default agent({
|
|
103
122
|
name: "My Agent",
|
|
104
|
-
...assemblyAIPipeline(),
|
|
105
123
|
});
|
|
106
124
|
```
|
|
107
125
|
|
|
108
|
-
|
|
109
|
-
the one key a published agent is guaranteed to have.
|
|
110
|
-
|
|
126
|
+
No provider fields means the default all-AssemblyAI pipeline: all three
|
|
127
|
+
stages bill to the one key a published agent is guaranteed to have. Pick
|
|
128
|
+
its voice with the `voice` field — this is the normal way to choose a
|
|
129
|
+
voice:
|
|
111
130
|
|
|
112
131
|
```ts
|
|
113
|
-
import { agent
|
|
114
|
-
import { assemblyAI as assemblyAITts } from "@alexkroman1/aai/tts";
|
|
132
|
+
import { agent } from "@alexkroman1/aai";
|
|
115
133
|
|
|
116
134
|
export default agent({
|
|
117
135
|
name: "My Agent",
|
|
118
|
-
|
|
119
|
-
tts: assemblyAITts({ voice: "paul" }),
|
|
136
|
+
voice: "paul",
|
|
120
137
|
});
|
|
121
138
|
```
|
|
122
139
|
|
|
123
|
-
|
|
140
|
+
Swap a single stage by declaring just that field — everything you leave
|
|
141
|
+
unset stays on the default. `llm` accepts the gateway model id as a plain
|
|
142
|
+
string:
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
import { agent } from "@alexkroman1/aai";
|
|
146
|
+
|
|
147
|
+
export default agent({
|
|
148
|
+
name: "My Agent",
|
|
149
|
+
llm: "claude-sonnet-4-6",
|
|
150
|
+
});
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`assemblyAIPipeline()` (from `@alexkroman1/aai`) is the explicit spelling of
|
|
154
|
+
the same default — spread it (`...assemblyAIPipeline({ region: "eu" })`) when
|
|
155
|
+
you want the three stages visible in the config or EU data residency across
|
|
156
|
+
STT and the LLM gateway. Speech-to-speech (S2S) mode is an explicit opt-in
|
|
157
|
+
via the `s2s` field — see below.
|
|
124
158
|
|
|
125
159
|
System prompt from file:
|
|
126
160
|
|
|
127
161
|
```ts
|
|
162
|
+
/// <reference types="vite/client" />
|
|
128
163
|
import { agent } from "@alexkroman1/aai";
|
|
129
164
|
import systemPrompt from "./system-prompt.md?raw";
|
|
130
165
|
export default agent({ name: "My Agent", systemPrompt });
|
|
@@ -138,33 +173,56 @@ explicit the modern spelling is `with { type: "json" }`, but plain is fine.
|
|
|
138
173
|
|
|
139
174
|
## Pipeline mode
|
|
140
175
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
176
|
+
Pipeline mode is the default: omitting `stt`/`llm`/`tts` (and `s2s`) gives
|
|
177
|
+
you the all-AssemblyAI pipeline, and any stage you do declare replaces just
|
|
178
|
+
that stage — the rest keep the default.
|
|
179
|
+
|
|
180
|
+
**S2S mode is an explicit opt-in.** Setting `s2s: assemblyAIS2s()` (imported
|
|
181
|
+
from `@alexkroman1/aai`, next to `agent()`) selects AssemblyAI's
|
|
182
|
+
speech-to-speech Voice Agent API: STT, the LLM loop, and TTS run
|
|
183
|
+
service-side in one socket. Fewer moving parts, but you cannot choose the
|
|
184
|
+
model, swap a provider, or tune a stage. There is no way to reach S2S by
|
|
185
|
+
omission — only the `s2s` field selects it, and it is mutually exclusive
|
|
186
|
+
with the `stt`/`llm`/`tts` triple.
|
|
187
|
+
|
|
188
|
+
```ts
|
|
189
|
+
import { agent, assemblyAIS2s } from "@alexkroman1/aai";
|
|
190
|
+
|
|
191
|
+
export default agent({
|
|
192
|
+
name: "My Agent",
|
|
193
|
+
s2s: assemblyAIS2s(),
|
|
194
|
+
});
|
|
195
|
+
```
|
|
144
196
|
|
|
145
|
-
**Prefer pipeline mode** —
|
|
197
|
+
**Prefer pipeline mode** — the default — unless the user specifically
|
|
146
198
|
asks for the speech-to-speech API. Nearly every template ships this way, and
|
|
147
199
|
it is what the App Builder defaults to. The host runs the LLM loop locally
|
|
148
|
-
(Vercel AI SDK) with your chosen STT, LLM, and TTS. You
|
|
200
|
+
(Vercel AI SDK) with your chosen STT, LLM, and TTS. You want explicit
|
|
201
|
+
providers when:
|
|
149
202
|
|
|
150
203
|
- you want a specific LLM (Anthropic, OpenAI, Gemini, Mistral, xAI, Groq,
|
|
151
204
|
hundreds of models via OpenRouter, or 25+ models via the AssemblyAI
|
|
152
205
|
LLM Gateway)
|
|
153
|
-
- you want a specific STT model or TTS
|
|
206
|
+
- you want a specific STT model, or a non-AssemblyAI TTS provider (for the
|
|
207
|
+
default pipeline's voice, use the `voice` field instead)
|
|
154
208
|
- you need to swap providers without changing agent code
|
|
155
209
|
|
|
156
|
-
**The rule:**
|
|
157
|
-
|
|
210
|
+
**The rule:** declare only the stages you're changing — any subset of
|
|
211
|
+
`stt`, `llm`, `tts`; each unset stage runs on the AssemblyAI default.
|
|
212
|
+
Combining `s2s` with any pipeline provider or pipeline-only tuning field is
|
|
213
|
+
a compile error naming the rule, as is `voice` alongside an explicit `tts`
|
|
214
|
+
descriptor (the descriptor owns its own voice). A raw config that skips
|
|
215
|
+
`agent()` is still checked at parse time.
|
|
158
216
|
|
|
159
217
|
```ts
|
|
160
218
|
import { agent } from "@alexkroman1/aai";
|
|
161
|
-
import {
|
|
219
|
+
import { assemblyAIStt } from "@alexkroman1/aai/stt";
|
|
162
220
|
import { anthropic } from "@alexkroman1/aai/llm";
|
|
163
221
|
import { cartesia } from "@alexkroman1/aai/tts";
|
|
164
222
|
|
|
165
223
|
export default agent({
|
|
166
224
|
name: "My Agent",
|
|
167
|
-
stt:
|
|
225
|
+
stt: assemblyAIStt({ model: "universal-3-5-pro" }),
|
|
168
226
|
llm: anthropic({ model: "claude-haiku-4-5" }),
|
|
169
227
|
tts: cartesia(),
|
|
170
228
|
});
|
|
@@ -186,11 +244,12 @@ user speaks again.
|
|
|
186
244
|
**Voice-UX tuning (pipeline only):** `minBargeInWords` controls how many
|
|
187
245
|
words of user speech interrupt the assistant mid-reply (default 2, so
|
|
188
246
|
one-word backchannels like "yeah" don't cut it off);
|
|
189
|
-
`interruptionMinDurationMs` adds
|
|
190
|
-
|
|
191
|
-
detection (how long a pause ends the user's turn)
|
|
192
|
-
provider: `
|
|
193
|
-
|
|
247
|
+
`interruptionMinDurationMs` adds a sustained-speech gate on top (default
|
|
248
|
+
500 ms; `0` disables; interim transcripts only — committed turns always
|
|
249
|
+
land). End-of-turn detection (how long a pause ends the user's turn)
|
|
250
|
+
belongs to the STT provider: `assemblyAIStt({ minTurnSilenceMs })` (default
|
|
251
|
+
2000 ms) / `deepgram({ endpointing })` (default 1500 ms), so mid-utterance
|
|
252
|
+
pauses don't split a request.
|
|
194
253
|
`holdPhrase` is spoken when a turn opens with a tool call and no speech.
|
|
195
254
|
`falseInterruptionTimeoutMs` resumes an interrupted reply when a barge-in
|
|
196
255
|
turns out to be noise (no user turn commits within the window).
|
|
@@ -202,34 +261,34 @@ for the providers you actually use.
|
|
|
202
261
|
|
|
203
262
|
### STT — `@alexkroman1/aai/stt`
|
|
204
263
|
|
|
205
|
-
| Factory
|
|
206
|
-
|
|
|
207
|
-
| `
|
|
208
|
-
| `deepgram`
|
|
209
|
-
| `elevenlabs`
|
|
210
|
-
| `soniox`
|
|
264
|
+
| Factory | Default model | Env var |
|
|
265
|
+
| --------------- | ---------------------- | -------------------- |
|
|
266
|
+
| `assemblyAIStt` | `"universal-3-5-pro"` | `ASSEMBLYAI_API_KEY` |
|
|
267
|
+
| `deepgram` | `"nova-3"` | `DEEPGRAM_API_KEY` |
|
|
268
|
+
| `elevenlabs` | `"scribe_v2_realtime"` | `ELEVENLABS_API_KEY` |
|
|
269
|
+
| `soniox` | `"stt-rt-v3"` | `SONIOX_API_KEY` |
|
|
211
270
|
|
|
212
271
|
All STT factories accept `{ model?: string, ... }`. Bare calls
|
|
213
272
|
(`deepgram()`, `soniox()`, etc.) use the default model.
|
|
214
273
|
|
|
215
|
-
`
|
|
274
|
+
`assemblyAIStt` accepts an optional `region: "eu"` for EU data residency —
|
|
216
275
|
it routes streaming transcription to AssemblyAI's EU endpoints. EU-region
|
|
217
276
|
API keys require it; the US endpoints reject them. Example:
|
|
218
|
-
`
|
|
277
|
+
`assemblyAIStt({ model: "universal-3-5-pro", region: "eu" })`.
|
|
219
278
|
|
|
220
279
|
### LLM — `@alexkroman1/aai/llm`
|
|
221
280
|
|
|
222
|
-
| Factory
|
|
223
|
-
|
|
|
224
|
-
| `anthropic`
|
|
225
|
-
| `openai`
|
|
226
|
-
| `google`
|
|
227
|
-
| `mistral`
|
|
228
|
-
| `xai`
|
|
229
|
-
| `groq`
|
|
230
|
-
| `openrouter
|
|
231
|
-
| `gateway`
|
|
232
|
-
| `
|
|
281
|
+
| Factory | SDK package | Env var |
|
|
282
|
+
| --------------- | ------------------- | ------------------------------ |
|
|
283
|
+
| `anthropic` | `@ai-sdk/anthropic` | `ANTHROPIC_API_KEY` |
|
|
284
|
+
| `openai` | `@ai-sdk/openai` | `OPENAI_API_KEY` |
|
|
285
|
+
| `google` | `@ai-sdk/google` | `GOOGLE_GENERATIVE_AI_API_KEY` |
|
|
286
|
+
| `mistral` | `@ai-sdk/mistral` | `MISTRAL_API_KEY` |
|
|
287
|
+
| `xai` | `@ai-sdk/xai` | `XAI_API_KEY` |
|
|
288
|
+
| `groq` | `@ai-sdk/groq` | `GROQ_API_KEY` |
|
|
289
|
+
| `openrouter` | `@ai-sdk/openai` | `OPENROUTER_API_KEY` |
|
|
290
|
+
| `gateway` | `ai` (built in) | `AI_GATEWAY_API_KEY` |
|
|
291
|
+
| `assemblyAILlm` | `@ai-sdk/openai` | `ASSEMBLYAI_API_KEY` |
|
|
233
292
|
|
|
234
293
|
LLM factories require `{ model: string }`. Example:
|
|
235
294
|
`anthropic({ model: "claude-haiku-4-5" })`.
|
|
@@ -246,62 +305,60 @@ hundreds of models addressed as `"creator/model"`, e.g.
|
|
|
246
305
|
`gateway({ model: "zai/glm-4.6" })`. It needs no extra SDK install
|
|
247
306
|
(the gateway client ships inside the `ai` package).
|
|
248
307
|
|
|
249
|
-
`
|
|
308
|
+
`assemblyAILlm` routes through the [AssemblyAI LLM
|
|
250
309
|
Gateway](https://www.assemblyai.com/docs/llm-gateway) — an
|
|
251
310
|
OpenAI-compatible endpoint fronting 25+ models (Claude, GPT, Gemini,
|
|
252
|
-
etc.) with the same API key as AssemblyAI STT.
|
|
253
|
-
`
|
|
254
|
-
factory, so alias one when using both:
|
|
311
|
+
etc.) with the same API key as AssemblyAI STT. A bare model-id string on
|
|
312
|
+
`llm` is shorthand for it, and unset stages keep the AssemblyAI default:
|
|
255
313
|
|
|
256
314
|
```ts
|
|
257
|
-
import {
|
|
258
|
-
import { assemblyAI as assemblyAILlm } from "@alexkroman1/aai/llm";
|
|
315
|
+
import { agent } from "@alexkroman1/aai";
|
|
259
316
|
|
|
260
317
|
export default agent({
|
|
261
318
|
name: "My Agent",
|
|
262
|
-
|
|
263
|
-
llm: assemblyAILlm({ model: "claude-sonnet-4-6" }),
|
|
264
|
-
tts: cartesia(),
|
|
319
|
+
llm: "claude-sonnet-4-6",
|
|
265
320
|
});
|
|
266
321
|
```
|
|
267
322
|
|
|
268
|
-
|
|
323
|
+
`assemblyAILlm({ model, region: "eu" })` is the explicit form; `region`
|
|
324
|
+
selects EU data residency.
|
|
325
|
+
|
|
326
|
+
Mixing providers works the same way — declare the stages you're changing:
|
|
269
327
|
|
|
270
328
|
```ts
|
|
271
329
|
import { agent } from "@alexkroman1/aai";
|
|
272
|
-
import {
|
|
273
|
-
import { assemblyAI as assemblyAILlm } from "@alexkroman1/aai/llm";
|
|
274
|
-
import { assemblyAI as assemblyAITts } from "@alexkroman1/aai/tts";
|
|
330
|
+
import { cartesia } from "@alexkroman1/aai/tts";
|
|
275
331
|
|
|
276
332
|
export default agent({
|
|
277
333
|
name: "My Agent",
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
tts: assemblyAITts({ voice: "vera" }),
|
|
334
|
+
llm: "claude-sonnet-4-6",
|
|
335
|
+
tts: cartesia(),
|
|
281
336
|
});
|
|
282
337
|
```
|
|
283
338
|
|
|
284
339
|
### TTS — `@alexkroman1/aai/tts`
|
|
285
340
|
|
|
286
|
-
| Factory
|
|
287
|
-
|
|
|
288
|
-
| `
|
|
289
|
-
| `cartesia`
|
|
290
|
-
| `rime`
|
|
341
|
+
| Factory | Default voice | Env var |
|
|
342
|
+
| --------------- | ---------------------------------------- | -------------------- |
|
|
343
|
+
| `assemblyAITts` | `"jane"` | `ASSEMBLYAI_API_KEY` |
|
|
344
|
+
| `cartesia` | `"f786b574-daa5-4673-aa0c-cbe3e8534c02"` | `CARTESIA_API_KEY` |
|
|
345
|
+
| `rime` | `"cove"` (model `mistv2`) | `RIME_API_KEY` |
|
|
291
346
|
|
|
292
|
-
Bare calls (`
|
|
347
|
+
Bare calls (`assemblyAITts()`, `cartesia()`, `rime()`) use the defaults.
|
|
293
348
|
Override with `{ voice, model, language }`.
|
|
294
349
|
|
|
295
350
|
**AssemblyAI TTS** shares `ASSEMBLYAI_API_KEY` with AssemblyAI STT and the
|
|
296
|
-
LLM Gateway, so an all-AssemblyAI pipeline needs exactly one secret.
|
|
351
|
+
LLM Gateway, so an all-AssemblyAI pipeline needs exactly one secret. On the
|
|
352
|
+
default pipeline, `agent({ voice: "michael" })` is the shorthand for
|
|
353
|
+
`tts: assemblyAITts({ voice: "michael" })` — same catalog, same rules. Each
|
|
297
354
|
voice speaks one language, and this is the whole catalog — **a voice not on
|
|
298
355
|
this list is rejected after the socket opens, which leaves the agent
|
|
299
356
|
connected, "ready", and permanently silent**, so pick one from here rather
|
|
300
357
|
than guessing a plausible name:
|
|
301
358
|
|
|
302
359
|
- **English, US accent**: `alba`, `anna`, `charles`, `eve`, `george`,
|
|
303
|
-
`jane
|
|
304
|
-
- **English, UK accent**: `paul`, `vera`
|
|
360
|
+
`jane` (the default), `jean`, `mary`, `michael`
|
|
361
|
+
- **English, UK accent**: `paul`, `vera`
|
|
305
362
|
- **Native accent, code-switches with English**: `estelle` (fr),
|
|
306
363
|
`giovanni` (it), `juergen` (de), `lola` (es), `rafael` (pt)
|
|
307
364
|
|
|
@@ -312,8 +369,7 @@ Set
|
|
|
312
369
|
`language` only alongside a voice that speaks it, as an ISO 639-1 code —
|
|
313
370
|
`"en"`, `"fr"`, `"de"`, `"it"`, `"pt"`, `"es"` are the six the catalog
|
|
314
371
|
covers, and the SDK translates each to the full name the service wants.
|
|
315
|
-
Anything else fails at session start.
|
|
316
|
-
`assemblyAI` in `/stt`, `/llm`, and `/tts`, alias on import.
|
|
372
|
+
Anything else fails at session start.
|
|
317
373
|
|
|
318
374
|
**Rime quirk:** language uses ISO 639-3 three-letter codes (e.g. `"eng"`
|
|
319
375
|
not `"en"`).
|
|
@@ -323,13 +379,13 @@ Set provider keys the same way as any secret: `.env` for local dev,
|
|
|
323
379
|
|
|
324
380
|
## `tool()` API
|
|
325
381
|
|
|
326
|
-
```ts
|
|
382
|
+
```ts no-check
|
|
327
383
|
import { tool } from "@alexkroman1/aai";
|
|
328
384
|
import { z } from "zod";
|
|
329
385
|
|
|
330
386
|
const myTool = tool({
|
|
331
387
|
description: string; // shown to LLM — decides when to call
|
|
332
|
-
|
|
388
|
+
inputSchema?: z.ZodObject; // Zod schema (omit for no-arg tools)
|
|
333
389
|
execute(args, ctx): unknown; // sync or async
|
|
334
390
|
});
|
|
335
391
|
```
|
|
@@ -339,13 +395,13 @@ same way in `aai dev` and deployed.
|
|
|
339
395
|
|
|
340
396
|
### `ctx` (ToolContext)
|
|
341
397
|
|
|
342
|
-
```ts
|
|
398
|
+
```ts no-check
|
|
343
399
|
ctx.env: Readonly<Record<string, string>> // secrets from .env / aai secret put
|
|
344
400
|
ctx.state: S // per-session mutable state (agent's `state` factory)
|
|
345
401
|
ctx.db: Db // SQL database, needs storage enabled (see Database section)
|
|
346
402
|
ctx.messages: readonly Message[] // conversation history [{role, content}]
|
|
347
403
|
ctx.sessionId: string // unique session ID
|
|
348
|
-
ctx.send(event: string, data: unknown): void // push custom event to browser client
|
|
404
|
+
ctx.send(event: string, data: unknown): void // push custom event to browser client (silently dropped over 64 KB JSON)
|
|
349
405
|
ctx.generate(opts): Promise<{ text, object? }> // one-shot LLM call (host-side)
|
|
350
406
|
```
|
|
351
407
|
|
|
@@ -353,7 +409,7 @@ ctx.generate(opts): Promise<{ text, object? }> // one-shot LLM call (host-side)
|
|
|
353
409
|
the project's tsconfig turns off `noImplicitAny`, so both of these compile
|
|
354
410
|
with no annotations and no errors:
|
|
355
411
|
|
|
356
|
-
```ts
|
|
412
|
+
```ts no-check
|
|
357
413
|
ctx.state.count++;
|
|
358
414
|
ctx.state.incidents.filter((i) => i.status === "open");
|
|
359
415
|
```
|
|
@@ -366,7 +422,7 @@ declare empty.** With `noImplicitAny` off, TypeScript does not widen an empty
|
|
|
366
422
|
initializer from what you later assign, so `[]` stays `never[]` and `null`
|
|
367
423
|
stays `null` — forever, whether or not a callback is involved:
|
|
368
424
|
|
|
369
|
-
```ts
|
|
425
|
+
```ts no-check
|
|
370
426
|
const items = []; // never[] → items.push(x) is an error
|
|
371
427
|
let best = null; // null → best = {...} is an error
|
|
372
428
|
const [picks, set] = useState([]); // never[] in a client, same thing
|
|
@@ -418,10 +474,11 @@ compile error, which is the point.
|
|
|
418
474
|
|
|
419
475
|
`ctx.generate({ prompt, system?, llm?, schema?, temperature?, maxOutputTokens? })`
|
|
420
476
|
runs one LLM generation on the host. It defaults to the agent's pipeline
|
|
421
|
-
`llm`; pass an `llm` descriptor (from `@alexkroman1/aai/llm`)
|
|
422
|
-
provider whose API key is in the agent's secrets —
|
|
423
|
-
agents use it.
|
|
424
|
-
|
|
477
|
+
`llm`; pass an `llm` descriptor (from `@alexkroman1/aai/llm`) or a model-id
|
|
478
|
+
string to use another provider whose API key is in the agent's secrets —
|
|
479
|
+
that's also how S2S agents use it. Pass a Zod schema as `schema` for typed
|
|
480
|
+
structured output (`generateObject`-style): the result's `object` carries
|
|
481
|
+
the parsed, typed value. A plain JSON Schema object also works.
|
|
425
482
|
|
|
426
483
|
### Inline tool example
|
|
427
484
|
|
|
@@ -434,7 +491,7 @@ export default agent({
|
|
|
434
491
|
tools: {
|
|
435
492
|
get_weather: tool({
|
|
436
493
|
description: "Get current weather for a city",
|
|
437
|
-
|
|
494
|
+
inputSchema: z.object({
|
|
438
495
|
city: z.string().describe("City name"),
|
|
439
496
|
}),
|
|
440
497
|
async execute({ city }, ctx) {
|
|
@@ -453,7 +510,7 @@ export default agent({
|
|
|
453
510
|
them, and they are not on `ctx`. When your own `execute` needs one, import
|
|
454
511
|
it:
|
|
455
512
|
|
|
456
|
-
```ts
|
|
513
|
+
```ts no-check
|
|
457
514
|
import { fetchJson, visitWebpage, webSearch } from "@alexkroman1/aai/tools";
|
|
458
515
|
|
|
459
516
|
execute: async ({ city }) => await fetchJson(`https://api.example.com/${city}`),
|
|
@@ -482,18 +539,18 @@ Wrapping `webSearch` in a single custom tool is the mistake to avoid — it
|
|
|
482
539
|
replaces "the model searches as needed" with one fixed query-and-summarize
|
|
483
540
|
pipeline, and no amount of prompting gets the flexibility back.
|
|
484
541
|
|
|
485
|
-
**`
|
|
486
|
-
but its VALUE must be a plain `z.object(...)` — so all of these
|
|
487
|
-
errors:
|
|
542
|
+
**`inputSchema` is a Zod object, or absent.** The field itself is
|
|
543
|
+
optional, but its VALUE must be a plain `z.object(...)` — so all of these
|
|
544
|
+
are type errors:
|
|
488
545
|
|
|
489
|
-
```ts
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
546
|
+
```ts no-check
|
|
547
|
+
inputSchema: z.undefined(), // ✗ ZodUndefined
|
|
548
|
+
inputSchema: z.void(), // ✗
|
|
549
|
+
inputSchema: z.object({ q: z.string() }).optional(), // ✗ ZodOptional
|
|
493
550
|
```
|
|
494
551
|
|
|
495
552
|
For a tool with no arguments write `tool({ description, execute })`, or
|
|
496
|
-
`
|
|
553
|
+
`inputSchema: z.object({})` if you prefer it explicit. To make an individual
|
|
497
554
|
argument optional, put `.optional()` on the FIELD, never on the object:
|
|
498
555
|
`z.object({ notes: z.string().optional() })`.
|
|
499
556
|
|
|
@@ -514,14 +571,14 @@ import { z } from "zod";
|
|
|
514
571
|
|
|
515
572
|
export const rollDice = tool({
|
|
516
573
|
description: "Roll dice",
|
|
517
|
-
|
|
574
|
+
inputSchema: z.object({ sides: z.number() }),
|
|
518
575
|
execute({ sides }) {
|
|
519
576
|
return Math.floor(Math.random() * sides) + 1;
|
|
520
577
|
},
|
|
521
578
|
});
|
|
522
579
|
```
|
|
523
580
|
|
|
524
|
-
```ts
|
|
581
|
+
```ts no-check
|
|
525
582
|
// agent.ts
|
|
526
583
|
import { agent } from "@alexkroman1/aai";
|
|
527
584
|
import { rollDice } from "./tools/roll_dice.ts";
|
|
@@ -584,7 +641,7 @@ call it from `execute` — see the builtin table above.
|
|
|
584
641
|
Persistent SQL storage scoped per app, backed by the app's own Postgres
|
|
585
642
|
schema. Access via `ctx.db`:
|
|
586
643
|
|
|
587
|
-
```ts
|
|
644
|
+
```ts no-check
|
|
588
645
|
ctx.db.query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>
|
|
589
646
|
```
|
|
590
647
|
|
|
@@ -602,7 +659,7 @@ A query returning more than 1000 rows throws — always bound reads with
|
|
|
602
659
|
|
|
603
660
|
Create tables lazily from tool code and upsert with `on conflict`:
|
|
604
661
|
|
|
605
|
-
```ts
|
|
662
|
+
```ts no-check
|
|
606
663
|
await ctx.db.query(`create table if not exists app_state (
|
|
607
664
|
key text primary key,
|
|
608
665
|
value jsonb not null,
|
|
@@ -633,6 +690,7 @@ Always import `"@alexkroman1/aai-ui/styles.css"` first.
|
|
|
633
690
|
### Tier 1 — config only (default UI)
|
|
634
691
|
|
|
635
692
|
```tsx
|
|
693
|
+
/// <reference types="vite/client" />
|
|
636
694
|
import "@alexkroman1/aai-ui/styles.css";
|
|
637
695
|
import { client } from "@alexkroman1/aai-ui";
|
|
638
696
|
|
|
@@ -642,6 +700,7 @@ client({ name: "My Agent" });
|
|
|
642
700
|
### Tier 1 with sidebar
|
|
643
701
|
|
|
644
702
|
```tsx
|
|
703
|
+
/// <reference types="vite/client" />
|
|
645
704
|
import "@alexkroman1/aai-ui/styles.css";
|
|
646
705
|
import { client, useEvent } from "@alexkroman1/aai-ui";
|
|
647
706
|
import { useState } from "react";
|
|
@@ -664,11 +723,12 @@ client({ name: "My Agent", sidebar: Sidebar });
|
|
|
664
723
|
### Tier 2 — full custom component
|
|
665
724
|
|
|
666
725
|
```tsx
|
|
726
|
+
/// <reference types="vite/client" />
|
|
667
727
|
import "@alexkroman1/aai-ui/styles.css";
|
|
668
728
|
import { client, useSession } from "@alexkroman1/aai-ui";
|
|
669
729
|
|
|
670
730
|
function MyApp() {
|
|
671
|
-
const { messages, userTranscript, started, running, start, toggle,
|
|
731
|
+
const { messages, userTranscript, started, running, start, toggle, end } =
|
|
672
732
|
useSession();
|
|
673
733
|
return (
|
|
674
734
|
<div>
|
|
@@ -679,7 +739,7 @@ function MyApp() {
|
|
|
679
739
|
) : (
|
|
680
740
|
<>
|
|
681
741
|
<button onClick={toggle}>{running ? "Stop" : "Resume"}</button>
|
|
682
|
-
<button onClick={
|
|
742
|
+
<button onClick={end}>End</button>
|
|
683
743
|
</>
|
|
684
744
|
)}
|
|
685
745
|
</div>
|
|
@@ -714,22 +774,30 @@ to put it in.
|
|
|
714
774
|
| `state` | `AgentState` | `"disconnected"` `"connecting"` `"ready"` `"listening"` `"thinking"` `"speaking"` `"error"` |
|
|
715
775
|
| `messages` | `ChatMessage[]` | `{ role, content }` |
|
|
716
776
|
| `toolCalls` | `ToolCallInfo[]` | `{ callId, name, args, status, result? }` |
|
|
717
|
-
| `customEvents` | `
|
|
777
|
+
| `customEvents` | `AgentCustomEvent[]` | `{ id, event, data }` from `ctx.send()` |
|
|
718
778
|
| `userTranscript` | `string \| null` | `null` = not speaking, `""` = speech detected, string = text |
|
|
719
779
|
| `agentTranscript` | `string \| null` | `null` = not speaking, string = streaming response |
|
|
720
780
|
| `error` | `SessionError \| null` | `{ code, message }` |
|
|
721
781
|
| `started` | `boolean` | Whether session started |
|
|
722
782
|
| `running` | `boolean` | Whether session active |
|
|
723
783
|
|
|
724
|
-
Methods: `start()`, `toggle()`, `
|
|
725
|
-
`resetState()`.
|
|
784
|
+
Methods: `start()`, `toggle()`, `end()`, `reset()`, `cancel()`,
|
|
785
|
+
`disconnect()`, `resetState()`.
|
|
786
|
+
|
|
787
|
+
- `end()` hangs up: it flips `started` back to `false` (a start-screen UI
|
|
788
|
+
shows its Start control again) and the next `start()` is a brand-new
|
|
789
|
+
session — fresh per-session tool state, greeting included. Use it for
|
|
790
|
+
End/Hang up/New game buttons.
|
|
791
|
+
- `reset()` clears the conversation but keeps the call live (`started`
|
|
792
|
+
stays `true`) — the control stays on Stop/Resume, and per-session tool
|
|
793
|
+
state survives. Use it for a "clear chat" control, not for ending.
|
|
726
794
|
|
|
727
795
|
## UI hooks
|
|
728
796
|
|
|
729
797
|
**`useToolResult`** — fires once per completed tool call (deduplicates by
|
|
730
798
|
callId):
|
|
731
799
|
|
|
732
|
-
```ts
|
|
800
|
+
```ts no-check
|
|
733
801
|
useToolResult("tool_name", (result, toolCall) => { ... }) // one tool
|
|
734
802
|
useToolResult((toolName, result, toolCall) => { ... }) // all tools
|
|
735
803
|
useToolResult<ResultType>("tool_name", (result) => { ... }) // typed (optional)
|
|
@@ -746,7 +814,7 @@ also what you want for anything that can be a string, an array, or null.
|
|
|
746
814
|
|
|
747
815
|
**`useAgentState`** — the agent's session state, pushed automatically:
|
|
748
816
|
|
|
749
|
-
```ts
|
|
817
|
+
```ts no-check
|
|
750
818
|
// agent.ts
|
|
751
819
|
export default agent({
|
|
752
820
|
state: () => ({ cart: [] as Item[], staffPin: "" }),
|
|
@@ -772,7 +840,7 @@ after every tool call and is sent only when the result changed.
|
|
|
772
840
|
|
|
773
841
|
**`useEvent`** — fires for custom events from `ctx.send()`:
|
|
774
842
|
|
|
775
|
-
```ts
|
|
843
|
+
```ts no-check
|
|
776
844
|
useEvent<DataType>("event_name", (data) => { ... })
|
|
777
845
|
```
|
|
778
846
|
|
|
@@ -805,6 +873,7 @@ beside it; writing `<StartScreen ... />` self-closing is a `TS2741:
|
|
|
805
873
|
Property 'children' is missing` build error:
|
|
806
874
|
|
|
807
875
|
```tsx
|
|
876
|
+
/// <reference types="vite/client" />
|
|
808
877
|
import "@alexkroman1/aai-ui/styles.css";
|
|
809
878
|
import { ChatView, client, StartScreen } from "@alexkroman1/aai-ui";
|
|
810
879
|
|
|
@@ -895,9 +964,10 @@ Common mistakes when working in aai projects:
|
|
|
895
964
|
- **Filter large API responses before returning them from tools.** Return
|
|
896
965
|
values are injected into LLM context. Truncate, summarize, or extract
|
|
897
966
|
only what the model needs.
|
|
898
|
-
- **
|
|
899
|
-
|
|
900
|
-
|
|
967
|
+
- **Declare only the pipeline stages you're changing.** Unset stages of
|
|
968
|
+
`stt` / `llm` / `tts` default to AssemblyAI (omit all three for the full
|
|
969
|
+
default pipeline; `voice` picks its TTS voice). S2S needs an explicit
|
|
970
|
+
`s2s: assemblyAIS2s()` and takes no pipeline fields.
|
|
901
971
|
- **Never hardcode secrets.** Use `ctx.env.MY_KEY`. `.env` for local dev,
|
|
902
972
|
`aai secret put` for production.
|
|
903
973
|
- **Don't use `useEffect` + `toolCalls` to derive state.** Use
|
|
@@ -925,10 +995,10 @@ Common mistakes when working in aai projects:
|
|
|
925
995
|
|
|
926
996
|
## Constraints
|
|
927
997
|
|
|
928
|
-
- Tool `execute` return values go into LLM context
|
|
929
|
-
large API responses
|
|
930
|
-
- `fetch`
|
|
931
|
-
(SSRF
|
|
998
|
+
- Tool `execute` return values go into LLM context, capped at 4000 chars
|
|
999
|
+
(a truncation marker replaces the tail) — filter large API responses
|
|
1000
|
+
- Tool code uses plain `fetch` with open egress; the keyless web builtins
|
|
1001
|
+
screen private/internal IPs (SSRF) when running outside a sandbox
|
|
932
1002
|
- Agent code runs in a sandboxed worker — use `fetch` for HTTP, `ctx.env`
|
|
933
1003
|
for secrets
|
|
934
1004
|
- Tool execution timeout: 30 seconds
|