@memberjunction/ai-assemblyai 0.0.1 → 5.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assemblyAIRealtime.d.ts +367 -0
- package/dist/assemblyAIRealtime.d.ts.map +1 -0
- package/dist/assemblyAIRealtime.js +631 -0
- package/dist/assemblyAIRealtime.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/package.json +26 -7
- package/readme.md +48 -0
- package/README.md +0 -45
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
import { BaseRealtimeModel, type ClientRealtimeSessionConfig, type IRealtimeSession, type RealtimeSessionParams, type RealtimeToolDefinition, type RealtimeTranscript, type RealtimeToolCall, type RealtimeUsage, type RealtimeSessionError, type JSONObject } from '@memberjunction/ai';
|
|
2
|
+
/** The single Voice Agent websocket endpoint (auth rides as a `?token=` query parameter). */
|
|
3
|
+
export declare const ASSEMBLYAI_AGENT_WS_URL = "wss://agents.assemblyai.com/v1/ws";
|
|
4
|
+
/** REST endpoint that mints one-time, short-lived client tokens for browser-direct sessions. */
|
|
5
|
+
export declare const ASSEMBLYAI_TOKEN_URL = "https://agents.assemblyai.com/v1/token";
|
|
6
|
+
/**
|
|
7
|
+
* Lifetime requested for the one-time client token minted by {@link AssemblyAIRealtime.CreateClientSession}.
|
|
8
|
+
* The token is single-use: it authenticates ONE websocket open within this window; an
|
|
9
|
+
* already-open session continues past it.
|
|
10
|
+
*/
|
|
11
|
+
export declare const ASSEMBLYAI_TOKEN_TTL_SECONDS = 300;
|
|
12
|
+
/**
|
|
13
|
+
* The Voice Agent wire format: 16-bit signed little-endian PCM, mono, 24 kHz, base64-encoded,
|
|
14
|
+
* in BOTH directions (`input.audio` up, `reply.audio` down).
|
|
15
|
+
*/
|
|
16
|
+
export declare const ASSEMBLYAI_PCM_SAMPLE_RATE = 24000;
|
|
17
|
+
/** A parsed inbound websocket frame. The Voice Agent protocol multiplexes on `type`. */
|
|
18
|
+
export interface AssemblyAIServerEvent {
|
|
19
|
+
type?: string;
|
|
20
|
+
/** `session.ready` — the provider-assigned session id (also used by `session.resume`). */
|
|
21
|
+
session_id?: string;
|
|
22
|
+
/** `transcript.user[.delta]` / `transcript.agent` — the transcribed text. */
|
|
23
|
+
text?: string;
|
|
24
|
+
/** `transcript.*` — provider conversation-item id. */
|
|
25
|
+
item_id?: string;
|
|
26
|
+
/** `reply.started` / `transcript.agent` — id of the reply the frame belongs to. */
|
|
27
|
+
reply_id?: string;
|
|
28
|
+
/** `transcript.agent` — true when the turn was cut off by a barge-in. */
|
|
29
|
+
interrupted?: boolean;
|
|
30
|
+
/** `reply.audio` — one base64 PCM16 chunk of the agent's spoken output. */
|
|
31
|
+
data?: string;
|
|
32
|
+
/** `reply.done` — `'interrupted'` when the user barged in mid-reply; absent otherwise. */
|
|
33
|
+
status?: string;
|
|
34
|
+
/** `tool.call` — correlation id the result must echo. */
|
|
35
|
+
call_id?: string;
|
|
36
|
+
/** `tool.call` — the tool name. */
|
|
37
|
+
name?: string;
|
|
38
|
+
/** `tool.call` — the arguments, ALREADY PARSED into an object by the provider. */
|
|
39
|
+
arguments?: JSONObject;
|
|
40
|
+
/** `session.error` — provider error code. */
|
|
41
|
+
code?: string;
|
|
42
|
+
/** `session.error` — human-readable error message. */
|
|
43
|
+
message?: string;
|
|
44
|
+
/** `session.error` — the offending parameter, when applicable. */
|
|
45
|
+
param?: string;
|
|
46
|
+
/** `session.updated` — echo of the applied session object. */
|
|
47
|
+
session?: JSONObject;
|
|
48
|
+
/** `session.ended` — total billable session seconds. */
|
|
49
|
+
session_duration_seconds?: number;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The minimal outbound surface of the agent websocket the session depends on. Declaring the
|
|
53
|
+
* seam as an interface lets unit tests inject a fully in-memory fake that captures outbound
|
|
54
|
+
* frames and drives {@link AssemblyAIRealtimeSession.HandleServerEvent} with provider-shaped
|
|
55
|
+
* events — no websocket, no network.
|
|
56
|
+
*/
|
|
57
|
+
export interface AssemblyAIRealtimeSocket {
|
|
58
|
+
/** Sends one JSON-serialized client frame. */
|
|
59
|
+
send(data: string): void;
|
|
60
|
+
/** Terminates the underlying connection. */
|
|
61
|
+
close(): void;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Arguments handed to {@link AssemblyAIRealtime.connectAgentSocket}: the authenticated URL
|
|
65
|
+
* plus the lifecycle callbacks, so the seam owns the entire websocket dance and tests
|
|
66
|
+
* substitute it wholesale.
|
|
67
|
+
*/
|
|
68
|
+
export interface AssemblyAIConnectArgs {
|
|
69
|
+
/** The full `wss://…?token=…` URL (API key for server-bridged, temp token for client-direct). */
|
|
70
|
+
Url: string;
|
|
71
|
+
/** Invoked with each parsed inbound frame. */
|
|
72
|
+
OnMessage: (event: AssemblyAIServerEvent) => void;
|
|
73
|
+
/** Invoked on a websocket-level error (fatal — the session is unusable). */
|
|
74
|
+
OnError: (message: string) => void;
|
|
75
|
+
/** Invoked when the websocket closes. */
|
|
76
|
+
OnClose: (code?: number, reason?: string) => void;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Real-time, full-duplex driver for the **AssemblyAI Voice Agent API** (launched April 2026),
|
|
80
|
+
* implementing the Core {@link BaseRealtimeModel} primitive. Registers as `AssemblyAIRealtime`
|
|
81
|
+
* and is resolved for `MJ: AI Models` typed `Realtime` (API-key env alias:
|
|
82
|
+
* `AI_VENDOR_API_KEY__AssemblyAIRealtime`).
|
|
83
|
+
*
|
|
84
|
+
* **What the provider is:** a single-websocket speech-to-speech stack — Universal-3 Pro
|
|
85
|
+
* streaming ASR, server-side turn detection / barge-in, LLM reasoning, JSON-Schema tool
|
|
86
|
+
* calling, and conversational TTS — billed flat per session-hour. Unlike ElevenLabs there is
|
|
87
|
+
* no server-side agent object to manage: the ENTIRE session config (prompt, tools, voice,
|
|
88
|
+
* turn detection) is supplied per-session via the first `session.update` frame, so MJ's
|
|
89
|
+
* prompt/tool authority is native here, with no managed-agent ensure flow.
|
|
90
|
+
*
|
|
91
|
+
* **Model resolution:** the Voice Agent API exposes ONE endpoint and no model selection;
|
|
92
|
+
* `params.Model` is carried through to {@link ClientRealtimeSessionConfig.Model} for
|
|
93
|
+
* bookkeeping but plays no wire role.
|
|
94
|
+
*
|
|
95
|
+
* **Topologies:**
|
|
96
|
+
* - Server-bridged ({@link StartSession}): the driver opens the websocket itself
|
|
97
|
+
* (`?token=<api key>`), sends the session config, and resolves only after `session.ready`.
|
|
98
|
+
* - Client-direct ({@link CreateClientSession}): the driver mints a ONE-TIME, short-lived
|
|
99
|
+
* client token (`GET /v1/token`) — the browser opens its own socket with it (no API key
|
|
100
|
+
* ever leaves the server) and applies the server-built session object from the
|
|
101
|
+
* `SessionConfig` pact.
|
|
102
|
+
*
|
|
103
|
+
* **No usage events:** billing is a flat hourly session rate; the websocket reports no token
|
|
104
|
+
* usage, so sessions never emit {@link IRealtimeSession.OnUsage}. (The terminal
|
|
105
|
+
* `session.ended` frame carries duration seconds, but by then the consumer has typically
|
|
106
|
+
* closed — duration accounting lives in the provider dashboard.)
|
|
107
|
+
*/
|
|
108
|
+
export declare class AssemblyAIRealtime extends BaseRealtimeModel {
|
|
109
|
+
/**
|
|
110
|
+
* Opens a server-bridged session: connects the agent websocket authenticated with the
|
|
111
|
+
* API key, sends the full session config as the FIRST frame (`session.update` — prompt,
|
|
112
|
+
* tools, voice, turn detection), and resolves only once the server's `session.ready`
|
|
113
|
+
* confirms the config is applied (driver obligation #7 — "ready only after the config is
|
|
114
|
+
* applied"). Mic frames sent before that would be dropped by the provider, and
|
|
115
|
+
* `StartSession` not resolving until ready makes that unrepresentable for consumers.
|
|
116
|
+
*/
|
|
117
|
+
StartSession(params: RealtimeSessionParams): Promise<IRealtimeSession>;
|
|
118
|
+
/**
|
|
119
|
+
* AssemblyAI supports the client-direct topology natively: a one-time temp token minted
|
|
120
|
+
* server-side authenticates the browser's own websocket.
|
|
121
|
+
*/
|
|
122
|
+
get SupportsClientDirect(): boolean;
|
|
123
|
+
/**
|
|
124
|
+
* Mints the client-direct config: a ONE-TIME client token (connect window
|
|
125
|
+
* {@link ASSEMBLYAI_TOKEN_TTL_SECONDS}s) plus the **private pact** `SessionConfig` the
|
|
126
|
+
* same-keyed `'assemblyai'` client driver consumes: `{ session, config }`, where
|
|
127
|
+
* `session` is the wire-shaped `session.update` payload (server-authored system prompt,
|
|
128
|
+
* tools, voice, turn detection) and `config` passes `params.Config` through opaquely.
|
|
129
|
+
*/
|
|
130
|
+
CreateClientSession(params: RealtimeSessionParams): Promise<ClientRealtimeSessionConfig>;
|
|
131
|
+
/**
|
|
132
|
+
* Builds the `session` object for the `session.update` frame from the Core params:
|
|
133
|
+
* - `system_prompt`: the session system prompt, with `InitialContext` folded in under a
|
|
134
|
+
* "Prior context" heading — the protocol has no history-seeding channel, so prior
|
|
135
|
+
* conversation context rides the prompt.
|
|
136
|
+
* - `tools`: the Core tool set mapped to the provider's `{ type: 'function', … }` schema.
|
|
137
|
+
* - Recognized `params.Config` keys pass through to their wire slots: `voice` →
|
|
138
|
+
* `output.voice`, `greeting` (spoken on connect; omitted = no auto-greeting),
|
|
139
|
+
* `turn_detection` → `input.turn_detection`, `keyterms` → `input.keyterms`.
|
|
140
|
+
*/
|
|
141
|
+
static BuildSessionObject(params: RealtimeSessionParams): JSONObject;
|
|
142
|
+
/** Folds optional prior context into the system prompt (no history channel exists). */
|
|
143
|
+
static ComposeSystemPrompt(systemPrompt: string, initialContext?: string): string;
|
|
144
|
+
/** Maps a Core tool definition up to a Voice Agent `function` tool schema. */
|
|
145
|
+
static MapToolToFunction(tool: RealtimeToolDefinition): JSONObject;
|
|
146
|
+
/**
|
|
147
|
+
* Canonical, order-insensitive fingerprint of a tool set (same scheme as the Gemini and
|
|
148
|
+
* ElevenLabs realtime drivers) — used by {@link AssemblyAIRealtimeSession.RegisterTools}
|
|
149
|
+
* to no-op identical re-registrations.
|
|
150
|
+
*/
|
|
151
|
+
static ToolSetFingerprint(tools: RealtimeToolDefinition[]): string;
|
|
152
|
+
/**
|
|
153
|
+
* REST seam: mints a ONE-TIME client token for a browser-direct session
|
|
154
|
+
* (`GET /v1/token?expires_in_seconds=…` with `Authorization: Bearer <api key>`).
|
|
155
|
+
*/
|
|
156
|
+
protected mintClientToken(expiresInSeconds: number): Promise<string>;
|
|
157
|
+
/**
|
|
158
|
+
* Transport seam for the server-bridged agent websocket. Production speaks the raw Voice
|
|
159
|
+
* Agent protocol over the platform-global `WebSocket` (browsers / Node 22+). Resolves
|
|
160
|
+
* once the socket is OPEN; unit tests override this to return an in-memory fake.
|
|
161
|
+
*/
|
|
162
|
+
protected connectAgentSocket(args: AssemblyAIConnectArgs): Promise<AssemblyAIRealtimeSocket>;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Concrete {@link IRealtimeSession} backed by a raw AssemblyAI Voice Agent websocket.
|
|
166
|
+
*
|
|
167
|
+
* Owns the inbound translation (flat snake_case wire events → Core events) and the outbound
|
|
168
|
+
* translation (Core calls → wire frames). Created by {@link AssemblyAIRealtime.StartSession};
|
|
169
|
+
* never instantiated directly by consumers.
|
|
170
|
+
*
|
|
171
|
+
* Provider-behavior notes (the contract deltas a consumer should know):
|
|
172
|
+
* - **User transcripts have DELTAS, agent transcripts are FINAL-only.** The socket emits
|
|
173
|
+
* `transcript.user.delta` fragments followed by a final `transcript.user`; agent turns
|
|
174
|
+
* arrive as one final `transcript.agent` (carrying `interrupted: true` after a barge-in,
|
|
175
|
+
* in which case the text reflects the truncated turn — already the authoritative record,
|
|
176
|
+
* no ElevenLabs-style correction event follows).
|
|
177
|
+
* - **Interruption signal is `reply.done` with `status: 'interrupted'`** — the provider owns
|
|
178
|
+
* turn detection AND tracks its own audio emission, so this is the authoritative "user
|
|
179
|
+
* speech cut off active model output" signal (raw `input.speech.started` fires on EVERY
|
|
180
|
+
* user utterance and is NOT an interruption per the base contract; the server bridge also
|
|
181
|
+
* cannot see host-side playback, so it defers to the provider's verdict).
|
|
182
|
+
* - **No usage events** — see the driver-level note.
|
|
183
|
+
* - **{@link RequestSpokenUpdate} is NATIVE** (`reply.create` carries per-response
|
|
184
|
+
* instructions — no ElevenLabs-style user-turn emulation, no fidelity caveat), queued
|
|
185
|
+
* behind any in-flight response per the contract's collision rule.
|
|
186
|
+
* - **{@link SendContextNote} is EMULATED via the mutable `system_prompt`**: the protocol
|
|
187
|
+
* has no non-interrupting context channel, but `session.update` may rewrite the prompt
|
|
188
|
+
* mid-session without triggering generation — notes are appended under a "Background
|
|
189
|
+
* updates" heading and the full prompt is re-sent. Never interrupts; the model sees the
|
|
190
|
+
* notes the next time it speaks.
|
|
191
|
+
* - **{@link RegisterTools} re-declares NATIVELY**: `tools` is a mutable `session.update`
|
|
192
|
+
* field, so (unlike ElevenLabs) a genuinely different post-start set is applied to the
|
|
193
|
+
* live session; an identical set is a silent no-op per the idempotency rule.
|
|
194
|
+
*/
|
|
195
|
+
export declare class AssemblyAIRealtimeSession implements IRealtimeSession {
|
|
196
|
+
private socket;
|
|
197
|
+
private outputHandler;
|
|
198
|
+
private transcriptHandler;
|
|
199
|
+
private toolCallHandler;
|
|
200
|
+
private interruptionHandler;
|
|
201
|
+
private errorHandler;
|
|
202
|
+
/** True once Close() ran — an expected close must not surface as a fatal error. */
|
|
203
|
+
private closedByConsumer;
|
|
204
|
+
/** Resolves when `session.ready` arrives; rejects on transport death. */
|
|
205
|
+
private readyPromise;
|
|
206
|
+
private resolveReady;
|
|
207
|
+
private rejectReady;
|
|
208
|
+
private readyReceived;
|
|
209
|
+
/** The wire-shaped `session` object sent in the initial `session.update`. */
|
|
210
|
+
private sessionObject;
|
|
211
|
+
/** The base system prompt (with InitialContext folded in) — context notes append to it. */
|
|
212
|
+
private basePrompt;
|
|
213
|
+
/** Accumulated {@link SendContextNote} texts, re-sent with the full prompt each time. */
|
|
214
|
+
private contextNotes;
|
|
215
|
+
/**
|
|
216
|
+
* Whether an agent reply is currently in flight (`reply.started` / first `reply.audio`
|
|
217
|
+
* sets it; `reply.done` clears it; a `tool.call` clears it WITHOUT draining — deadlock
|
|
218
|
+
* guard). Consumed by {@link enqueueOrRun} so the native {@link RequestSpokenUpdate}
|
|
219
|
+
* never triggers a `reply.create` into — and thereby collides with — an active reply.
|
|
220
|
+
*/
|
|
221
|
+
private responseActive;
|
|
222
|
+
/** Sends deferred while a reply is in flight; drained in order at the next boundary. */
|
|
223
|
+
private queuedSends;
|
|
224
|
+
/**
|
|
225
|
+
* Fingerprint of the tool set currently declared on the session;
|
|
226
|
+
* {@link RegisterTools} compares against it to no-op identical re-registrations.
|
|
227
|
+
*/
|
|
228
|
+
private currentToolsFingerprint;
|
|
229
|
+
constructor(sessionObject: JSONObject);
|
|
230
|
+
/** Binds the underlying socket. Called by the driver once the websocket is open. */
|
|
231
|
+
AttachSocket(socket: AssemblyAIRealtimeSocket): void;
|
|
232
|
+
/**
|
|
233
|
+
* Sends the initial `session.update` frame carrying the full server-authored session
|
|
234
|
+
* config (prompt, tools, voice, turn detection). Always the FIRST client frame — the
|
|
235
|
+
* provider drops audio sent before the session is configured and ready.
|
|
236
|
+
*/
|
|
237
|
+
SendSessionUpdate(): void;
|
|
238
|
+
/**
|
|
239
|
+
* Resolves once the server's `session.ready` arrives (the provider's confirmation that
|
|
240
|
+
* the session config is applied); rejects if the transport dies first. Awaited by
|
|
241
|
+
* {@link AssemblyAIRealtime.StartSession} so the session is never handed to a consumer
|
|
242
|
+
* before it is actually configured.
|
|
243
|
+
*/
|
|
244
|
+
WaitForReady(): Promise<void>;
|
|
245
|
+
/** @inheritdoc — streams one PCM16 (24 kHz mono) frame as a base64 `input.audio` chunk. */
|
|
246
|
+
SendInput(chunk: ArrayBuffer): void;
|
|
247
|
+
/**
|
|
248
|
+
* @inheritdoc
|
|
249
|
+
*
|
|
250
|
+
* `tools` is a MUTABLE `session.update` field on this provider, so re-declaration is
|
|
251
|
+
* native: an identical set (order-insensitively) is a silent no-op per the contract's
|
|
252
|
+
* idempotency rule; a different set is applied to the live session immediately.
|
|
253
|
+
*/
|
|
254
|
+
RegisterTools(tools: RealtimeToolDefinition[]): Promise<void>;
|
|
255
|
+
/** Records the tool set declared at connect time. Called by {@link AssemblyAIRealtime.StartSession}. */
|
|
256
|
+
SetConnectTimeTools(tools: RealtimeToolDefinition[]): void;
|
|
257
|
+
/** @inheritdoc */
|
|
258
|
+
OnOutput(handler: (chunk: ArrayBuffer) => void): void;
|
|
259
|
+
/** @inheritdoc */
|
|
260
|
+
OnTranscript(handler: (t: RealtimeTranscript) => void): void;
|
|
261
|
+
/** @inheritdoc */
|
|
262
|
+
OnToolCall(handler: (call: RealtimeToolCall) => void): void;
|
|
263
|
+
/** @inheritdoc */
|
|
264
|
+
OnInterruption(handler: () => void): void;
|
|
265
|
+
/**
|
|
266
|
+
* @inheritdoc
|
|
267
|
+
*
|
|
268
|
+
* Never fires: the Voice Agent websocket reports no token usage — billing is a flat
|
|
269
|
+
* hourly session rate (see the driver-level note).
|
|
270
|
+
*/
|
|
271
|
+
OnUsage(_handler: (u: RealtimeUsage) => void): void;
|
|
272
|
+
/** @inheritdoc */
|
|
273
|
+
OnError(handler: (error: RealtimeSessionError) => void): void;
|
|
274
|
+
/**
|
|
275
|
+
* Surfaces a websocket-level failure as a FATAL session error — the transport is gone,
|
|
276
|
+
* so the consumer should finalize cleanly instead of idling (driver obligation #6).
|
|
277
|
+
*/
|
|
278
|
+
HandleTransportError(message: string): void;
|
|
279
|
+
/**
|
|
280
|
+
* Surfaces an UNEXPECTED socket close as a fatal error (expected closes — the consumer
|
|
281
|
+
* called {@link Close} — are silent). The provider hard-closes at token expiry and when
|
|
282
|
+
* it ends the session itself, so this is also how credential / session death reaches the
|
|
283
|
+
* consumer.
|
|
284
|
+
*/
|
|
285
|
+
HandleTransportClose(code?: number, reason?: string): void;
|
|
286
|
+
/**
|
|
287
|
+
* @inheritdoc
|
|
288
|
+
*
|
|
289
|
+
* Sends `tool.result` correlated by `call_id`. Sent IMMEDIATELY (never queued): the
|
|
290
|
+
* provider's own guidance is "send it the moment your tool returns — no buffering, no
|
|
291
|
+
* waiting on reply.done"; it owns the continuation and speaks the result as soon as the
|
|
292
|
+
* floor allows. The wire `result` slot expects a JSON-STRING (not an object), which is
|
|
293
|
+
* exactly the contract's `output` shape — it passes through verbatim. The reply flag is
|
|
294
|
+
* set eagerly so a queued narration can't slip in before the spoken result.
|
|
295
|
+
*/
|
|
296
|
+
SendToolResult(callID: string, output: string): Promise<void>;
|
|
297
|
+
/**
|
|
298
|
+
* @inheritdoc
|
|
299
|
+
*
|
|
300
|
+
* EMULATED via the mutable `system_prompt` (the protocol has no purpose-built context
|
|
301
|
+
* channel): the note is appended under a "Background updates" heading and the FULL
|
|
302
|
+
* prompt is re-sent via `session.update` — a config write, so it never triggers or
|
|
303
|
+
* disturbs generation and is sent immediately even mid-reply. The model has the notes
|
|
304
|
+
* available the next time it speaks.
|
|
305
|
+
*/
|
|
306
|
+
SendContextNote(text: string): void;
|
|
307
|
+
/**
|
|
308
|
+
* @inheritdoc
|
|
309
|
+
*
|
|
310
|
+
* NATIVE on this provider: `reply.create` carries one-off per-response instructions.
|
|
311
|
+
* **Collision behavior: queue.** A `reply.create` sent mid-reply would collide with the
|
|
312
|
+
* in-flight generation, so the send is deferred until the active reply completes.
|
|
313
|
+
*/
|
|
314
|
+
RequestSpokenUpdate(instructions: string): void;
|
|
315
|
+
/**
|
|
316
|
+
* @inheritdoc
|
|
317
|
+
*
|
|
318
|
+
* Sends `session.end` BEFORE closing the socket — without it the provider holds the
|
|
319
|
+
* session (billable) for a 30-second resume window.
|
|
320
|
+
*/
|
|
321
|
+
Close(): Promise<void>;
|
|
322
|
+
/**
|
|
323
|
+
* Entry point for an inbound websocket frame. Multiplexes on `type` to focused
|
|
324
|
+
* per-concern handlers so each translation unit stays small and testable.
|
|
325
|
+
*/
|
|
326
|
+
HandleServerEvent(event: AssemblyAIServerEvent): void;
|
|
327
|
+
/** Decodes one base64 reply-audio frame and forwards it as a raw `ArrayBuffer`. */
|
|
328
|
+
private handleReplyAudio;
|
|
329
|
+
/**
|
|
330
|
+
* Reply boundary. `status: 'interrupted'` is the provider's authoritative true-barge-in
|
|
331
|
+
* verdict (user speech cut off active output — see the class note) and is surfaced via
|
|
332
|
+
* {@link OnInterruption} BEFORE the boundary releases the floor and drains queued sends.
|
|
333
|
+
*/
|
|
334
|
+
private handleReplyDone;
|
|
335
|
+
/**
|
|
336
|
+
* Surfaces a `tool.call` to the consumer. The model has yielded the floor pending the
|
|
337
|
+
* result, so the busy flag is cleared (deadlock guard — driver obligation #2) WITHOUT
|
|
338
|
+
* draining the queue (a queued narration must not trigger a reply between the tool call
|
|
339
|
+
* and its result; it drains at the next real reply boundary). The provider emits
|
|
340
|
+
* `arguments` ALREADY PARSED, so it is re-stringified to honor the Core contract's
|
|
341
|
+
* JSON-string `Arguments` shape.
|
|
342
|
+
*/
|
|
343
|
+
private handleToolCall;
|
|
344
|
+
/**
|
|
345
|
+
* The provider ended the session itself. After a consumer {@link Close} (which sends
|
|
346
|
+
* `session.end`) this is the expected acknowledgment and stays silent; otherwise it is
|
|
347
|
+
* surfaced as FATAL so the consumer finalizes instead of idling on a dying socket.
|
|
348
|
+
*/
|
|
349
|
+
private handleSessionEnded;
|
|
350
|
+
/** Reply boundary: releases the busy flag and drains queued sends in order. */
|
|
351
|
+
private completeResponse;
|
|
352
|
+
/** Emits a transcript event (user turns have deltas; agent turns are final-only). */
|
|
353
|
+
private emitTranscript;
|
|
354
|
+
/** Runs a send immediately when idle; otherwise queues it for the next reply boundary. */
|
|
355
|
+
private enqueueOrRun;
|
|
356
|
+
/** The base prompt plus every accumulated context note under a "Background updates" heading. */
|
|
357
|
+
private composePromptWithNotes;
|
|
358
|
+
/** JSON-serializes and sends one client frame (throws if the socket was never attached). */
|
|
359
|
+
private sendFrame;
|
|
360
|
+
/** Rejects a still-pending ready wait (transport death / consumer close during startup). */
|
|
361
|
+
private failReadyWait;
|
|
362
|
+
/** Drops all registered handlers so a closed session can't fire stale callbacks. */
|
|
363
|
+
private clearHandlers;
|
|
364
|
+
/** Decodes a base64 audio payload into a freshly-allocated `ArrayBuffer`. */
|
|
365
|
+
private static Base64ToArrayBuffer;
|
|
366
|
+
}
|
|
367
|
+
//# sourceMappingURL=assemblyAIRealtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assemblyAIRealtime.d.ts","sourceRoot":"","sources":["../src/assemblyAIRealtime.ts"],"names":[],"mappings":"AAGA,OAAO,EACH,iBAAiB,EACjB,KAAK,2BAA2B,EAChC,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC3B,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,UAAU,EAClB,MAAM,oBAAoB,CAAC;AAG5B,6FAA6F;AAC7F,eAAO,MAAM,uBAAuB,sCAAsC,CAAC;AAE3E,gGAAgG;AAChG,eAAO,MAAM,oBAAoB,2CAA2C,CAAC;AAE7E;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAEhD;;;GAGG;AACH,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAMhD,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0FAA0F;IAC1F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mFAAmF;IACnF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0FAA0F;IAC1F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,wDAAwD;IACxD,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAID;;;;;GAKG;AACH,MAAM,WAAW,wBAAwB;IACrC,8CAA8C;IAC9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IAClC,iGAAiG;IACjG,GAAG,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,SAAS,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAClD,4EAA4E;IAC5E,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,yCAAyC;IACzC,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;CACrD;AAsBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBACa,kBAAmB,SAAQ,iBAAiB;IACrD;;;;;;;OAOG;IACU,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAenF;;;OAGG;IACH,IAAoB,oBAAoB,IAAI,OAAO,CAElD;IAED;;;;;;OAMG;IACmB,mBAAmB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;IAgB9G;;;;;;;;;OASG;WACW,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,GAAG,UAAU;IA4B3E,uFAAuF;WACzE,mBAAmB,CAAC,YAAY,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM;IAKxF,8EAA8E;WAChE,iBAAiB,CAAC,IAAI,EAAE,sBAAsB,GAAG,UAAU;IAWzE;;;;OAIG;WACW,kBAAkB,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,MAAM;IAUzE;;;OAGG;cACa,eAAe,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmB1E;;;;OAIG;cACa,kBAAkB,CAAC,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,wBAAwB,CAAC;CAiCrG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,yBAA0B,YAAW,gBAAgB;IAC9D,OAAO,CAAC,MAAM,CAAyC;IAEvD,OAAO,CAAC,aAAa,CAA+C;IACpE,OAAO,CAAC,iBAAiB,CAAkD;IAC3E,OAAO,CAAC,eAAe,CAAmD;IAC1E,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,YAAY,CAAwD;IAC5E,mFAAmF;IACnF,OAAO,CAAC,gBAAgB,CAAS;IAEjC,yEAAyE;IACzE,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,aAAa,CAAS;IAE9B,6EAA6E;IAC7E,OAAO,CAAC,aAAa,CAAa;IAClC,2FAA2F;IAC3F,OAAO,CAAC,UAAU,CAAS;IAC3B,yFAAyF;IACzF,OAAO,CAAC,YAAY,CAAgB;IAEpC;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAS;IAE/B,wFAAwF;IACxF,OAAO,CAAC,WAAW,CAAyB;IAE5C;;;OAGG;IACH,OAAO,CAAC,uBAAuB,CAA6C;gBAEhE,aAAa,EAAE,UAAU;IAarC,oFAAoF;IAC7E,YAAY,CAAC,MAAM,EAAE,wBAAwB,GAAG,IAAI;IAI3D;;;;OAIG;IACI,iBAAiB,IAAI,IAAI;IAIhC;;;;;OAKG;IACI,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpC,2FAA2F;IACpF,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAI1C;;;;;;OAMG;IACU,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1E,wGAAwG;IACjG,mBAAmB,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,IAAI;IAIjE,kBAAkB;IACX,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAI5D,kBAAkB;IACX,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI;IAInE,kBAAkB;IACX,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAIlE,kBAAkB;IACX,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD;;;;;OAKG;IACI,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAI1D,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI;IAIpE;;;OAGG;IACI,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAKlD;;;;;OAKG;IACI,oBAAoB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAUjE;;;;;;;;;OASG;IACU,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1E;;;;;;;;OAQG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAK1C;;;;;;OAMG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAOtD;;;;;OAKG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;OAGG;IACI,iBAAiB,CAAC,KAAK,EAAE,qBAAqB,GAAG,IAAI;IA+C5D,mFAAmF;IACnF,OAAO,CAAC,gBAAgB;IAQxB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAStB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAS1B,+EAA+E;IAC/E,OAAO,CAAC,gBAAgB;IAQxB,qFAAqF;IACrF,OAAO,CAAC,cAAc;IAOtB,0FAA0F;IAC1F,OAAO,CAAC,YAAY;IAQpB,gGAAgG;IAChG,OAAO,CAAC,sBAAsB;IAO9B,4FAA4F;IAC5F,OAAO,CAAC,SAAS;IAOjB,4FAA4F;IAC5F,OAAO,CAAC,aAAa;IAUrB,oFAAoF;IACpF,OAAO,CAAC,aAAa;IASrB,6EAA6E;IAC7E,OAAO,CAAC,MAAM,CAAC,mBAAmB;CAMrC"}
|
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
// AssemblyAI Voice Agent API — realtime (voice) driver.
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var AssemblyAIRealtime_1;
|
|
9
|
+
// MemberJunction AI core contract
|
|
10
|
+
import { BaseRealtimeModel, } from '@memberjunction/ai';
|
|
11
|
+
import { RegisterClass } from '@memberjunction/global';
|
|
12
|
+
/** The single Voice Agent websocket endpoint (auth rides as a `?token=` query parameter). */
|
|
13
|
+
export const ASSEMBLYAI_AGENT_WS_URL = 'wss://agents.assemblyai.com/v1/ws';
|
|
14
|
+
/** REST endpoint that mints one-time, short-lived client tokens for browser-direct sessions. */
|
|
15
|
+
export const ASSEMBLYAI_TOKEN_URL = 'https://agents.assemblyai.com/v1/token';
|
|
16
|
+
/**
|
|
17
|
+
* Lifetime requested for the one-time client token minted by {@link AssemblyAIRealtime.CreateClientSession}.
|
|
18
|
+
* The token is single-use: it authenticates ONE websocket open within this window; an
|
|
19
|
+
* already-open session continues past it.
|
|
20
|
+
*/
|
|
21
|
+
export const ASSEMBLYAI_TOKEN_TTL_SECONDS = 300;
|
|
22
|
+
/**
|
|
23
|
+
* The Voice Agent wire format: 16-bit signed little-endian PCM, mono, 24 kHz, base64-encoded,
|
|
24
|
+
* in BOTH directions (`input.audio` up, `reply.audio` down).
|
|
25
|
+
*/
|
|
26
|
+
export const ASSEMBLYAI_PCM_SAMPLE_RATE = 24000;
|
|
27
|
+
/**
|
|
28
|
+
* Real-time, full-duplex driver for the **AssemblyAI Voice Agent API** (launched April 2026),
|
|
29
|
+
* implementing the Core {@link BaseRealtimeModel} primitive. Registers as `AssemblyAIRealtime`
|
|
30
|
+
* and is resolved for `MJ: AI Models` typed `Realtime` (API-key env alias:
|
|
31
|
+
* `AI_VENDOR_API_KEY__AssemblyAIRealtime`).
|
|
32
|
+
*
|
|
33
|
+
* **What the provider is:** a single-websocket speech-to-speech stack — Universal-3 Pro
|
|
34
|
+
* streaming ASR, server-side turn detection / barge-in, LLM reasoning, JSON-Schema tool
|
|
35
|
+
* calling, and conversational TTS — billed flat per session-hour. Unlike ElevenLabs there is
|
|
36
|
+
* no server-side agent object to manage: the ENTIRE session config (prompt, tools, voice,
|
|
37
|
+
* turn detection) is supplied per-session via the first `session.update` frame, so MJ's
|
|
38
|
+
* prompt/tool authority is native here, with no managed-agent ensure flow.
|
|
39
|
+
*
|
|
40
|
+
* **Model resolution:** the Voice Agent API exposes ONE endpoint and no model selection;
|
|
41
|
+
* `params.Model` is carried through to {@link ClientRealtimeSessionConfig.Model} for
|
|
42
|
+
* bookkeeping but plays no wire role.
|
|
43
|
+
*
|
|
44
|
+
* **Topologies:**
|
|
45
|
+
* - Server-bridged ({@link StartSession}): the driver opens the websocket itself
|
|
46
|
+
* (`?token=<api key>`), sends the session config, and resolves only after `session.ready`.
|
|
47
|
+
* - Client-direct ({@link CreateClientSession}): the driver mints a ONE-TIME, short-lived
|
|
48
|
+
* client token (`GET /v1/token`) — the browser opens its own socket with it (no API key
|
|
49
|
+
* ever leaves the server) and applies the server-built session object from the
|
|
50
|
+
* `SessionConfig` pact.
|
|
51
|
+
*
|
|
52
|
+
* **No usage events:** billing is a flat hourly session rate; the websocket reports no token
|
|
53
|
+
* usage, so sessions never emit {@link IRealtimeSession.OnUsage}. (The terminal
|
|
54
|
+
* `session.ended` frame carries duration seconds, but by then the consumer has typically
|
|
55
|
+
* closed — duration accounting lives in the provider dashboard.)
|
|
56
|
+
*/
|
|
57
|
+
let AssemblyAIRealtime = AssemblyAIRealtime_1 = class AssemblyAIRealtime extends BaseRealtimeModel {
|
|
58
|
+
/**
|
|
59
|
+
* Opens a server-bridged session: connects the agent websocket authenticated with the
|
|
60
|
+
* API key, sends the full session config as the FIRST frame (`session.update` — prompt,
|
|
61
|
+
* tools, voice, turn detection), and resolves only once the server's `session.ready`
|
|
62
|
+
* confirms the config is applied (driver obligation #7 — "ready only after the config is
|
|
63
|
+
* applied"). Mic frames sent before that would be dropped by the provider, and
|
|
64
|
+
* `StartSession` not resolving until ready makes that unrepresentable for consumers.
|
|
65
|
+
*/
|
|
66
|
+
async StartSession(params) {
|
|
67
|
+
const session = new AssemblyAIRealtimeSession(AssemblyAIRealtime_1.BuildSessionObject(params));
|
|
68
|
+
session.SetConnectTimeTools(params.Tools ?? []);
|
|
69
|
+
const socket = await this.connectAgentSocket({
|
|
70
|
+
Url: `${ASSEMBLYAI_AGENT_WS_URL}?token=${encodeURIComponent(this.apiKey)}`,
|
|
71
|
+
OnMessage: (event) => session.HandleServerEvent(event),
|
|
72
|
+
OnError: (message) => session.HandleTransportError(message),
|
|
73
|
+
OnClose: (code, reason) => session.HandleTransportClose(code, reason),
|
|
74
|
+
});
|
|
75
|
+
session.AttachSocket(socket);
|
|
76
|
+
session.SendSessionUpdate();
|
|
77
|
+
await session.WaitForReady();
|
|
78
|
+
return session;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* AssemblyAI supports the client-direct topology natively: a one-time temp token minted
|
|
82
|
+
* server-side authenticates the browser's own websocket.
|
|
83
|
+
*/
|
|
84
|
+
get SupportsClientDirect() {
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Mints the client-direct config: a ONE-TIME client token (connect window
|
|
89
|
+
* {@link ASSEMBLYAI_TOKEN_TTL_SECONDS}s) plus the **private pact** `SessionConfig` the
|
|
90
|
+
* same-keyed `'assemblyai'` client driver consumes: `{ session, config }`, where
|
|
91
|
+
* `session` is the wire-shaped `session.update` payload (server-authored system prompt,
|
|
92
|
+
* tools, voice, turn detection) and `config` passes `params.Config` through opaquely.
|
|
93
|
+
*/
|
|
94
|
+
async CreateClientSession(params) {
|
|
95
|
+
const token = await this.mintClientToken(ASSEMBLYAI_TOKEN_TTL_SECONDS);
|
|
96
|
+
return {
|
|
97
|
+
Provider: 'assemblyai',
|
|
98
|
+
Model: params.Model,
|
|
99
|
+
EphemeralToken: token,
|
|
100
|
+
ExpiresAt: new Date(Date.now() + ASSEMBLYAI_TOKEN_TTL_SECONDS * 1000).toISOString(),
|
|
101
|
+
SessionConfig: {
|
|
102
|
+
session: AssemblyAIRealtime_1.BuildSessionObject(params),
|
|
103
|
+
config: params.Config ?? {},
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
// ── Session-object construction (the wire-shaped `session.update` payload) ──
|
|
108
|
+
/**
|
|
109
|
+
* Builds the `session` object for the `session.update` frame from the Core params:
|
|
110
|
+
* - `system_prompt`: the session system prompt, with `InitialContext` folded in under a
|
|
111
|
+
* "Prior context" heading — the protocol has no history-seeding channel, so prior
|
|
112
|
+
* conversation context rides the prompt.
|
|
113
|
+
* - `tools`: the Core tool set mapped to the provider's `{ type: 'function', … }` schema.
|
|
114
|
+
* - Recognized `params.Config` keys pass through to their wire slots: `voice` →
|
|
115
|
+
* `output.voice`, `greeting` (spoken on connect; omitted = no auto-greeting),
|
|
116
|
+
* `turn_detection` → `input.turn_detection`, `keyterms` → `input.keyterms`.
|
|
117
|
+
*/
|
|
118
|
+
static BuildSessionObject(params) {
|
|
119
|
+
const session = {
|
|
120
|
+
system_prompt: AssemblyAIRealtime_1.ComposeSystemPrompt(params.SystemPrompt, params.InitialContext),
|
|
121
|
+
};
|
|
122
|
+
const tools = params.Tools ?? [];
|
|
123
|
+
if (tools.length > 0) {
|
|
124
|
+
session['tools'] = tools.map((tool) => AssemblyAIRealtime_1.MapToolToFunction(tool));
|
|
125
|
+
}
|
|
126
|
+
const config = params.Config ?? {};
|
|
127
|
+
if (typeof config['greeting'] === 'string') {
|
|
128
|
+
session['greeting'] = config['greeting'];
|
|
129
|
+
}
|
|
130
|
+
if (typeof config['voice'] === 'string') {
|
|
131
|
+
session['output'] = { voice: config['voice'] };
|
|
132
|
+
}
|
|
133
|
+
const input = {};
|
|
134
|
+
if (config['turn_detection'] !== null && typeof config['turn_detection'] === 'object' && !Array.isArray(config['turn_detection'])) {
|
|
135
|
+
input['turn_detection'] = config['turn_detection'];
|
|
136
|
+
}
|
|
137
|
+
if (Array.isArray(config['keyterms'])) {
|
|
138
|
+
input['keyterms'] = config['keyterms'];
|
|
139
|
+
}
|
|
140
|
+
if (Object.keys(input).length > 0) {
|
|
141
|
+
session['input'] = input;
|
|
142
|
+
}
|
|
143
|
+
return session;
|
|
144
|
+
}
|
|
145
|
+
/** Folds optional prior context into the system prompt (no history channel exists). */
|
|
146
|
+
static ComposeSystemPrompt(systemPrompt, initialContext) {
|
|
147
|
+
const context = initialContext?.trim();
|
|
148
|
+
return context ? `${systemPrompt}\n\n## Prior context\n${context}` : systemPrompt;
|
|
149
|
+
}
|
|
150
|
+
/** Maps a Core tool definition up to a Voice Agent `function` tool schema. */
|
|
151
|
+
static MapToolToFunction(tool) {
|
|
152
|
+
return {
|
|
153
|
+
type: 'function',
|
|
154
|
+
name: tool.Name,
|
|
155
|
+
description: tool.Description,
|
|
156
|
+
// The Core ParametersSchema is a JSON-schema object — the same shape the Voice
|
|
157
|
+
// Agent's `tools[].parameters` slot accepts.
|
|
158
|
+
parameters: tool.ParametersSchema,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Canonical, order-insensitive fingerprint of a tool set (same scheme as the Gemini and
|
|
163
|
+
* ElevenLabs realtime drivers) — used by {@link AssemblyAIRealtimeSession.RegisterTools}
|
|
164
|
+
* to no-op identical re-registrations.
|
|
165
|
+
*/
|
|
166
|
+
static ToolSetFingerprint(tools) {
|
|
167
|
+
return JSON.stringify([...tools]
|
|
168
|
+
.sort((a, b) => a.Name.localeCompare(b.Name))
|
|
169
|
+
.map((t) => ({ Name: t.Name, Description: t.Description, ParametersSchema: t.ParametersSchema })));
|
|
170
|
+
}
|
|
171
|
+
// ── Overridable REST / transport seams (tests inject fakes — no network) ──
|
|
172
|
+
/**
|
|
173
|
+
* REST seam: mints a ONE-TIME client token for a browser-direct session
|
|
174
|
+
* (`GET /v1/token?expires_in_seconds=…` with `Authorization: Bearer <api key>`).
|
|
175
|
+
*/
|
|
176
|
+
async mintClientToken(expiresInSeconds) {
|
|
177
|
+
const fetchFn = globalThis.fetch;
|
|
178
|
+
if (!fetchFn) {
|
|
179
|
+
throw new Error('AssemblyAIRealtime.CreateClientSession requires a global fetch (Node 18+ or a browser runtime).');
|
|
180
|
+
}
|
|
181
|
+
const response = await fetchFn(`${ASSEMBLYAI_TOKEN_URL}?expires_in_seconds=${expiresInSeconds}`, {
|
|
182
|
+
method: 'GET',
|
|
183
|
+
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
184
|
+
});
|
|
185
|
+
if (!response.ok) {
|
|
186
|
+
throw new Error(`AssemblyAI token mint failed with HTTP ${response.status}`);
|
|
187
|
+
}
|
|
188
|
+
const body = (await response.json());
|
|
189
|
+
if (!body.token) {
|
|
190
|
+
throw new Error('AssemblyAI token mint returned no token');
|
|
191
|
+
}
|
|
192
|
+
return body.token;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Transport seam for the server-bridged agent websocket. Production speaks the raw Voice
|
|
196
|
+
* Agent protocol over the platform-global `WebSocket` (browsers / Node 22+). Resolves
|
|
197
|
+
* once the socket is OPEN; unit tests override this to return an in-memory fake.
|
|
198
|
+
*/
|
|
199
|
+
async connectAgentSocket(args) {
|
|
200
|
+
const WS = globalThis.WebSocket;
|
|
201
|
+
if (!WS) {
|
|
202
|
+
throw new Error('AssemblyAIRealtime.StartSession requires a global WebSocket (Node 22+ or a browser runtime).');
|
|
203
|
+
}
|
|
204
|
+
return new Promise((resolve, reject) => {
|
|
205
|
+
const ws = new WS(args.Url);
|
|
206
|
+
let opened = false;
|
|
207
|
+
ws.onopen = () => {
|
|
208
|
+
opened = true;
|
|
209
|
+
resolve({ send: (data) => ws.send(data), close: () => ws.close() });
|
|
210
|
+
};
|
|
211
|
+
ws.onmessage = (event) => {
|
|
212
|
+
try {
|
|
213
|
+
args.OnMessage(JSON.parse(String(event.data)));
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
/* non-JSON frame — ignore */
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
ws.onerror = () => {
|
|
220
|
+
args.OnError('AssemblyAI agent websocket error');
|
|
221
|
+
if (!opened) {
|
|
222
|
+
reject(new Error('AssemblyAI agent websocket failed to open'));
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
ws.onclose = (event) => {
|
|
226
|
+
args.OnClose(event.code, event.reason);
|
|
227
|
+
if (!opened) {
|
|
228
|
+
reject(new Error('AssemblyAI agent websocket closed before opening'));
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
};
|
|
234
|
+
AssemblyAIRealtime = AssemblyAIRealtime_1 = __decorate([
|
|
235
|
+
RegisterClass(BaseRealtimeModel, 'AssemblyAIRealtime')
|
|
236
|
+
], AssemblyAIRealtime);
|
|
237
|
+
export { AssemblyAIRealtime };
|
|
238
|
+
/**
|
|
239
|
+
* Concrete {@link IRealtimeSession} backed by a raw AssemblyAI Voice Agent websocket.
|
|
240
|
+
*
|
|
241
|
+
* Owns the inbound translation (flat snake_case wire events → Core events) and the outbound
|
|
242
|
+
* translation (Core calls → wire frames). Created by {@link AssemblyAIRealtime.StartSession};
|
|
243
|
+
* never instantiated directly by consumers.
|
|
244
|
+
*
|
|
245
|
+
* Provider-behavior notes (the contract deltas a consumer should know):
|
|
246
|
+
* - **User transcripts have DELTAS, agent transcripts are FINAL-only.** The socket emits
|
|
247
|
+
* `transcript.user.delta` fragments followed by a final `transcript.user`; agent turns
|
|
248
|
+
* arrive as one final `transcript.agent` (carrying `interrupted: true` after a barge-in,
|
|
249
|
+
* in which case the text reflects the truncated turn — already the authoritative record,
|
|
250
|
+
* no ElevenLabs-style correction event follows).
|
|
251
|
+
* - **Interruption signal is `reply.done` with `status: 'interrupted'`** — the provider owns
|
|
252
|
+
* turn detection AND tracks its own audio emission, so this is the authoritative "user
|
|
253
|
+
* speech cut off active model output" signal (raw `input.speech.started` fires on EVERY
|
|
254
|
+
* user utterance and is NOT an interruption per the base contract; the server bridge also
|
|
255
|
+
* cannot see host-side playback, so it defers to the provider's verdict).
|
|
256
|
+
* - **No usage events** — see the driver-level note.
|
|
257
|
+
* - **{@link RequestSpokenUpdate} is NATIVE** (`reply.create` carries per-response
|
|
258
|
+
* instructions — no ElevenLabs-style user-turn emulation, no fidelity caveat), queued
|
|
259
|
+
* behind any in-flight response per the contract's collision rule.
|
|
260
|
+
* - **{@link SendContextNote} is EMULATED via the mutable `system_prompt`**: the protocol
|
|
261
|
+
* has no non-interrupting context channel, but `session.update` may rewrite the prompt
|
|
262
|
+
* mid-session without triggering generation — notes are appended under a "Background
|
|
263
|
+
* updates" heading and the full prompt is re-sent. Never interrupts; the model sees the
|
|
264
|
+
* notes the next time it speaks.
|
|
265
|
+
* - **{@link RegisterTools} re-declares NATIVELY**: `tools` is a mutable `session.update`
|
|
266
|
+
* field, so (unlike ElevenLabs) a genuinely different post-start set is applied to the
|
|
267
|
+
* live session; an identical set is a silent no-op per the idempotency rule.
|
|
268
|
+
*/
|
|
269
|
+
export class AssemblyAIRealtimeSession {
|
|
270
|
+
constructor(sessionObject) {
|
|
271
|
+
this.socket = null;
|
|
272
|
+
this.outputHandler = null;
|
|
273
|
+
this.transcriptHandler = null;
|
|
274
|
+
this.toolCallHandler = null;
|
|
275
|
+
this.interruptionHandler = null;
|
|
276
|
+
this.errorHandler = null;
|
|
277
|
+
/** True once Close() ran — an expected close must not surface as a fatal error. */
|
|
278
|
+
this.closedByConsumer = false;
|
|
279
|
+
this.resolveReady = null;
|
|
280
|
+
this.rejectReady = null;
|
|
281
|
+
this.readyReceived = false;
|
|
282
|
+
/** Accumulated {@link SendContextNote} texts, re-sent with the full prompt each time. */
|
|
283
|
+
this.contextNotes = [];
|
|
284
|
+
/**
|
|
285
|
+
* Whether an agent reply is currently in flight (`reply.started` / first `reply.audio`
|
|
286
|
+
* sets it; `reply.done` clears it; a `tool.call` clears it WITHOUT draining — deadlock
|
|
287
|
+
* guard). Consumed by {@link enqueueOrRun} so the native {@link RequestSpokenUpdate}
|
|
288
|
+
* never triggers a `reply.create` into — and thereby collides with — an active reply.
|
|
289
|
+
*/
|
|
290
|
+
this.responseActive = false;
|
|
291
|
+
/** Sends deferred while a reply is in flight; drained in order at the next boundary. */
|
|
292
|
+
this.queuedSends = [];
|
|
293
|
+
/**
|
|
294
|
+
* Fingerprint of the tool set currently declared on the session;
|
|
295
|
+
* {@link RegisterTools} compares against it to no-op identical re-registrations.
|
|
296
|
+
*/
|
|
297
|
+
this.currentToolsFingerprint = AssemblyAIRealtime.ToolSetFingerprint([]);
|
|
298
|
+
this.sessionObject = sessionObject;
|
|
299
|
+
this.basePrompt = typeof sessionObject['system_prompt'] === 'string' ? sessionObject['system_prompt'] : '';
|
|
300
|
+
this.readyPromise = new Promise((resolve, reject) => {
|
|
301
|
+
this.resolveReady = resolve;
|
|
302
|
+
this.rejectReady = reject;
|
|
303
|
+
});
|
|
304
|
+
// The promise is always consumed by WaitForReady before any rejection can fire
|
|
305
|
+
// (StartSession awaits it immediately), but guard against unhandled-rejection noise
|
|
306
|
+
// if a transport error lands between construction and the await.
|
|
307
|
+
this.readyPromise.catch(() => undefined);
|
|
308
|
+
}
|
|
309
|
+
/** Binds the underlying socket. Called by the driver once the websocket is open. */
|
|
310
|
+
AttachSocket(socket) {
|
|
311
|
+
this.socket = socket;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Sends the initial `session.update` frame carrying the full server-authored session
|
|
315
|
+
* config (prompt, tools, voice, turn detection). Always the FIRST client frame — the
|
|
316
|
+
* provider drops audio sent before the session is configured and ready.
|
|
317
|
+
*/
|
|
318
|
+
SendSessionUpdate() {
|
|
319
|
+
this.sendFrame({ type: 'session.update', session: this.sessionObject });
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Resolves once the server's `session.ready` arrives (the provider's confirmation that
|
|
323
|
+
* the session config is applied); rejects if the transport dies first. Awaited by
|
|
324
|
+
* {@link AssemblyAIRealtime.StartSession} so the session is never handed to a consumer
|
|
325
|
+
* before it is actually configured.
|
|
326
|
+
*/
|
|
327
|
+
WaitForReady() {
|
|
328
|
+
return this.readyPromise;
|
|
329
|
+
}
|
|
330
|
+
/** @inheritdoc — streams one PCM16 (24 kHz mono) frame as a base64 `input.audio` chunk. */
|
|
331
|
+
SendInput(chunk) {
|
|
332
|
+
this.sendFrame({ type: 'input.audio', audio: Buffer.from(new Uint8Array(chunk)).toString('base64') });
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* @inheritdoc
|
|
336
|
+
*
|
|
337
|
+
* `tools` is a MUTABLE `session.update` field on this provider, so re-declaration is
|
|
338
|
+
* native: an identical set (order-insensitively) is a silent no-op per the contract's
|
|
339
|
+
* idempotency rule; a different set is applied to the live session immediately.
|
|
340
|
+
*/
|
|
341
|
+
async RegisterTools(tools) {
|
|
342
|
+
const fingerprint = AssemblyAIRealtime.ToolSetFingerprint(tools);
|
|
343
|
+
if (fingerprint === this.currentToolsFingerprint) {
|
|
344
|
+
return; // identical to the declared set — silent no-op
|
|
345
|
+
}
|
|
346
|
+
this.currentToolsFingerprint = fingerprint;
|
|
347
|
+
this.sendFrame({
|
|
348
|
+
type: 'session.update',
|
|
349
|
+
session: { tools: tools.map((tool) => AssemblyAIRealtime.MapToolToFunction(tool)) },
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
/** Records the tool set declared at connect time. Called by {@link AssemblyAIRealtime.StartSession}. */
|
|
353
|
+
SetConnectTimeTools(tools) {
|
|
354
|
+
this.currentToolsFingerprint = AssemblyAIRealtime.ToolSetFingerprint(tools);
|
|
355
|
+
}
|
|
356
|
+
/** @inheritdoc */
|
|
357
|
+
OnOutput(handler) {
|
|
358
|
+
this.outputHandler = handler;
|
|
359
|
+
}
|
|
360
|
+
/** @inheritdoc */
|
|
361
|
+
OnTranscript(handler) {
|
|
362
|
+
this.transcriptHandler = handler;
|
|
363
|
+
}
|
|
364
|
+
/** @inheritdoc */
|
|
365
|
+
OnToolCall(handler) {
|
|
366
|
+
this.toolCallHandler = handler;
|
|
367
|
+
}
|
|
368
|
+
/** @inheritdoc */
|
|
369
|
+
OnInterruption(handler) {
|
|
370
|
+
this.interruptionHandler = handler;
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* @inheritdoc
|
|
374
|
+
*
|
|
375
|
+
* Never fires: the Voice Agent websocket reports no token usage — billing is a flat
|
|
376
|
+
* hourly session rate (see the driver-level note).
|
|
377
|
+
*/
|
|
378
|
+
OnUsage(_handler) {
|
|
379
|
+
// intentionally unbound — no usage events exist on this provider surface
|
|
380
|
+
}
|
|
381
|
+
/** @inheritdoc */
|
|
382
|
+
OnError(handler) {
|
|
383
|
+
this.errorHandler = handler;
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Surfaces a websocket-level failure as a FATAL session error — the transport is gone,
|
|
387
|
+
* so the consumer should finalize cleanly instead of idling (driver obligation #6).
|
|
388
|
+
*/
|
|
389
|
+
HandleTransportError(message) {
|
|
390
|
+
this.failReadyWait(message);
|
|
391
|
+
this.errorHandler?.({ Message: message, Fatal: true });
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Surfaces an UNEXPECTED socket close as a fatal error (expected closes — the consumer
|
|
395
|
+
* called {@link Close} — are silent). The provider hard-closes at token expiry and when
|
|
396
|
+
* it ends the session itself, so this is also how credential / session death reaches the
|
|
397
|
+
* consumer.
|
|
398
|
+
*/
|
|
399
|
+
HandleTransportClose(code, reason) {
|
|
400
|
+
if (this.closedByConsumer) {
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
const detail = [code != null ? `code ${code}` : null, reason || null].filter(Boolean).join(' — ');
|
|
404
|
+
const message = `AssemblyAI agent session closed unexpectedly${detail ? ` (${detail})` : ''}`;
|
|
405
|
+
this.failReadyWait(message);
|
|
406
|
+
this.errorHandler?.({ Message: message, Fatal: true });
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* @inheritdoc
|
|
410
|
+
*
|
|
411
|
+
* Sends `tool.result` correlated by `call_id`. Sent IMMEDIATELY (never queued): the
|
|
412
|
+
* provider's own guidance is "send it the moment your tool returns — no buffering, no
|
|
413
|
+
* waiting on reply.done"; it owns the continuation and speaks the result as soon as the
|
|
414
|
+
* floor allows. The wire `result` slot expects a JSON-STRING (not an object), which is
|
|
415
|
+
* exactly the contract's `output` shape — it passes through verbatim. The reply flag is
|
|
416
|
+
* set eagerly so a queued narration can't slip in before the spoken result.
|
|
417
|
+
*/
|
|
418
|
+
async SendToolResult(callID, output) {
|
|
419
|
+
this.sendFrame({ type: 'tool.result', call_id: callID, result: output });
|
|
420
|
+
this.responseActive = true; // the result's spoken continuation is imminent
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* @inheritdoc
|
|
424
|
+
*
|
|
425
|
+
* EMULATED via the mutable `system_prompt` (the protocol has no purpose-built context
|
|
426
|
+
* channel): the note is appended under a "Background updates" heading and the FULL
|
|
427
|
+
* prompt is re-sent via `session.update` — a config write, so it never triggers or
|
|
428
|
+
* disturbs generation and is sent immediately even mid-reply. The model has the notes
|
|
429
|
+
* available the next time it speaks.
|
|
430
|
+
*/
|
|
431
|
+
SendContextNote(text) {
|
|
432
|
+
this.contextNotes.push(text);
|
|
433
|
+
this.sendFrame({ type: 'session.update', session: { system_prompt: this.composePromptWithNotes() } });
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* @inheritdoc
|
|
437
|
+
*
|
|
438
|
+
* NATIVE on this provider: `reply.create` carries one-off per-response instructions.
|
|
439
|
+
* **Collision behavior: queue.** A `reply.create` sent mid-reply would collide with the
|
|
440
|
+
* in-flight generation, so the send is deferred until the active reply completes.
|
|
441
|
+
*/
|
|
442
|
+
RequestSpokenUpdate(instructions) {
|
|
443
|
+
this.enqueueOrRun(() => {
|
|
444
|
+
this.responseActive = true; // the instructed reply is now in flight
|
|
445
|
+
this.sendFrame({ type: 'reply.create', instructions });
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* @inheritdoc
|
|
450
|
+
*
|
|
451
|
+
* Sends `session.end` BEFORE closing the socket — without it the provider holds the
|
|
452
|
+
* session (billable) for a 30-second resume window.
|
|
453
|
+
*/
|
|
454
|
+
async Close() {
|
|
455
|
+
this.closedByConsumer = true;
|
|
456
|
+
this.failReadyWait('session closed by consumer before session.ready arrived');
|
|
457
|
+
if (this.socket) {
|
|
458
|
+
try {
|
|
459
|
+
this.sendFrame({ type: 'session.end' });
|
|
460
|
+
}
|
|
461
|
+
catch {
|
|
462
|
+
/* socket already dead — closing anyway */
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
this.socket?.close();
|
|
466
|
+
this.socket = null;
|
|
467
|
+
this.clearHandlers();
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Entry point for an inbound websocket frame. Multiplexes on `type` to focused
|
|
471
|
+
* per-concern handlers so each translation unit stays small and testable.
|
|
472
|
+
*/
|
|
473
|
+
HandleServerEvent(event) {
|
|
474
|
+
switch (event.type) {
|
|
475
|
+
case 'session.ready':
|
|
476
|
+
this.readyReceived = true;
|
|
477
|
+
this.resolveReady?.();
|
|
478
|
+
break;
|
|
479
|
+
case 'session.updated':
|
|
480
|
+
break; // config-apply confirmation — nothing to surface
|
|
481
|
+
case 'reply.started':
|
|
482
|
+
this.responseActive = true;
|
|
483
|
+
break;
|
|
484
|
+
case 'reply.audio':
|
|
485
|
+
this.handleReplyAudio(event.data);
|
|
486
|
+
break;
|
|
487
|
+
case 'transcript.user.delta':
|
|
488
|
+
this.emitTranscript('user', event.text, false);
|
|
489
|
+
break;
|
|
490
|
+
case 'transcript.user':
|
|
491
|
+
this.emitTranscript('user', event.text, true);
|
|
492
|
+
break;
|
|
493
|
+
case 'transcript.agent':
|
|
494
|
+
this.emitTranscript('assistant', event.text, true);
|
|
495
|
+
break;
|
|
496
|
+
case 'reply.done':
|
|
497
|
+
this.handleReplyDone(event.status);
|
|
498
|
+
break;
|
|
499
|
+
case 'tool.call':
|
|
500
|
+
this.handleToolCall(event);
|
|
501
|
+
break;
|
|
502
|
+
case 'input.speech.started':
|
|
503
|
+
case 'input.speech.stopped':
|
|
504
|
+
break; // raw VAD telemetry — NOT an interruption signal (see the class note)
|
|
505
|
+
case 'session.error':
|
|
506
|
+
this.errorHandler?.({
|
|
507
|
+
Message: event.message ?? 'AssemblyAI session error',
|
|
508
|
+
Code: event.code,
|
|
509
|
+
Fatal: false,
|
|
510
|
+
});
|
|
511
|
+
break;
|
|
512
|
+
case 'session.ended':
|
|
513
|
+
this.handleSessionEnded();
|
|
514
|
+
break;
|
|
515
|
+
default:
|
|
516
|
+
break; // unknown / future frame types are ignored
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
/** Decodes one base64 reply-audio frame and forwards it as a raw `ArrayBuffer`. */
|
|
520
|
+
handleReplyAudio(audioBase64) {
|
|
521
|
+
if (!audioBase64) {
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
this.responseActive = true;
|
|
525
|
+
this.outputHandler?.(AssemblyAIRealtimeSession.Base64ToArrayBuffer(audioBase64));
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Reply boundary. `status: 'interrupted'` is the provider's authoritative true-barge-in
|
|
529
|
+
* verdict (user speech cut off active output — see the class note) and is surfaced via
|
|
530
|
+
* {@link OnInterruption} BEFORE the boundary releases the floor and drains queued sends.
|
|
531
|
+
*/
|
|
532
|
+
handleReplyDone(status) {
|
|
533
|
+
if (status === 'interrupted') {
|
|
534
|
+
this.interruptionHandler?.();
|
|
535
|
+
}
|
|
536
|
+
this.completeResponse();
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Surfaces a `tool.call` to the consumer. The model has yielded the floor pending the
|
|
540
|
+
* result, so the busy flag is cleared (deadlock guard — driver obligation #2) WITHOUT
|
|
541
|
+
* draining the queue (a queued narration must not trigger a reply between the tool call
|
|
542
|
+
* and its result; it drains at the next real reply boundary). The provider emits
|
|
543
|
+
* `arguments` ALREADY PARSED, so it is re-stringified to honor the Core contract's
|
|
544
|
+
* JSON-string `Arguments` shape.
|
|
545
|
+
*/
|
|
546
|
+
handleToolCall(event) {
|
|
547
|
+
this.responseActive = false;
|
|
548
|
+
this.toolCallHandler?.({
|
|
549
|
+
CallID: event.call_id ?? '',
|
|
550
|
+
ToolName: event.name ?? '',
|
|
551
|
+
Arguments: JSON.stringify(event.arguments ?? {}),
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* The provider ended the session itself. After a consumer {@link Close} (which sends
|
|
556
|
+
* `session.end`) this is the expected acknowledgment and stays silent; otherwise it is
|
|
557
|
+
* surfaced as FATAL so the consumer finalizes instead of idling on a dying socket.
|
|
558
|
+
*/
|
|
559
|
+
handleSessionEnded() {
|
|
560
|
+
if (this.closedByConsumer) {
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
const message = 'AssemblyAI agent session ended by the provider';
|
|
564
|
+
this.failReadyWait(message);
|
|
565
|
+
this.errorHandler?.({ Message: message, Fatal: true });
|
|
566
|
+
}
|
|
567
|
+
/** Reply boundary: releases the busy flag and drains queued sends in order. */
|
|
568
|
+
completeResponse() {
|
|
569
|
+
this.responseActive = false;
|
|
570
|
+
while (!this.responseActive && this.queuedSends.length > 0) {
|
|
571
|
+
const send = this.queuedSends.shift();
|
|
572
|
+
send?.();
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
/** Emits a transcript event (user turns have deltas; agent turns are final-only). */
|
|
576
|
+
emitTranscript(role, text, isFinal) {
|
|
577
|
+
if (!this.transcriptHandler || !text || text.trim().length === 0) {
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
this.transcriptHandler({ Role: role, Text: text, IsFinal: isFinal });
|
|
581
|
+
}
|
|
582
|
+
/** Runs a send immediately when idle; otherwise queues it for the next reply boundary. */
|
|
583
|
+
enqueueOrRun(send) {
|
|
584
|
+
if (this.responseActive) {
|
|
585
|
+
this.queuedSends.push(send);
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
send();
|
|
589
|
+
}
|
|
590
|
+
/** The base prompt plus every accumulated context note under a "Background updates" heading. */
|
|
591
|
+
composePromptWithNotes() {
|
|
592
|
+
if (this.contextNotes.length === 0) {
|
|
593
|
+
return this.basePrompt;
|
|
594
|
+
}
|
|
595
|
+
return `${this.basePrompt}\n\n## Background updates\n${this.contextNotes.map((n) => `- ${n}`).join('\n')}`;
|
|
596
|
+
}
|
|
597
|
+
/** JSON-serializes and sends one client frame (throws if the socket was never attached). */
|
|
598
|
+
sendFrame(frame) {
|
|
599
|
+
if (!this.socket) {
|
|
600
|
+
throw new Error('AssemblyAI realtime session is not open (no socket attached or it was closed).');
|
|
601
|
+
}
|
|
602
|
+
this.socket.send(JSON.stringify(frame));
|
|
603
|
+
}
|
|
604
|
+
/** Rejects a still-pending ready wait (transport death / consumer close during startup). */
|
|
605
|
+
failReadyWait(message) {
|
|
606
|
+
if (!this.readyReceived && this.rejectReady) {
|
|
607
|
+
const reject = this.rejectReady;
|
|
608
|
+
this.rejectReady = null;
|
|
609
|
+
this.resolveReady = null;
|
|
610
|
+
this.readyReceived = true; // nothing further can resolve/reject it
|
|
611
|
+
reject(new Error(message));
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
/** Drops all registered handlers so a closed session can't fire stale callbacks. */
|
|
615
|
+
clearHandlers() {
|
|
616
|
+
this.outputHandler = null;
|
|
617
|
+
this.transcriptHandler = null;
|
|
618
|
+
this.toolCallHandler = null;
|
|
619
|
+
this.interruptionHandler = null;
|
|
620
|
+
this.queuedSends = [];
|
|
621
|
+
this.responseActive = false;
|
|
622
|
+
}
|
|
623
|
+
/** Decodes a base64 audio payload into a freshly-allocated `ArrayBuffer`. */
|
|
624
|
+
static Base64ToArrayBuffer(base64) {
|
|
625
|
+
const bytes = Buffer.from(base64, 'base64');
|
|
626
|
+
const out = new ArrayBuffer(bytes.byteLength);
|
|
627
|
+
new Uint8Array(out).set(bytes);
|
|
628
|
+
return out;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
//# sourceMappingURL=assemblyAIRealtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assemblyAIRealtime.js","sourceRoot":"","sources":["../src/assemblyAIRealtime.ts"],"names":[],"mappings":"AAAA,wDAAwD;;;;;;;;AAExD,kCAAkC;AAClC,OAAO,EACH,iBAAiB,GAUpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,6FAA6F;AAC7F,MAAM,CAAC,MAAM,uBAAuB,GAAG,mCAAmC,CAAC;AAE3E,gGAAgG;AAChG,MAAM,CAAC,MAAM,oBAAoB,GAAG,wCAAwC,CAAC;AAE7E;;;;GAIG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAEhD;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,KAAK,CAAC;AA4FhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEI,IAAM,kBAAkB,0BAAxB,MAAM,kBAAmB,SAAQ,iBAAiB;IACrD;;;;;;;OAOG;IACI,KAAK,CAAC,YAAY,CAAC,MAA6B;QACnD,MAAM,OAAO,GAAG,IAAI,yBAAyB,CAAC,oBAAkB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7F,OAAO,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC;YACzC,GAAG,EAAE,GAAG,uBAAuB,UAAU,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAC1E,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC;YACtD,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,oBAAoB,CAAC,OAAO,CAAC;YAC3D,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC;SACxE,CAAC,CAAC;QACH,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC7B,OAAO,CAAC,iBAAiB,EAAE,CAAC;QAC5B,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;QAC7B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACH,IAAoB,oBAAoB;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACa,KAAK,CAAC,mBAAmB,CAAC,MAA6B;QACnE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,4BAA4B,CAAC,CAAC;QACvE,OAAO;YACH,QAAQ,EAAE,YAAY;YACtB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,4BAA4B,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE;YACnF,aAAa,EAAE;gBACX,OAAO,EAAE,oBAAkB,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBACtD,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;aAC9B;SACJ,CAAC;IACN,CAAC;IAED,+EAA+E;IAE/E;;;;;;;;;OASG;IACI,MAAM,CAAC,kBAAkB,CAAC,MAA6B;QAC1D,MAAM,OAAO,GAAe;YACxB,aAAa,EAAE,oBAAkB,CAAC,mBAAmB,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,cAAc,CAAC;SACpG,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,oBAAkB,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,CAAC;QACD,MAAM,KAAK,GAAe,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,gBAAgB,CAAC,KAAK,IAAI,IAAI,OAAO,MAAM,CAAC,gBAAgB,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;YAChI,KAAK,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YACpC,KAAK,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,uFAAuF;IAChF,MAAM,CAAC,mBAAmB,CAAC,YAAoB,EAAE,cAAuB;QAC3E,MAAM,OAAO,GAAG,cAAc,EAAE,IAAI,EAAE,CAAC;QACvC,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,YAAY,yBAAyB,OAAO,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;IACtF,CAAC;IAED,8EAA8E;IACvE,MAAM,CAAC,iBAAiB,CAAC,IAA4B;QACxD,OAAO;YACH,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,+EAA+E;YAC/E,6CAA6C;YAC7C,UAAU,EAAE,IAAI,CAAC,gBAAgB;SACpC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,kBAAkB,CAAC,KAA+B;QAC5D,OAAO,IAAI,CAAC,SAAS,CACjB,CAAC,GAAG,KAAK,CAAC;aACL,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,CACxG,CAAC;IACN,CAAC;IAED,6EAA6E;IAE7E;;;OAGG;IACO,KAAK,CAAC,eAAe,CAAC,gBAAwB;QACpD,MAAM,OAAO,GAAI,UAA+C,CAAC,KAAK,CAAC;QACvE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;QACvH,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,oBAAoB,uBAAuB,gBAAgB,EAAE,EAAE;YAC7F,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,EAAE;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAuB,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,kBAAkB,CAAC,IAA2B;QAC1D,MAAM,EAAE,GAAI,UAAkF,CAAC,SAAS,CAAC;QACzG,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC;QACpH,CAAC;QACD,OAAO,IAAI,OAAO,CAA2B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7D,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE;gBACb,MAAM,GAAG,IAAI,CAAC;gBACd,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACxE,CAAC,CAAC;YACF,EAAE,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;gBACrB,IAAI,CAAC;oBACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAA0B,CAAC,CAAC;gBAC5E,CAAC;gBAAC,MAAM,CAAC;oBACL,6BAA6B;gBACjC,CAAC;YACL,CAAC,CAAC;YACF,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE;gBACd,IAAI,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;gBACjD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,CAAC;gBACnE,CAAC;YACL,CAAC,CAAC;YACF,EAAE,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;gBACnB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACvC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,CAAC,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC,CAAC;gBAC1E,CAAC;YACL,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;CACJ,CAAA;AA3LY,kBAAkB;IAD9B,aAAa,CAAC,iBAAiB,EAAE,oBAAoB,CAAC;GAC1C,kBAAkB,CA2L9B;;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,yBAAyB;IAyClC,YAAY,aAAyB;QAxC7B,WAAM,GAAoC,IAAI,CAAC;QAE/C,kBAAa,GAA0C,IAAI,CAAC;QAC5D,sBAAiB,GAA6C,IAAI,CAAC;QACnE,oBAAe,GAA8C,IAAI,CAAC;QAClE,wBAAmB,GAAwB,IAAI,CAAC;QAChD,iBAAY,GAAmD,IAAI,CAAC;QAC5E,mFAAmF;QAC3E,qBAAgB,GAAG,KAAK,CAAC;QAIzB,iBAAY,GAAwB,IAAI,CAAC;QACzC,gBAAW,GAAoC,IAAI,CAAC;QACpD,kBAAa,GAAG,KAAK,CAAC;QAM9B,yFAAyF;QACjF,iBAAY,GAAa,EAAE,CAAC;QAEpC;;;;;WAKG;QACK,mBAAc,GAAG,KAAK,CAAC;QAE/B,wFAAwF;QAChF,gBAAW,GAAsB,EAAE,CAAC;QAE5C;;;WAGG;QACK,4BAAuB,GAAG,kBAAkB,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAGxE,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,OAAO,aAAa,CAAC,eAAe,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3G,IAAI,CAAC,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;YAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC9B,CAAC,CAAC,CAAC;QACH,+EAA+E;QAC/E,oFAAoF;QACpF,iEAAiE;QACjE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,oFAAoF;IAC7E,YAAY,CAAC,MAAgC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED;;;;OAIG;IACI,iBAAiB;QACpB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;OAKG;IACI,YAAY;QACf,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,2FAA2F;IACpF,SAAS,CAAC,KAAkB;QAC/B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,aAAa,CAAC,KAA+B;QACtD,MAAM,WAAW,GAAG,kBAAkB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACjE,IAAI,WAAW,KAAK,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/C,OAAO,CAAC,+CAA+C;QAC3D,CAAC;QACD,IAAI,CAAC,uBAAuB,GAAG,WAAW,CAAC;QAC3C,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE;SACtF,CAAC,CAAC;IACP,CAAC;IAED,wGAAwG;IACjG,mBAAmB,CAAC,KAA+B;QACtD,IAAI,CAAC,uBAAuB,GAAG,kBAAkB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAChF,CAAC;IAED,kBAAkB;IACX,QAAQ,CAAC,OAAqC;QACjD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IACjC,CAAC;IAED,kBAAkB;IACX,YAAY,CAAC,OAAwC;QACxD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;IACrC,CAAC;IAED,kBAAkB;IACX,UAAU,CAAC,OAAyC;QACvD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED,kBAAkB;IACX,cAAc,CAAC,OAAmB;QACrC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACI,OAAO,CAAC,QAAoC;QAC/C,yEAAyE;IAC7E,CAAC;IAED,kBAAkB;IACX,OAAO,CAAC,OAA8C;QACzD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;OAGG;IACI,oBAAoB,CAAC,OAAe;QACvC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;OAKG;IACI,oBAAoB,CAAC,IAAa,EAAE,MAAe;QACtD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClG,MAAM,OAAO,GAAG,+CAA+C,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC9F,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,MAAc;QACtD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,+CAA+C;IAC/E,CAAC;IAED;;;;;;;;OAQG;IACI,eAAe,CAAC,IAAY;QAC/B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,sBAAsB,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED;;;;;;OAMG;IACI,mBAAmB,CAAC,YAAoB;QAC3C,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE;YACnB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,wCAAwC;YACpE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,KAAK;QACd,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,CAAC,aAAa,CAAC,yDAAyD,CAAC,CAAC;QAC9E,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACL,0CAA0C;YAC9C,CAAC;QACL,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,aAAa,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACI,iBAAiB,CAAC,KAA4B;QACjD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,eAAe;gBAChB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;gBACtB,MAAM;YACV,KAAK,iBAAiB;gBAClB,MAAM,CAAC,iDAAiD;YAC5D,KAAK,eAAe;gBAChB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,MAAM;YACV,KAAK,aAAa;gBACd,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClC,MAAM;YACV,KAAK,uBAAuB;gBACxB,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC/C,MAAM;YACV,KAAK,iBAAiB;gBAClB,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC9C,MAAM;YACV,KAAK,kBAAkB;gBACnB,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACnD,MAAM;YACV,KAAK,YAAY;gBACb,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM;YACV,KAAK,WAAW;gBACZ,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC3B,MAAM;YACV,KAAK,sBAAsB,CAAC;YAC5B,KAAK,sBAAsB;gBACvB,MAAM,CAAC,sEAAsE;YACjF,KAAK,eAAe;gBAChB,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,0BAA0B;oBACpD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK;iBACf,CAAC,CAAC;gBACH,MAAM;YACV,KAAK,eAAe;gBAChB,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC1B,MAAM;YACV;gBACI,MAAM,CAAC,2CAA2C;QAC1D,CAAC;IACL,CAAC;IAED,mFAAmF;IAC3E,gBAAgB,CAAC,WAA+B;QACpD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,aAAa,EAAE,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;IACrF,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,MAA0B;QAC9C,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;YAC3B,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACK,cAAc,CAAC,KAA4B;QAC/C,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,MAAM,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE;YAC3B,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE;YAC1B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC;SACnD,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACK,kBAAkB;QACtB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,gDAAgD,CAAC;QACjE,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,+EAA+E;IACvE,gBAAgB;QACpB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,EAAE,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED,qFAAqF;IAC7E,cAAc,CAAC,IAA0B,EAAE,IAAwB,EAAE,OAAgB;QACzF,IAAI,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/D,OAAO;QACX,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,0FAA0F;IAClF,YAAY,CAAC,IAAgB;QACjC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED,gGAAgG;IACxF,sBAAsB;QAC1B,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,UAAU,8BAA8B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC/G,CAAC;IAED,4FAA4F;IACpF,SAAS,CAAC,KAAiB;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,gFAAgF,CAAC,CAAC;QACtG,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,4FAA4F;IACpF,aAAa,CAAC,OAAe;QACjC,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,wCAAwC;YACnE,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED,oFAAoF;IAC5E,aAAa;QACjB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;IAChC,CAAC;IAED,6EAA6E;IACrE,MAAM,CAAC,mBAAmB,CAAC,MAAc;QAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC;IACf,CAAC;CACJ"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,sBAAsB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAC1F,cAAc,sBAAsB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,10 +1,29 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/ai-assemblyai",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "5.41.0",
|
|
5
|
+
"description": "MemberJunction Wrapper for AssemblyAI - Voice Agent API realtime driver",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"/dist"
|
|
10
|
+
],
|
|
11
|
+
"scripts": {
|
|
12
|
+
"build": "tsc && tsc-alias -f",
|
|
13
|
+
"test": "vitest run",
|
|
14
|
+
"test:watch": "vitest"
|
|
15
|
+
},
|
|
16
|
+
"author": "MemberJunction.com",
|
|
17
|
+
"license": "ISC",
|
|
18
|
+
"devDependencies": {
|
|
19
|
+
"typescript": "^5.9.3"
|
|
20
|
+
},
|
|
21
|
+
"dependencies": {
|
|
22
|
+
"@memberjunction/ai": "5.41.0",
|
|
23
|
+
"@memberjunction/global": "5.41.0"
|
|
24
|
+
},
|
|
25
|
+
"repository": {
|
|
26
|
+
"type": "git",
|
|
27
|
+
"url": "https://github.com/MemberJunction/MJ"
|
|
28
|
+
}
|
|
10
29
|
}
|
package/readme.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# @memberjunction/ai-assemblyai
|
|
2
|
+
|
|
3
|
+
MemberJunction provider package for **AssemblyAI**, currently exposing the **Voice Agent API**
|
|
4
|
+
(launched April 2026) as a realtime, full-duplex, tool-calling model.
|
|
5
|
+
|
|
6
|
+
## What this package provides
|
|
7
|
+
|
|
8
|
+
- **`AssemblyAIRealtime`** — a `BaseRealtimeModel` driver (registered via
|
|
9
|
+
`@RegisterClass(BaseRealtimeModel, 'AssemblyAIRealtime')`) for AssemblyAI's single-websocket
|
|
10
|
+
speech-to-speech stack: Universal-3 Pro streaming ASR, server-side turn detection and
|
|
11
|
+
barge-in, LLM reasoning, JSON-Schema tool calling, and conversational TTS.
|
|
12
|
+
- **`AssemblyAIRealtimeSession`** — the `IRealtimeSession` implementation backing the
|
|
13
|
+
server-bridged topology.
|
|
14
|
+
|
|
15
|
+
The matching **browser-direct client driver** (`AssemblyAIRealtimeClient`, ClassFactory key
|
|
16
|
+
`'assemblyai'`) ships in `@memberjunction/ai-realtime-client`.
|
|
17
|
+
|
|
18
|
+
## Provider characteristics
|
|
19
|
+
|
|
20
|
+
| Concern | How this provider does it |
|
|
21
|
+
| --- | --- |
|
|
22
|
+
| Endpoint | One websocket: `wss://agents.assemblyai.com/v1/ws?token=…` |
|
|
23
|
+
| Session config | Native per-session `session.update` (prompt, tools, voice, turn detection) — no server-side agent object to manage |
|
|
24
|
+
| Audio | PCM16 mono **24 kHz**, base64, both directions (fixed — no negotiation) |
|
|
25
|
+
| Tool calling | JSON-Schema `function` tools; `tool.call` arguments arrive parsed; `tool.result` takes a JSON **string** |
|
|
26
|
+
| Tools mid-session | **Mutable** — `RegisterTools` re-declares natively via `session.update` |
|
|
27
|
+
| Narration (`RequestSpokenUpdate`) | **Native** via `reply.create` instructions |
|
|
28
|
+
| Context notes (`SendContextNote`) | Emulated via the **mutable `system_prompt`** ("Background updates" section) |
|
|
29
|
+
| Typed text (client `SendText`) | Emulated via `reply.create` instructions (no typed-input wire event) |
|
|
30
|
+
| Interruption | `reply.done` `status: 'interrupted'` (authoritative); the client driver also flushes on `input.speech.started` while output is active for snappier barge-in |
|
|
31
|
+
| Usage events | **None** — flat hourly session billing ($4.50/hr); `OnUsage` never fires |
|
|
32
|
+
| Client-direct | Supported — one-time temp token minted via `GET /v1/token` (Bearer API key) |
|
|
33
|
+
| Session end | `Close()`/`Disconnect()` send `session.end` first — skipping it leaves a billable 30-second resume hold |
|
|
34
|
+
|
|
35
|
+
## Configuration
|
|
36
|
+
|
|
37
|
+
- API key env alias: `AI_VENDOR_API_KEY__AssemblyAIRealtime`
|
|
38
|
+
- `RealtimeSessionParams.Config` passthrough keys: `voice`, `greeting`, `turn_detection`
|
|
39
|
+
(object), `keyterms` (string array)
|
|
40
|
+
- `Model` / `APIName` plays no wire role (single endpoint, no model selection) — the metadata
|
|
41
|
+
row uses `voice-agent`
|
|
42
|
+
|
|
43
|
+
## Docs
|
|
44
|
+
|
|
45
|
+
- Product: https://www.assemblyai.com/products/voice-agent-api
|
|
46
|
+
- API reference: https://www.assemblyai.com/docs/voice-agents/voice-agent-api
|
|
47
|
+
- Token endpoint: `GET https://agents.assemblyai.com/v1/token?expires_in_seconds=…`
|
|
48
|
+
- Full MJ realtime architecture (topologies, co-agent model, four-provider capability matrix): [guides/REALTIME_CO_AGENTS_GUIDE.md](../../../../guides/REALTIME_CO_AGENTS_GUIDE.md)
|
package/README.md
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
# @memberjunction/ai-assemblyai
|
|
2
|
-
|
|
3
|
-
## ⚠️ IMPORTANT NOTICE ⚠️
|
|
4
|
-
|
|
5
|
-
**This package is created solely for the purpose of setting up OIDC (OpenID Connect) trusted publishing with npm.**
|
|
6
|
-
|
|
7
|
-
This is **NOT** a functional package and contains **NO** code or functionality beyond the OIDC setup configuration.
|
|
8
|
-
|
|
9
|
-
## Purpose
|
|
10
|
-
|
|
11
|
-
This package exists to:
|
|
12
|
-
1. Configure OIDC trusted publishing for the package name `@memberjunction/ai-assemblyai`
|
|
13
|
-
2. Enable secure, token-less publishing from CI/CD workflows
|
|
14
|
-
3. Establish provenance for packages published under this name
|
|
15
|
-
|
|
16
|
-
## What is OIDC Trusted Publishing?
|
|
17
|
-
|
|
18
|
-
OIDC trusted publishing allows package maintainers to publish packages directly from their CI/CD workflows without needing to manage npm access tokens. Instead, it uses OpenID Connect to establish trust between the CI/CD provider (like GitHub Actions) and npm.
|
|
19
|
-
|
|
20
|
-
## Setup Instructions
|
|
21
|
-
|
|
22
|
-
To properly configure OIDC trusted publishing for this package:
|
|
23
|
-
|
|
24
|
-
1. Go to [npmjs.com](https://www.npmjs.com/) and navigate to your package settings
|
|
25
|
-
2. Configure the trusted publisher (e.g., GitHub Actions)
|
|
26
|
-
3. Specify the repository and workflow that should be allowed to publish
|
|
27
|
-
4. Use the configured workflow to publish your actual package
|
|
28
|
-
|
|
29
|
-
## DO NOT USE THIS PACKAGE
|
|
30
|
-
|
|
31
|
-
This package is a placeholder for OIDC configuration only. It:
|
|
32
|
-
- Contains no executable code
|
|
33
|
-
- Provides no functionality
|
|
34
|
-
- Should not be installed as a dependency
|
|
35
|
-
- Exists only for administrative purposes
|
|
36
|
-
|
|
37
|
-
## More Information
|
|
38
|
-
|
|
39
|
-
For more details about npm's trusted publishing feature, see:
|
|
40
|
-
- [npm Trusted Publishing Documentation](https://docs.npmjs.com/generating-provenance-statements)
|
|
41
|
-
- [GitHub Actions OIDC Documentation](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect)
|
|
42
|
-
|
|
43
|
-
---
|
|
44
|
-
|
|
45
|
-
**Maintained for OIDC setup purposes only**
|