@alexkroman1/aai-ui 0.12.3 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +253 -0
- package/dist/_colors-CpZO-88A.js +78 -0
- package/dist/_module-url-C13kAJ87.js +15 -0
- package/dist/_page-shell.d.ts +11 -0
- package/dist/_recover-run.d.ts +79 -0
- package/dist/_repeat-until.d.ts +30 -0
- package/dist/_run-controls.d.ts +33 -0
- package/dist/_submission-state.d.ts +92 -0
- package/dist/_tap-to-talk-state.d.ts +88 -0
- package/dist/_utils-Mi-SCFA1.js +47 -0
- package/dist/_utils.d.ts +26 -0
- package/dist/_web-storage.d.ts +43 -0
- package/dist/_workflow-api-ref.d.ts +37 -0
- package/dist/_workflow-files.d.ts +34 -0
- package/dist/_workflow-test-defs.d.ts +34 -0
- package/dist/aai-logo-D8pMVn9_.js +85 -0
- package/dist/agent-state-labels.d.ts +65 -0
- package/dist/agent-state.d.ts +108 -0
- package/dist/audio/capture.d.ts +44 -0
- package/dist/audio/index.d.ts +18 -0
- package/dist/audio/index.js +431 -0
- package/dist/audio/preconnect.d.ts +87 -0
- package/dist/audio/voice-io.d.ts +112 -0
- package/dist/build-default-client.d.ts +1 -0
- package/dist/chat-view-BCbI7gD4.js +396 -0
- package/dist/client-config-C7fQrioM.js +139 -0
- package/dist/client-config.d.ts +119 -0
- package/dist/client-dir.d.ts +44 -0
- package/dist/client-dir.js +58 -0
- package/dist/client-identity.d.ts +131 -0
- package/dist/client-tool.d.ts +51 -0
- package/dist/components/_colors.d.ts +73 -0
- package/dist/components/_form-readiness.d.ts +56 -0
- package/dist/components/_form-values.d.ts +19 -0
- package/dist/components/aai-logo.d.ts +12 -0
- package/dist/components/audio-result.d.ts +94 -0
- package/dist/components/auto-scroll.d.ts +74 -0
- package/dist/components/bullet-list.d.ts +74 -0
- package/dist/components/button.d.ts +29 -21
- package/dist/components/button.js +62 -23
- package/dist/components/chat-view.d.ts +24 -18
- package/dist/components/chat-view.js +2 -60
- package/dist/components/console-shell.d.ts +85 -0
- package/dist/components/controls.d.ts +21 -6
- package/dist/components/controls.js +31 -19
- package/dist/components/conversation-view.d.ts +130 -0
- package/dist/components/eyebrow.d.ts +12 -0
- package/dist/components/facts.d.ts +81 -0
- package/dist/components/form-fields.d.ts +142 -0
- package/dist/components/form-types.d.ts +73 -0
- package/dist/components/form.d.ts +96 -0
- package/dist/components/markdown.d.ts +57 -0
- package/dist/components/message-list.d.ts +34 -7
- package/dist/components/message-list.js +2 -75
- package/dist/components/session-controls.d.ts +125 -0
- package/dist/components/session-error-banner.d.ts +69 -0
- package/dist/components/session-state-dot.d.ts +111 -0
- package/dist/components/sidebar-layout.d.ts +31 -10
- package/dist/components/sidebar-layout.js +32 -12
- package/dist/components/start-screen.d.ts +17 -6
- package/dist/components/start-screen.js +34 -14
- package/dist/components/tool-call-block.d.ts +15 -14
- package/dist/components/tool-call-block.js +2 -117
- package/dist/components/tool-call-row.d.ts +75 -0
- package/dist/components/tool-config-context.d.ts +22 -0
- package/dist/components/upload-progress.d.ts +76 -0
- package/dist/components/url-chips.d.ts +29 -0
- package/dist/components/workflow-fields.d.ts +46 -0
- package/dist/components/workflow-pending-note.d.ts +81 -0
- package/dist/components/workflow-progress.d.ts +81 -0
- package/dist/components/workflow-run-error.d.ts +48 -0
- package/dist/components/workflow-run-panel.d.ts +79 -0
- package/dist/context.d.ts +288 -0
- package/dist/context.js +382 -0
- package/dist/conversation-log.d.ts +88 -0
- package/dist/default-client/assets/_module-url-BX0RuRU2.js +1 -0
- package/dist/default-client/assets/audio-mpgipo2Q.js +1 -0
- package/dist/default-client/assets/capture-processor-D0q7Qrdd.js +90 -0
- package/dist/default-client/assets/index-8ihb1Urm.css +2 -0
- package/dist/default-client/assets/index-DPfmSFmc.js +126 -0
- package/dist/default-client/assets/playback-processor-CjDbc9iH.js +314 -0
- package/dist/default-client/favicon.ico +0 -0
- package/dist/default-client/index.html +15 -0
- package/dist/default-client-entry.d.ts +6 -0
- package/dist/default-client.d.ts +19 -0
- package/dist/define-client.d.ts +169 -26
- package/dist/define-client.js +209 -55
- package/dist/eyebrow-UfmSz9yy.js +27 -0
- package/dist/hooks-DN4gHYsj.js +182 -0
- package/dist/hooks.d.ts +172 -0
- package/dist/hooks.js +3 -0
- package/dist/hooks.test-d.d.ts +14 -0
- package/dist/inbox-protocol.d.ts +128 -0
- package/dist/inbox.d.ts +132 -0
- package/dist/index.d.ts +126 -29
- package/dist/index.js +5500 -15
- package/dist/internal.d.ts +43 -0
- package/dist/internal.js +8 -0
- package/dist/linked-client.d.ts +70 -0
- package/dist/message-list-DHZcLhRA.js +682 -0
- package/dist/notice-player.d.ts +70 -0
- package/dist/page.d.ts +114 -0
- package/dist/phone.d.ts +50 -0
- package/dist/route-fetch.d.ts +79 -0
- package/dist/session/audio-effects.d.ts +37 -0
- package/dist/session/audio-setup.d.ts +57 -0
- package/dist/session/audio-state.d.ts +157 -0
- package/dist/session/browser-session.d.ts +39 -0
- package/dist/session/close.d.ts +26 -0
- package/dist/session/dial.d.ts +65 -0
- package/dist/session/handshake.d.ts +60 -0
- package/dist/session/index.d.ts +17 -0
- package/dist/session/index.js +2 -0
- package/dist/session/messages.d.ts +63 -0
- package/dist/session/mic.d.ts +64 -0
- package/dist/session/preconnect.d.ts +48 -0
- package/dist/session/reconnect.d.ts +25 -0
- package/dist/session/resume-store.d.ts +57 -0
- package/dist/session/state.d.ts +146 -0
- package/dist/session/ticket.d.ts +44 -0
- package/dist/session/types.d.ts +476 -0
- package/dist/session/url.d.ts +17 -0
- package/dist/session/user-turn.d.ts +55 -0
- package/dist/session-D9jM5-TF.js +2140 -0
- package/dist/stored-value.d.ts +103 -0
- package/dist/tool-call-block-uXtMOUWN.js +204 -0
- package/dist/tool-config-context-Es4YUzV2.js +19 -0
- package/dist/types.d.ts +272 -34
- package/dist/types.js +35 -4
- package/dist/upload/files.d.ts +83 -0
- package/dist/upload/index.d.ts +15 -0
- package/dist/upload/pause.d.ts +31 -0
- package/dist/upload/recall.d.ts +67 -0
- package/dist/upload/report.d.ts +26 -0
- package/dist/upload/session.d.ts +99 -0
- package/dist/url-chips-oyy_Eiqh.js +239 -0
- package/dist/use-client-runs.d.ts +87 -0
- package/dist/use-conversation-log.d.ts +86 -0
- package/dist/use-conversation.d.ts +131 -0
- package/dist/use-copy.d.ts +74 -0
- package/dist/use-download-url.d.ts +84 -0
- package/dist/use-flash.d.ts +61 -0
- package/dist/use-inbox.d.ts +101 -0
- package/dist/use-push-to-talk.d.ts +105 -0
- package/dist/use-route-mutation.d.ts +113 -0
- package/dist/use-route.d.ts +82 -0
- package/dist/use-run-key.d.ts +133 -0
- package/dist/use-session-controls.d.ts +69 -0
- package/dist/use-session-id.d.ts +57 -0
- package/dist/use-tap-to-talk.d.ts +138 -0
- package/dist/use-user-transcript-uyHhzy4d.js +73 -0
- package/dist/use-user-transcript.d.ts +75 -0
- package/dist/use-workflow-form.d.ts +285 -0
- package/dist/use-workflow-progress.d.ts +137 -0
- package/dist/use-workflow-run-DdhWmbx-.js +697 -0
- package/dist/use-workflow-run.d.ts +100 -0
- package/dist/use-workflow-runs.d.ts +71 -0
- package/dist/use-workflow-stream.d.ts +154 -0
- package/dist/use-workflows.d.ts +77 -0
- package/dist/workflow-client.d.ts +136 -0
- package/dist/workflow-def-types.d.ts +31 -0
- package/dist/workflow-events.d.ts +39 -0
- package/dist/workflow-field-kind.d.ts +95 -0
- package/dist/workflow-status-labels.d.ts +35 -0
- package/dist/worklets/_module-url.d.ts +10 -0
- package/dist/worklets/_playback-bench-harness.d.ts +181 -0
- package/dist/worklets/_playback-bench-host.d.ts +63 -0
- package/dist/worklets/_playback-bench-page.d.ts +65 -0
- package/dist/worklets/_tts-trace-harness.d.ts +142 -0
- package/dist/worklets/_worklet-test-utils.d.ts +27 -0
- package/dist/worklets/capture-processor.d.ts +4 -2
- package/dist/worklets/capture-processor.js +73 -33
- package/dist/worklets/playback-processor.d.ts +4 -2
- package/dist/worklets/playback-processor.js +258 -42
- package/package.json +59 -32
- package/styles.css +152 -30
- package/dist/_jsdom-setup.d.ts +0 -0
- package/dist/audio.d.ts +0 -45
- package/dist/audio.js +0 -121
- package/dist/client-context.d.ts +0 -33
- package/dist/client-context.js +0 -15
- package/dist/client-handler.d.ts +0 -43
- package/dist/components/app.d.ts +0 -23
- package/dist/components/app.js +0 -41
- package/dist/components/error-banner.d.ts +0 -21
- package/dist/components/error-banner.js +0 -27
- package/dist/components/message-bubble.d.ts +0 -23
- package/dist/components/message-bubble.js +0 -35
- package/dist/components/state-indicator.d.ts +0 -29
- package/dist/components/state-indicator.js +0 -37
- package/dist/components/thinking-indicator.d.ts +0 -16
- package/dist/components/thinking-indicator.js +0 -34
- package/dist/components/tool-icons.d.ts +0 -17
- package/dist/components/tool-icons.js +0 -128
- package/dist/components/transcript.d.ts +0 -25
- package/dist/components/transcript.js +0 -35
- package/dist/session-CWB7vmqz.js +0 -429
- package/dist/session.d.ts +0 -69
- package/dist/session.js +0 -2
- package/dist/signals.d.ts +0 -120
- package/dist/signals.js +0 -161
- package/dist/tsdown.config.d.ts +0 -2
- package/dist/types.test-d.d.ts +0 -7
|
@@ -0,0 +1,2140 @@
|
|
|
1
|
+
import { n as pageBaseUrl } from "./_utils-Mi-SCFA1.js";
|
|
2
|
+
import { r as loadClientConfig, t as buildAgentUrl } from "./client-config-C7fQrioM.js";
|
|
3
|
+
import { MIC_SEND_MAX_BUFFERED_BYTES } from "./types.js";
|
|
4
|
+
import { SessionEventSchema, errorMessage, safeJsonParse } from "@alexkroman1/aai";
|
|
5
|
+
import { SESSION_AUTH_PROTOCOL_PREFIX, SESSION_PROTOCOL, lenientParse } from "@alexkroman1/aai/protocol";
|
|
6
|
+
import { CLIENT_ID_RE, MAX_CLIENT_MESSAGES, RESUME_ID_RE, WS_OPEN, createEpoch, toArgsRecord } from "@alexkroman1/aai/internal";
|
|
7
|
+
import { omitUndefined } from "@alexkroman1/aai/utils";
|
|
8
|
+
import { and, assign, createActor, fromPromise, not, setup, stateIn } from "xstate";
|
|
9
|
+
import ReconnectingWebSocket from "partysocket/ws";
|
|
10
|
+
//#region src/_web-storage.ts
|
|
11
|
+
/**
|
|
12
|
+
* The store, or nothing.
|
|
13
|
+
*
|
|
14
|
+
* Not exported: a caller that reaches for the store itself has stepped outside
|
|
15
|
+
* the guard, which is the whole point of this module.
|
|
16
|
+
*/
|
|
17
|
+
function storeFor(kind) {
|
|
18
|
+
return kind === "local" ? globalThis.localStorage : globalThis.sessionStorage;
|
|
19
|
+
}
|
|
20
|
+
/** The stored value, or undefined — for a missing entry and an absent store alike. */
|
|
21
|
+
function storageGet(kind, key) {
|
|
22
|
+
try {
|
|
23
|
+
return storeFor(kind)?.getItem(key) ?? void 0;
|
|
24
|
+
} catch {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** Remember a value. A store that refuses is a no-op, never a throw. */
|
|
29
|
+
function storageSet(kind, key, value) {
|
|
30
|
+
try {
|
|
31
|
+
storeFor(kind)?.setItem(key, value);
|
|
32
|
+
} catch {}
|
|
33
|
+
}
|
|
34
|
+
/** Forget a value. Nothing stored and no store are the same outcome. */
|
|
35
|
+
function storageRemove(kind, key) {
|
|
36
|
+
try {
|
|
37
|
+
storeFor(kind)?.removeItem(key);
|
|
38
|
+
} catch {}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* A storage key namespaced by a URL.
|
|
42
|
+
*
|
|
43
|
+
* `target` is resolved against the document, so a relative path ("./", the
|
|
44
|
+
* default-client case) and the absolute form of the same agent agree on one
|
|
45
|
+
* key.
|
|
46
|
+
*
|
|
47
|
+
* @param prefix - The owning module's namespace, e.g. `"aai:session:"`.
|
|
48
|
+
* @param target - What to resolve — an agent's `platformUrl`, or `"./"` for the
|
|
49
|
+
* page's own directory.
|
|
50
|
+
*/
|
|
51
|
+
function urlSlot(prefix, target) {
|
|
52
|
+
try {
|
|
53
|
+
return `${prefix}${new URL(target, globalThis.location?.href).href}`;
|
|
54
|
+
} catch {
|
|
55
|
+
return `${prefix}${target}`;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//#endregion
|
|
59
|
+
//#region src/client-identity.ts
|
|
60
|
+
/**
|
|
61
|
+
* Who this browser is to an agent: the id `mountClient({ client: "auto" })`
|
|
62
|
+
* sends as `?client=`, the inbox HOLDER id of this tab, and the
|
|
63
|
+
* `session.identity` handle both are read from.
|
|
64
|
+
*
|
|
65
|
+
* ## Why the SDK mints the client id
|
|
66
|
+
*
|
|
67
|
+
* A client id is what a tool hands a workflow run (`sessionClientId(ctx)`) so
|
|
68
|
+
* the run's `stepNotifyClient` can reach this browser AFTER the session that
|
|
69
|
+
* started it has closed — a reminder due tomorrow is delivered to whoever holds
|
|
70
|
+
* the id tomorrow. So it must be the SAME id on the session socket and on the
|
|
71
|
+
* inbox socket, and it must outlive the tab. Every app that wanted a reminder in
|
|
72
|
+
* a browser wrote the same thirty lines (a `localStorage` read, a validity
|
|
73
|
+
* check, a fallback for private mode) and got the second half of this module
|
|
74
|
+
* wrong: see the holder below.
|
|
75
|
+
*
|
|
76
|
+
* - **`localStorage`, keyed by the agent's URL** (`_web-storage.ts`'s rule). Two
|
|
77
|
+
* agents served from one origin — every deployed agent, at `/:slug/` — must
|
|
78
|
+
* not share one: the id is the ONLY credential for a client's durable
|
|
79
|
+
* conversation (`VoiceSessionOptions.client`), so agent A's server, which
|
|
80
|
+
* receives it, could otherwise present it to agent B and read B's history.
|
|
81
|
+
* - **Unguessable**: 128 random bits, from `crypto.getRandomValues` — which,
|
|
82
|
+
* unlike `crypto.randomUUID`, exists outside a secure context, and `aai dev`
|
|
83
|
+
* on a LAN address is plain `http:`.
|
|
84
|
+
* - **Private mode** (storage throws or is absent) degrades to an id for THIS
|
|
85
|
+
* TAB, held in memory: the session and inbox still agree, a reminder set in
|
|
86
|
+
* the tab reaches the tab, and nothing outlives it.
|
|
87
|
+
*
|
|
88
|
+
* ## Why the holder is per TAB, not per browser
|
|
89
|
+
*
|
|
90
|
+
* The inbox keeps one socket per (client, holder) pair, and the same pair again
|
|
91
|
+
* REPLACES the old socket — right for a device back from a Wi-Fi drop, whose old
|
|
92
|
+
* socket is half-dead. Every tab of a browser shares `localStorage`, so an app
|
|
93
|
+
* that used the browser-wide id as the holder had two tabs presenting the same
|
|
94
|
+
* pair: each connect knocked the other tab off, the other reconnected on its
|
|
95
|
+
* backoff and knocked this one off, about once a second, forever. The holder is
|
|
96
|
+
* the browser id plus a suffix minted once per PAGE LOAD, so tabs coexist as
|
|
97
|
+
* holders of one client and every one of them is offered the notice.
|
|
98
|
+
*
|
|
99
|
+
* @module
|
|
100
|
+
*/
|
|
101
|
+
const PREFIX$1 = "aai:client:";
|
|
102
|
+
/** Lowercase hex of `bytes` random bytes. */
|
|
103
|
+
function randomHex(bytes) {
|
|
104
|
+
const buf = crypto.getRandomValues(new Uint8Array(bytes));
|
|
105
|
+
return Array.from(buf, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* This page load's suffix: minted once per JS realm, which is once per tab per
|
|
109
|
+
* load. Not stored anywhere — that is the point (see the module doc).
|
|
110
|
+
*/
|
|
111
|
+
const TAB = randomHex(4);
|
|
112
|
+
/**
|
|
113
|
+
* Ids already resolved in this realm, per storage slot. It is what makes a
|
|
114
|
+
* private-mode id stable for the tab (nothing else remembers it) and saves a
|
|
115
|
+
* storage read on every connection attempt otherwise.
|
|
116
|
+
*/
|
|
117
|
+
const minted = /* @__PURE__ */ new Map();
|
|
118
|
+
/**
|
|
119
|
+
* This browser's stable client id for the agent at `platformUrl` —
|
|
120
|
+
* `browser-<32 hex>`, minted once and kept in `localStorage`. The id
|
|
121
|
+
* `mountClient({ client: "auto" })` sends; call it yourself to compose it, e.g.
|
|
122
|
+
* a page joined to a speaker that sends the speaker's id instead:
|
|
123
|
+
*
|
|
124
|
+
* @example
|
|
125
|
+
* ```ts
|
|
126
|
+
* import { browserClientId, mountClient } from "@alexkroman1/aai-ui";
|
|
127
|
+
*
|
|
128
|
+
* declare function linkedSpeaker(): string | undefined;
|
|
129
|
+
*
|
|
130
|
+
* mountClient({ client: () => linkedSpeaker() ?? browserClientId() });
|
|
131
|
+
* ```
|
|
132
|
+
*
|
|
133
|
+
* A stored value that is not a valid client id (letters, digits, `-`, `_`, at
|
|
134
|
+
* most 64) is replaced. Where storage is unavailable the id lasts for this tab.
|
|
135
|
+
*
|
|
136
|
+
* @param platformUrl - The agent's base URL; the id is per agent. Defaults to
|
|
137
|
+
* the page's own, which is `mountClient()`'s default too.
|
|
138
|
+
* @returns The id — the same one on every call for the same agent.
|
|
139
|
+
*
|
|
140
|
+
* @public
|
|
141
|
+
*/
|
|
142
|
+
function browserClientId(platformUrl = pageBaseUrl()) {
|
|
143
|
+
const key = urlSlot(PREFIX$1, platformUrl);
|
|
144
|
+
const known = minted.get(key);
|
|
145
|
+
if (known) return known;
|
|
146
|
+
const stored = storageGet("local", key);
|
|
147
|
+
const id = stored && CLIENT_ID_RE.test(stored) ? stored : `browser-${randomHex(16)}`;
|
|
148
|
+
if (id !== stored) storageSet("local", key, id);
|
|
149
|
+
minted.set(key, id);
|
|
150
|
+
return id;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* This TAB's inbox holder id for the agent at `platformUrl`: the browser id
|
|
154
|
+
* plus a per-page-load suffix. Never the browser id alone — see the module doc
|
|
155
|
+
* for the two tabs that knocked each other off once a second.
|
|
156
|
+
*
|
|
157
|
+
* @internal
|
|
158
|
+
*/
|
|
159
|
+
function inboxHolderId(platformUrl) {
|
|
160
|
+
return `${browserClientId(platformUrl)}-${TAB}`;
|
|
161
|
+
}
|
|
162
|
+
/** The client option as the dialer reads it: `"auto"` resolved, everything else as given. */
|
|
163
|
+
function resolveClientOption(client, platformUrl) {
|
|
164
|
+
return client === "auto" ? () => browserClientId(platformUrl) : client;
|
|
165
|
+
}
|
|
166
|
+
/** A string-or-getter option resolved NOW, trimmed; an empty answer is none. */
|
|
167
|
+
function resolveReported(value) {
|
|
168
|
+
const trimmed = (typeof value === "function" ? value() : value)?.trim();
|
|
169
|
+
return trimmed === "" ? void 0 : trimmed;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Build a session's {@link SessionIdentity}. `sessionId` and `ticket` are the
|
|
173
|
+
* dialer's (read through `source`, which may answer nothing): it owns the id
|
|
174
|
+
* because it sees every `config` frame, `end()` and `resume()`, and the ticket
|
|
175
|
+
* because it holds the `token` option and the server's `client-config` latch.
|
|
176
|
+
*/
|
|
177
|
+
function createSessionIdentity(options, source) {
|
|
178
|
+
const client = resolveClientOption(options.client, options.platformUrl);
|
|
179
|
+
return Object.freeze({
|
|
180
|
+
platformUrl: options.platformUrl,
|
|
181
|
+
clientId: () => resolveReported(client),
|
|
182
|
+
holderId: () => inboxHolderId(options.platformUrl),
|
|
183
|
+
sessionId: () => source()?.sessionId(),
|
|
184
|
+
ticket: () => source()?.ticket()
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
//#endregion
|
|
188
|
+
//#region src/session/audio-setup.ts
|
|
189
|
+
let audioModulesPromise = null;
|
|
190
|
+
/**
|
|
191
|
+
* Load the audio module and worklet sources, memoized so a prefetch at
|
|
192
|
+
* `connect()` time overlaps the chunk fetch with the WebSocket handshake
|
|
193
|
+
* instead of serializing it behind the server's `config` frame. A failed
|
|
194
|
+
* load clears the memo so the next attempt retries the import.
|
|
195
|
+
*/
|
|
196
|
+
function loadAudioModules() {
|
|
197
|
+
audioModulesPromise ??= Promise.all([
|
|
198
|
+
import("./audio/index.js"),
|
|
199
|
+
import("./worklets/capture-processor.js").then((m) => m.default),
|
|
200
|
+
import("./worklets/playback-processor.js").then((m) => m.default)
|
|
201
|
+
]).catch((err) => {
|
|
202
|
+
audioModulesPromise = null;
|
|
203
|
+
throw err;
|
|
204
|
+
});
|
|
205
|
+
return audioModulesPromise;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Open one audio path: load the worklets, ask for the microphone, and build
|
|
209
|
+
* the {@link VoiceIO} wired to `callbacks`.
|
|
210
|
+
*
|
|
211
|
+
* Rejects when any of the three refuses — a denied mic prompt being the
|
|
212
|
+
* ordinary case. The caller owns closing what this resolves: a path that comes
|
|
213
|
+
* up for a connection nobody wants any more is a live microphone, and no
|
|
214
|
+
* amount of cancellation upstream releases the device.
|
|
215
|
+
*/
|
|
216
|
+
async function openAudioPath(config, callbacks, preConnect) {
|
|
217
|
+
const pre = await preConnect ?? void 0;
|
|
218
|
+
try {
|
|
219
|
+
return await openVoiceIO(config, callbacks, pre);
|
|
220
|
+
} catch (err) {
|
|
221
|
+
pre?.close();
|
|
222
|
+
throw err;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
async function openVoiceIO(config, callbacks, pre) {
|
|
226
|
+
const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await loadAudioModules();
|
|
227
|
+
return createVoiceIO({
|
|
228
|
+
preConnect: pre,
|
|
229
|
+
onPreConnectAudio: callbacks.onPreConnectAudio,
|
|
230
|
+
sttSampleRate: config.sampleRate,
|
|
231
|
+
ttsSampleRate: config.ttsSampleRate,
|
|
232
|
+
captureWorkletSrc: captureWorklet,
|
|
233
|
+
playbackWorkletSrc: playbackWorklet,
|
|
234
|
+
onMicData: callbacks.onMicData,
|
|
235
|
+
onPlaybackProgress: callbacks.onProgress,
|
|
236
|
+
onError: (err) => callbacks.onFailure(err.message)
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
//#endregion
|
|
240
|
+
//#region src/session/types.ts
|
|
241
|
+
/**
|
|
242
|
+
* The two liveness fields at rest.
|
|
243
|
+
*
|
|
244
|
+
* `running` and `recording` ride with almost every state transition and were
|
|
245
|
+
* spread as a pair of literals at seven sites across three modules — the same
|
|
246
|
+
* shape `session/state.ts` folded `state` and `error` out of, one field
|
|
247
|
+
* short. Naming it relates them: a transition that ends the call says so once,
|
|
248
|
+
* and a reader looking for "what stops the mic" finds one symbol rather than a
|
|
249
|
+
* grep.
|
|
250
|
+
*
|
|
251
|
+
* It is deliberately NOT the whole snapshot patch — a transition still supplies
|
|
252
|
+
* its own `agentState.apply(...)` projection beside this.
|
|
253
|
+
*/
|
|
254
|
+
const STOPPED = {
|
|
255
|
+
running: false,
|
|
256
|
+
recording: false
|
|
257
|
+
};
|
|
258
|
+
/**
|
|
259
|
+
* A turn boundary: end the current turn and settle whatever it was playing.
|
|
260
|
+
*
|
|
261
|
+
* The two calls are one fact and were written out at four sites — `cancel()`
|
|
262
|
+
* and `reset()` here, `reply.cancelled` and `session.reset` on the server side
|
|
263
|
+
* — where the pair is load-bearing in both halves. The bump stops a stale drain
|
|
264
|
+
* continuation from stamping `"listening"` over a state the session has since
|
|
265
|
+
* moved to; the flush settles the interrupted turn's `done()` so it cannot
|
|
266
|
+
* strand.
|
|
267
|
+
*
|
|
268
|
+
* Two further sites bump WITHOUT flushing (`cleanupAudio`, a committed user
|
|
269
|
+
* transcript) and stay spelled out, which is the point of naming this one: a
|
|
270
|
+
* bump on its own now reads as a deliberate choice rather than a forgotten
|
|
271
|
+
* flush.
|
|
272
|
+
*/
|
|
273
|
+
function bargeIn(conn, audio) {
|
|
274
|
+
conn.turn.bump();
|
|
275
|
+
audio.flush();
|
|
276
|
+
}
|
|
277
|
+
//#endregion
|
|
278
|
+
//#region src/session/audio-effects.ts
|
|
279
|
+
/** Build the effects for one session's audio path. */
|
|
280
|
+
function createAudioEffects(deps) {
|
|
281
|
+
const { conn, updateState, agentState, sendJson, mic, preConnect } = deps;
|
|
282
|
+
/**
|
|
283
|
+
* Wait for `io`'s playback queue to drain, then go back to listening.
|
|
284
|
+
*
|
|
285
|
+
* Captures `conn.turn` so a completion (or failure) landing after a turn
|
|
286
|
+
* boundary — a barge-in, a committed user turn, an audio-path teardown — is
|
|
287
|
+
* discarded instead of overwriting the newer turn's, or a dead session's,
|
|
288
|
+
* state.
|
|
289
|
+
*/
|
|
290
|
+
function settleWhenDrained(io) {
|
|
291
|
+
const at = conn.turn.current();
|
|
292
|
+
io.done().then(() => {
|
|
293
|
+
if (!conn.turn.isCurrent(at)) return;
|
|
294
|
+
updateState(agentState.apply({ type: "LISTEN" }));
|
|
295
|
+
}).catch((err) => {
|
|
296
|
+
console.warn("Audio playback done failed:", err);
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
return {
|
|
300
|
+
open: (config, callbacks) => (deps.openAudioPath ?? openAudioPath)(config, callbacks, preConnect.take()),
|
|
301
|
+
sendMicAudio: (pcm16) => {
|
|
302
|
+
try {
|
|
303
|
+
mic.sendAudio(pcm16);
|
|
304
|
+
} catch {
|
|
305
|
+
console.debug("[aai-ui] sendAudio dropped: connection closed");
|
|
306
|
+
}
|
|
307
|
+
},
|
|
308
|
+
sendPreConnectAudio: (chunks) => {
|
|
309
|
+
try {
|
|
310
|
+
mic.sendBuffered(chunks);
|
|
311
|
+
} catch {
|
|
312
|
+
console.debug("[aai-ui] pre-connect audio dropped: connection closed");
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
reportProgress: (bufferedMs) => {
|
|
316
|
+
try {
|
|
317
|
+
sendJson({
|
|
318
|
+
type: "playback_progress",
|
|
319
|
+
bufferedMs
|
|
320
|
+
});
|
|
321
|
+
} catch {}
|
|
322
|
+
},
|
|
323
|
+
announceReady: () => sendJson({ type: "audio_ready" }),
|
|
324
|
+
micLive: () => updateState({ recording: true }),
|
|
325
|
+
listen: () => updateState(agentState.apply({ type: "LISTEN" })),
|
|
326
|
+
settleWhenDrained,
|
|
327
|
+
reportFailure: (message) => {
|
|
328
|
+
updateState({
|
|
329
|
+
...agentState.apply({
|
|
330
|
+
type: "FAILED",
|
|
331
|
+
error: {
|
|
332
|
+
code: "audio",
|
|
333
|
+
message,
|
|
334
|
+
fatal: false
|
|
335
|
+
}
|
|
336
|
+
}),
|
|
337
|
+
...STOPPED
|
|
338
|
+
});
|
|
339
|
+
},
|
|
340
|
+
endTurn: () => conn.turn.bump(),
|
|
341
|
+
release: (io) => {
|
|
342
|
+
io.close().catch(() => {});
|
|
343
|
+
}
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
//#endregion
|
|
347
|
+
//#region src/session/audio-state.ts
|
|
348
|
+
/**
|
|
349
|
+
* The browser session's AUDIO PATH — the mic grant, the worklets, and the
|
|
350
|
+
* {@link VoiceIO} they produce — as a statechart.
|
|
351
|
+
*
|
|
352
|
+
* It was a latch, an epoch counter and three buffers spread across
|
|
353
|
+
* `ConnState`, and every one of them was a question the position already
|
|
354
|
+
* answers:
|
|
355
|
+
*
|
|
356
|
+
* - **`audioSetupInFlight` was a dedup, and dedup is a state.**
|
|
357
|
+
* `initAudioCapture` opened with `if (conn.audioSetupInFlight) return` and
|
|
358
|
+
* closed with "only the init that still owns the flag may clear it", which is
|
|
359
|
+
* an ownership problem a boolean cannot express and a position does not have.
|
|
360
|
+
* Here `starting` simply does not handle `START`, and there is no flag to
|
|
361
|
+
* hand back.
|
|
362
|
+
* - **`conn.generation` existed for this module and nothing else.** Four
|
|
363
|
+
* `bump()` sites, four `isCurrent(gen)` reads, and its own field on
|
|
364
|
+
* `ConnState`. All four bumps sat immediately after a `cleanupAudio()` that
|
|
365
|
+
* had *just* cleared the in-flight flag — the reconnect close handler said so
|
|
366
|
+
* outright ("cleanupAudio just cleared the in-flight flag, so a survivor
|
|
367
|
+
* would otherwise pass the same-generation guard and double-run"). Two
|
|
368
|
+
* mechanisms for one question, the second one layered on because the first
|
|
369
|
+
* could not stop the work already running. A stopped `invoke` can, so the
|
|
370
|
+
* counter is gone.
|
|
371
|
+
* - **A counter cannot tell two bring-ups in ONE generation apart, and they
|
|
372
|
+
* happen.** A repeated `config` frame on a live connection re-opened the path
|
|
373
|
+
* without bumping anything, so the outgoing `VoiceIO`'s `onError` still
|
|
374
|
+
* passed `isCurrent(gen)` and could tear down the replacement that had just
|
|
375
|
+
* taken its slot. `PROGRESS` and `IO_FAILED` carry the instance that fired
|
|
376
|
+
* them now, and identity is exact where a counter was approximate.
|
|
377
|
+
* - **`preInitAudio` / `preInitDone` are only meaningful while the path is
|
|
378
|
+
* down.** As connection fields they were readable — and writable — from
|
|
379
|
+
* anywhere; as context of `down`/`starting` they are cleared by entering
|
|
380
|
+
* `down` rather than by three call sites remembering to.
|
|
381
|
+
*
|
|
382
|
+
* ## What cancellation does NOT do, and the one guard that had to survive
|
|
383
|
+
*
|
|
384
|
+
* Stopping the `bringUp` actor hides its resolution; it does not close a
|
|
385
|
+
* microphone the browser has already granted. So `bringUp` checks its own
|
|
386
|
+
* `signal` after `openAudioPath` settles and releases what it built. That is
|
|
387
|
+
* the single remaining "is this still wanted?" test, and it lives in the
|
|
388
|
+
* producer — which owns the value nobody adopted — rather than at the four
|
|
389
|
+
* consumer sites that used to re-ask.
|
|
390
|
+
*
|
|
391
|
+
* ## Effects stay in the session core
|
|
392
|
+
*
|
|
393
|
+
* The machine owns WHEN. Every HOW — the socket frame, the snapshot write, the
|
|
394
|
+
* turn-boundary drain — arrives as an injected {@link AudioPathEffects} call,
|
|
395
|
+
* the same shape `s2s-lifecycle.ts` uses. So this module touches no socket and
|
|
396
|
+
* no snapshot.
|
|
397
|
+
*/
|
|
398
|
+
/**
|
|
399
|
+
* Cap on audio chunks buffered while the path is coming up. ~100 chunks at
|
|
400
|
+
* typical S2S chunk sizes is well over a second of audio — far longer than a
|
|
401
|
+
* bring-up takes in practice, but bounded against pathological cases
|
|
402
|
+
* (mic-permission stalls).
|
|
403
|
+
*/
|
|
404
|
+
const MAX_BUFFERED_CHUNKS = 100;
|
|
405
|
+
/**
|
|
406
|
+
* The rejection a bring-up abandons itself with.
|
|
407
|
+
*
|
|
408
|
+
* Never observed: the actor is already stopped when it is thrown, and XState
|
|
409
|
+
* drops a settlement from a stopped actor. It exists so the `async` function
|
|
410
|
+
* has a way to end that is not "resolve with a VoiceIO I have just closed".
|
|
411
|
+
*/
|
|
412
|
+
const ABANDONED = /* @__PURE__ */ new Error("audio path abandoned");
|
|
413
|
+
const audioPathMachine = setup({
|
|
414
|
+
types: {},
|
|
415
|
+
actors: {
|
|
416
|
+
/**
|
|
417
|
+
* One bring-up. An actor rather than a fire-and-forget `void init(...)`
|
|
418
|
+
* precisely so that leaving `starting` — a hang-up, a reconnect, a fatal
|
|
419
|
+
* frame — stops it, instead of letting it settle into a connection that is
|
|
420
|
+
* already over.
|
|
421
|
+
*/
|
|
422
|
+
bringUp: fromPromise(async ({ input, signal }) => {
|
|
423
|
+
let mine = null;
|
|
424
|
+
const io = await input.effects.open(input.config, {
|
|
425
|
+
onMicData: (pcm16) => input.effects.sendMicAudio(pcm16),
|
|
426
|
+
onPreConnectAudio: (chunks) => input.effects.sendPreConnectAudio(chunks),
|
|
427
|
+
onProgress: (bufferedMs) => {
|
|
428
|
+
if (mine) input.send({
|
|
429
|
+
type: "PROGRESS",
|
|
430
|
+
io: mine,
|
|
431
|
+
bufferedMs
|
|
432
|
+
});
|
|
433
|
+
},
|
|
434
|
+
onFailure: (message) => {
|
|
435
|
+
if (mine) input.send({
|
|
436
|
+
type: "IO_FAILED",
|
|
437
|
+
io: mine,
|
|
438
|
+
message
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
if (signal.aborted) {
|
|
443
|
+
input.effects.release(io);
|
|
444
|
+
throw ABANDONED;
|
|
445
|
+
}
|
|
446
|
+
mine = io;
|
|
447
|
+
return io;
|
|
448
|
+
}) },
|
|
449
|
+
guards: {
|
|
450
|
+
/**
|
|
451
|
+
* Did the live path report this, or one it replaced?
|
|
452
|
+
*
|
|
453
|
+
* Identity rather than `generation.isCurrent(gen)`: two bring-ups inside a
|
|
454
|
+
* single connection share a generation, and the counter passed for the
|
|
455
|
+
* instance that had just been closed.
|
|
456
|
+
*/
|
|
457
|
+
fromLiveIo: ({ context, event }) => (event.type === "PROGRESS" || event.type === "IO_FAILED") && context.io === event.io },
|
|
458
|
+
actions: {
|
|
459
|
+
rememberConfig: assign({ config: ({ context, event }) => event.type === "START" ? event.config : context.config }),
|
|
460
|
+
/** Hold a chunk that arrived too early, up to the cap. */
|
|
461
|
+
buffer: assign({ pending: ({ context, event }) => event.type === "CHUNK" && context.pending.length < MAX_BUFFERED_CHUNKS ? [...context.pending, event.chunk] : context.pending }),
|
|
462
|
+
/** Record a done that arrived too early. */
|
|
463
|
+
rememberDone: assign({ pendingDone: true }),
|
|
464
|
+
/** Answer it optimistically: with no audio pipeline there is nothing to wait for. */
|
|
465
|
+
listenNow: ({ context }) => context.effects.listen(),
|
|
466
|
+
/** Nothing buffered belongs to the next path. */
|
|
467
|
+
forgetBuffered: assign({
|
|
468
|
+
io: null,
|
|
469
|
+
pending: [],
|
|
470
|
+
pendingDone: false
|
|
471
|
+
}),
|
|
472
|
+
/** Releasing the path ends the turn it was playing — see `endTurn`. */
|
|
473
|
+
endTurn: ({ context }) => context.effects.endTurn(),
|
|
474
|
+
releaseIo: ({ context }) => {
|
|
475
|
+
if (context.io) context.effects.release(context.io);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}).createMachine({
|
|
479
|
+
id: "audioPath",
|
|
480
|
+
context: ({ input }) => ({
|
|
481
|
+
effects: input,
|
|
482
|
+
io: null,
|
|
483
|
+
pending: [],
|
|
484
|
+
pendingDone: false,
|
|
485
|
+
config: null
|
|
486
|
+
}),
|
|
487
|
+
initial: "down",
|
|
488
|
+
on: {
|
|
489
|
+
CHUNK: { actions: "buffer" },
|
|
490
|
+
DONE: { actions: ["rememberDone", "listenNow"] },
|
|
491
|
+
TEARDOWN: ".down"
|
|
492
|
+
},
|
|
493
|
+
states: {
|
|
494
|
+
/**
|
|
495
|
+
* No audio path. Entered on every release, so its entry is the one place
|
|
496
|
+
* that ends the turn and drops what was buffered for a path that is gone.
|
|
497
|
+
*/
|
|
498
|
+
down: {
|
|
499
|
+
entry: ["endTurn", "forgetBuffered"],
|
|
500
|
+
on: { START: {
|
|
501
|
+
target: "starting",
|
|
502
|
+
actions: "rememberConfig"
|
|
503
|
+
} }
|
|
504
|
+
},
|
|
505
|
+
/**
|
|
506
|
+
* A bring-up is in flight.
|
|
507
|
+
*
|
|
508
|
+
* `START` is absent deliberately: that is the whole of what
|
|
509
|
+
* `audioSetupInFlight` said, and a second `config` frame arriving mid-grant
|
|
510
|
+
* is now declined by the position rather than by a flag whose owner had to
|
|
511
|
+
* be tracked.
|
|
512
|
+
*/
|
|
513
|
+
starting: { invoke: {
|
|
514
|
+
src: "bringUp",
|
|
515
|
+
input: ({ context, self }) => ({
|
|
516
|
+
effects: context.effects,
|
|
517
|
+
config: context.config ?? {
|
|
518
|
+
sampleRate: 0,
|
|
519
|
+
ttsSampleRate: 0
|
|
520
|
+
},
|
|
521
|
+
send: (event) => {
|
|
522
|
+
self.send(event);
|
|
523
|
+
}
|
|
524
|
+
}),
|
|
525
|
+
onDone: {
|
|
526
|
+
target: "up",
|
|
527
|
+
actions: [
|
|
528
|
+
({ context, event }) => {
|
|
529
|
+
for (const chunk of context.pending) event.output.enqueue(chunk);
|
|
530
|
+
},
|
|
531
|
+
assign({
|
|
532
|
+
io: ({ event }) => event.output,
|
|
533
|
+
pending: []
|
|
534
|
+
}),
|
|
535
|
+
({ context, event }) => {
|
|
536
|
+
context.effects.announceReady();
|
|
537
|
+
context.effects.micLive();
|
|
538
|
+
if (context.pendingDone) context.effects.settleWhenDrained(event.output);
|
|
539
|
+
else context.effects.listen();
|
|
540
|
+
},
|
|
541
|
+
assign({ pendingDone: false })
|
|
542
|
+
]
|
|
543
|
+
},
|
|
544
|
+
onError: {
|
|
545
|
+
target: "down",
|
|
546
|
+
actions: ({ context, event }) => {
|
|
547
|
+
context.effects.reportFailure(`Microphone access failed: ${errorMessage(event.error)}`);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
} },
|
|
551
|
+
/**
|
|
552
|
+
* A live path. Its exit releases it, which covers all three ways out —
|
|
553
|
+
* teardown, a worklet death, and the repeated `config` frame that opens a
|
|
554
|
+
* replacement (the "defensive" close that used to sit inside the bring-up,
|
|
555
|
+
* where it could only run if the bring-up got that far).
|
|
556
|
+
*/
|
|
557
|
+
up: {
|
|
558
|
+
exit: "releaseIo",
|
|
559
|
+
on: {
|
|
560
|
+
START: {
|
|
561
|
+
target: "starting",
|
|
562
|
+
actions: "rememberConfig"
|
|
563
|
+
},
|
|
564
|
+
CHUNK: { actions: ({ context, event }) => context.io?.enqueue(event.chunk) },
|
|
565
|
+
DONE: { actions: ({ context }) => {
|
|
566
|
+
if (context.io) context.effects.settleWhenDrained(context.io);
|
|
567
|
+
} },
|
|
568
|
+
PROGRESS: {
|
|
569
|
+
guard: "fromLiveIo",
|
|
570
|
+
actions: ({ context, event }) => context.effects.reportProgress(event.bufferedMs)
|
|
571
|
+
},
|
|
572
|
+
IO_FAILED: {
|
|
573
|
+
guard: "fromLiveIo",
|
|
574
|
+
target: "down",
|
|
575
|
+
actions: ({ context, event }) => context.effects.reportFailure(event.message)
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
});
|
|
581
|
+
/** Create the audio path for one browser session. */
|
|
582
|
+
function createAudioPath(effects) {
|
|
583
|
+
const actor = createActor(audioPathMachine, { input: effects }).start();
|
|
584
|
+
return {
|
|
585
|
+
phase: () => actor.getSnapshot().value,
|
|
586
|
+
start: (config) => {
|
|
587
|
+
actor.send({
|
|
588
|
+
type: "START",
|
|
589
|
+
config
|
|
590
|
+
});
|
|
591
|
+
},
|
|
592
|
+
enqueue: (chunk) => {
|
|
593
|
+
actor.send({
|
|
594
|
+
type: "CHUNK",
|
|
595
|
+
chunk
|
|
596
|
+
});
|
|
597
|
+
},
|
|
598
|
+
done: () => {
|
|
599
|
+
actor.send({ type: "DONE" });
|
|
600
|
+
},
|
|
601
|
+
flush: () => actor.getSnapshot().context.io?.flush(),
|
|
602
|
+
teardown: () => {
|
|
603
|
+
actor.send({ type: "TEARDOWN" });
|
|
604
|
+
}
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
//#endregion
|
|
608
|
+
//#region src/session/close.ts
|
|
609
|
+
/**
|
|
610
|
+
* What a socket's CLOSE means to the caller.
|
|
611
|
+
*
|
|
612
|
+
* One decision, in its own module because `session/browser-session.ts` is at its length cap
|
|
613
|
+
* and because the decision is worth reading on its own: which endings are
|
|
614
|
+
* ordinary, and which carry something the user has to be told.
|
|
615
|
+
*
|
|
616
|
+
* @module
|
|
617
|
+
*/
|
|
618
|
+
/**
|
|
619
|
+
* The one close code whose reason is never worth showing.
|
|
620
|
+
*
|
|
621
|
+
* `1000` is a deliberate, successful close — the caller hanging up, or the
|
|
622
|
+
* server retiring a session it finished with. A browser reports a socket that
|
|
623
|
+
* closed with no status as `1005`, which carries no reason either, so testing
|
|
624
|
+
* for text is enough to exclude it.
|
|
625
|
+
*/
|
|
626
|
+
const NORMAL_CLOSE = 1e3;
|
|
627
|
+
/**
|
|
628
|
+
* What to REPORT for a socket that closed for good, or null for an ordinary end.
|
|
629
|
+
*
|
|
630
|
+
* A sentence the peer wrote beats one this file invented. A guest that cannot
|
|
631
|
+
* build its runtime closes 1011 with the remedy already in it — "Anthropic LLM:
|
|
632
|
+
* missing API key. Set ANTHROPIC_API_KEY in the agent env." — and this handler
|
|
633
|
+
* used to discard `event.reason` entirely, so the one thing a misconfigured
|
|
634
|
+
* deployment needed to be told surfaced as "WebSocket connection error", or as a
|
|
635
|
+
* plain disconnect with nothing at all. Measured against a deployed agent with no
|
|
636
|
+
* provider key.
|
|
637
|
+
*
|
|
638
|
+
* A NORMAL close carries no reason worth showing (1000 is the caller hanging up
|
|
639
|
+
* or the server retiring a finished session; a browser reports a status-less
|
|
640
|
+
* close as 1005, which has no reason either), so the two ordinary endings are
|
|
641
|
+
* untouched.
|
|
642
|
+
*/
|
|
643
|
+
function closeFailure(event, socketErrored) {
|
|
644
|
+
const refusal = event.reason.trim();
|
|
645
|
+
if (refusal !== "" && event.code !== NORMAL_CLOSE) return refusal;
|
|
646
|
+
return socketErrored ? "WebSocket connection error" : null;
|
|
647
|
+
}
|
|
648
|
+
//#endregion
|
|
649
|
+
//#region src/session/reconnect.ts
|
|
650
|
+
/**
|
|
651
|
+
* Automatic reconnection for the browser session socket, built on
|
|
652
|
+
* partysocket's `ReconnectingWebSocket`. Kept out of `session/browser-session.ts` so
|
|
653
|
+
* the state machine there reads as protocol logic, not socket plumbing.
|
|
654
|
+
*/
|
|
655
|
+
/**
|
|
656
|
+
* Backoff for automatic reconnects after an unexpected close (partysocket):
|
|
657
|
+
* exponential from 1s, capped at 15s, giving up after 10 attempts. Applies
|
|
658
|
+
* only to the default socket implementation — an injected
|
|
659
|
+
* `options.WebSocket` (tests) never reconnects on its own.
|
|
660
|
+
*/
|
|
661
|
+
const RECONNECT_OPTIONS = {
|
|
662
|
+
minReconnectionDelay: 1e3,
|
|
663
|
+
maxReconnectionDelay: 15e3,
|
|
664
|
+
reconnectionDelayGrowFactor: 2,
|
|
665
|
+
maxRetries: 10
|
|
666
|
+
};
|
|
667
|
+
/**
|
|
668
|
+
* Open partysocket's reconnecting WebSocket. The URL and the subprotocols are
|
|
669
|
+
* *providers*, re-evaluated on every attempt (async supported), so each retry
|
|
670
|
+
* picks up the current broker-named endpoint, resume URL and session ticket
|
|
671
|
+
* rather than the ones the session started with. partysocket calls the URL
|
|
672
|
+
* provider first and the protocols provider straight after it, in the same
|
|
673
|
+
* tick, for each attempt.
|
|
674
|
+
*/
|
|
675
|
+
function openReconnectingSocket(urlProvider, protocolsProvider) {
|
|
676
|
+
return new ReconnectingWebSocket(urlProvider, protocolsProvider, RECONNECT_OPTIONS);
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Force a fresh connection attempt on a reconnecting socket, reporting
|
|
680
|
+
* whether it was one. Used for a failure partysocket cannot see: a socket it
|
|
681
|
+
* considers open and healthy, whose peer never completed OUR handshake.
|
|
682
|
+
* Returns false for an injected `options.WebSocket` (tests), which has no
|
|
683
|
+
* reconnect machinery — the caller then treats the failure as terminal.
|
|
684
|
+
*/
|
|
685
|
+
function forceReconnect(socket) {
|
|
686
|
+
if (!(socket instanceof ReconnectingWebSocket)) return false;
|
|
687
|
+
socket.reconnect();
|
|
688
|
+
return true;
|
|
689
|
+
}
|
|
690
|
+
/**
|
|
691
|
+
* True while `socket` is a reconnecting socket that will retry after the
|
|
692
|
+
* close event currently being handled. partysocket schedules the retry
|
|
693
|
+
* *before* dispatching `close`, so `retryCount` already names the attempt
|
|
694
|
+
* just scheduled — at `maxRetries` it has given up.
|
|
695
|
+
*/
|
|
696
|
+
function reconnectPending(socket) {
|
|
697
|
+
return socket instanceof ReconnectingWebSocket && socket.shouldReconnect && socket.retryCount < RECONNECT_OPTIONS.maxRetries;
|
|
698
|
+
}
|
|
699
|
+
//#endregion
|
|
700
|
+
//#region src/session/resume-store.ts
|
|
701
|
+
/**
|
|
702
|
+
* Where a session id survives a page RELOAD.
|
|
703
|
+
*
|
|
704
|
+
* The id is what `?sessionId=` presents on reconnect, and it is the key the
|
|
705
|
+
* agent's slot state and event log live under — so a reload that cannot produce
|
|
706
|
+
* it starts a brand-new session, and a UI driven by `useAgentState` comes back
|
|
707
|
+
* empty even though the agent still holds the cart. The server side of the
|
|
708
|
+
* reconstitution was already built (`pushStateSnapshot` force-pushes the
|
|
709
|
+
* projection after hydration on every start, `state.updated` lands in
|
|
710
|
+
* `agentState`); what was missing is that nothing in the browser remembered the
|
|
711
|
+
* id across a reload. `onSessionId`/`resumeSessionId` let a client wire it by
|
|
712
|
+
* hand and exactly one of fourteen templates did, which is the shape of a
|
|
713
|
+
* default in the wrong place.
|
|
714
|
+
*
|
|
715
|
+
* **`sessionStorage`, deliberately, and this is the opposite call from the
|
|
716
|
+
* studio's session token.** A reload and a same-tab navigation survive it; a new
|
|
717
|
+
* tab and a visit tomorrow do not, which is what we want here rather than a
|
|
718
|
+
* limitation: presenting a day-old id suppresses the greeting
|
|
719
|
+
* (`parseWsUpgradeParams` keys that off the id's mere presence) and rejoins a
|
|
720
|
+
* conversation whose context is long gone. The studio token is a credential
|
|
721
|
+
* whose value is not being asked to sign out; this is a pointer into a live call.
|
|
722
|
+
*
|
|
723
|
+
* Keyed by the agent's own URL, so two agents served from one origin — which is
|
|
724
|
+
* every deployed agent, at `/:slug/` — cannot inherit each other's session.
|
|
725
|
+
*
|
|
726
|
+
* Every access is guarded, and the guard lives in `_web-storage.ts`: a session
|
|
727
|
+
* that cannot be remembered must degrade to today's behaviour rather than
|
|
728
|
+
* failing to start.
|
|
729
|
+
*/
|
|
730
|
+
const PREFIX = "aai:session:";
|
|
731
|
+
/** One agent's slot in storage. */
|
|
732
|
+
function keyFor(platformUrl) {
|
|
733
|
+
return urlSlot(PREFIX, platformUrl);
|
|
734
|
+
}
|
|
735
|
+
/** The stored session id for this agent, or undefined. @internal */
|
|
736
|
+
function readStoredSessionId(platformUrl) {
|
|
737
|
+
return storageGet("session", keyFor(platformUrl));
|
|
738
|
+
}
|
|
739
|
+
/** Remember this agent's session id for the next load. @internal */
|
|
740
|
+
function writeStoredSessionId(platformUrl, sessionId) {
|
|
741
|
+
storageSet("session", keyFor(platformUrl), sessionId);
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Forget it, so the next load is a NEW session.
|
|
745
|
+
*
|
|
746
|
+
* Called from `end()`, which is the clear-and-forget the "New Conversation"
|
|
747
|
+
* button runs: leaving the id behind there would have the next load rejoin the
|
|
748
|
+
* conversation the user just discarded, greeting suppressed.
|
|
749
|
+
*
|
|
750
|
+
* @internal
|
|
751
|
+
*/
|
|
752
|
+
function clearStoredSessionId(platformUrl) {
|
|
753
|
+
storageRemove("session", keyFor(platformUrl));
|
|
754
|
+
}
|
|
755
|
+
const TICKET_PREFIX = "aai:session-ticket:";
|
|
756
|
+
/**
|
|
757
|
+
* The last session ticket this agent's `client-config` issued — beside the id
|
|
758
|
+
* and for its reason: on the managed platform a ticket is BOUND to its session,
|
|
759
|
+
* and presenting it (`SESSION_TICKET_HEADER`) is what makes the broker re-mint
|
|
760
|
+
* for that session rather than a new one, so a reload that lost it cannot
|
|
761
|
+
* resume. As sensitive as the id it stands beside, and stored the same way.
|
|
762
|
+
*
|
|
763
|
+
* @internal
|
|
764
|
+
*/
|
|
765
|
+
function readStoredTicket(platformUrl) {
|
|
766
|
+
return storageGet("session", urlSlot(TICKET_PREFIX, platformUrl));
|
|
767
|
+
}
|
|
768
|
+
/** Remember the last server-issued ticket. @internal */
|
|
769
|
+
function writeStoredTicket(platformUrl, ticket) {
|
|
770
|
+
storageSet("session", urlSlot(TICKET_PREFIX, platformUrl), ticket);
|
|
771
|
+
}
|
|
772
|
+
/** Forget it, with the session it was for. @internal */
|
|
773
|
+
function clearStoredTicket(platformUrl) {
|
|
774
|
+
storageRemove("session", urlSlot(TICKET_PREFIX, platformUrl));
|
|
775
|
+
}
|
|
776
|
+
//#endregion
|
|
777
|
+
//#region src/session/ticket.ts
|
|
778
|
+
/**
|
|
779
|
+
* The session ticket one connection attempt presents — `VoiceSessionOptions.token`,
|
|
780
|
+
* else the `sessionToken` a server's `client-config` issued.
|
|
781
|
+
*
|
|
782
|
+
* A browser cannot set headers on a WebSocket, so the ticket rides the
|
|
783
|
+
* `Sec-WebSocket-Protocol` list as `aai.auth.<ticket>`, offered AFTER the plain
|
|
784
|
+
* `aai.session` protocol: a browser fails any handshake whose response selects
|
|
785
|
+
* none of its offers, and the server (`selectSessionProtocol` in
|
|
786
|
+
* `@alexkroman1/aai-runtime/auth`) selects the plain one so the ticket is never
|
|
787
|
+
* echoed back. A server older than tickets selects the first offer, which is
|
|
788
|
+
* the plain one too, so offering a ticket never fails a handshake.
|
|
789
|
+
*
|
|
790
|
+
* A value that is not a valid subprotocol token (RFC 6455 §4.1 — an HTTP
|
|
791
|
+
* token) would make `new WebSocket` throw, so it travels as `?token=` instead,
|
|
792
|
+
* the server's fallback. A minted ticket is base64url with one dot, and so is a
|
|
793
|
+
* JWT: neither takes that path.
|
|
794
|
+
*/
|
|
795
|
+
const HTTP_TOKEN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/;
|
|
796
|
+
function trimmed(value) {
|
|
797
|
+
const t = value?.trim();
|
|
798
|
+
return t === "" ? void 0 : t;
|
|
799
|
+
}
|
|
800
|
+
/**
|
|
801
|
+
* Ask the `token` option for THIS attempt's ticket. A getter that throws or
|
|
802
|
+
* rejects yields none — the attempt still dials, and a server that requires a
|
|
803
|
+
* ticket refuses it with a reason the session reports, rather than the attempt
|
|
804
|
+
* failing before a socket exists (partysocket reports a provider failure as an
|
|
805
|
+
* `error` with no `close`, which would leave the session on "connecting").
|
|
806
|
+
*/
|
|
807
|
+
async function resolveSessionToken(option, attempt) {
|
|
808
|
+
if (typeof option !== "function") return trimmed(option);
|
|
809
|
+
try {
|
|
810
|
+
return trimmed(await option(attempt));
|
|
811
|
+
} catch (err) {
|
|
812
|
+
console.warn("session-core: the `token` getter failed; dialing without a ticket", err);
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
/**
|
|
817
|
+
* The same, for an injected `WebSocket` constructor: that path opens its socket
|
|
818
|
+
* synchronously, so an asynchronous getter cannot be awaited there.
|
|
819
|
+
*/
|
|
820
|
+
function resolveSessionTokenSync(option, attempt) {
|
|
821
|
+
if (typeof option !== "function") return trimmed(option);
|
|
822
|
+
const value = option(attempt);
|
|
823
|
+
if (typeof value !== "string" && value !== void 0) throw new TypeError("VoiceSessionOptions.token: an injected `WebSocket` opens synchronously, so its `token` getter must return a string, not a Promise");
|
|
824
|
+
return trimmed(value);
|
|
825
|
+
}
|
|
826
|
+
/** How `token` travels on one attempt; nothing at all when there is no ticket. */
|
|
827
|
+
function ticketCarriage(token) {
|
|
828
|
+
if (token === void 0) return {};
|
|
829
|
+
const protocol = `${SESSION_AUTH_PROTOCOL_PREFIX}${token}`;
|
|
830
|
+
if (!HTTP_TOKEN.test(protocol)) return { queryToken: token };
|
|
831
|
+
return { protocols: [SESSION_PROTOCOL, protocol] };
|
|
832
|
+
}
|
|
833
|
+
//#endregion
|
|
834
|
+
//#region src/session/url.ts
|
|
835
|
+
/** Build the session WebSocket URL from the platform URL and resume state. */
|
|
836
|
+
function buildWsUrl(platformUrl, sessionId, report = {}) {
|
|
837
|
+
return applyResumeParams(buildAgentUrl(platformUrl, "websocket"), sessionId, report);
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Turn a broker-provided session URL (`sessionUrl` from `GET client-config`
|
|
841
|
+
* — the agent's live sandbox endpoint) into this attempt's connect URL.
|
|
842
|
+
*/
|
|
843
|
+
function buildBrokeredWsUrl(sessionUrl, sessionId, report = {}) {
|
|
844
|
+
return applyResumeParams(new URL(sessionUrl), sessionId, report);
|
|
845
|
+
}
|
|
846
|
+
const WS_PROTOCOLS = {
|
|
847
|
+
"https:": "wss:",
|
|
848
|
+
"http:": "ws:"
|
|
849
|
+
};
|
|
850
|
+
function applyResumeParams(wsUrl, sessionId, report) {
|
|
851
|
+
wsUrl.protocol = WS_PROTOCOLS[wsUrl.protocol] ?? wsUrl.protocol;
|
|
852
|
+
if (sessionId) wsUrl.searchParams.set("sessionId", sessionId);
|
|
853
|
+
if (report.location) wsUrl.searchParams.set("location", report.location);
|
|
854
|
+
if (report.phone) wsUrl.searchParams.set("phone", report.phone);
|
|
855
|
+
if (report.client) wsUrl.searchParams.set("client", report.client);
|
|
856
|
+
return wsUrl;
|
|
857
|
+
}
|
|
858
|
+
//#endregion
|
|
859
|
+
//#region src/session/dial.ts
|
|
860
|
+
/**
|
|
861
|
+
* How the next connection attempt is DIALLED, and the resume identity it dials
|
|
862
|
+
* with.
|
|
863
|
+
*
|
|
864
|
+
* Split out of `session/browser-session.ts` at the 500-line cap, along the seam that file
|
|
865
|
+
* already established when it moved socket plumbing into
|
|
866
|
+
* `session/reconnect.ts`: the state machine there reads as protocol logic,
|
|
867
|
+
* and this is the address it sends it to. What makes it one module rather than
|
|
868
|
+
* three extracted functions is that the three pieces of mutable state involved —
|
|
869
|
+
* the session id, whether this connection has ever completed a handshake, and
|
|
870
|
+
* whether the server is a broker — are read by nothing else in the core, and
|
|
871
|
+
* every one of them is only meaningful in the sentence "the URL for the next
|
|
872
|
+
* attempt".
|
|
873
|
+
*/
|
|
874
|
+
/** `url` with the `?token=` fallback applied when the ticket rides the URL. */
|
|
875
|
+
function withQueryToken(url, carriage) {
|
|
876
|
+
if (carriage.queryToken !== void 0) url.searchParams.set("token", carriage.queryToken);
|
|
877
|
+
return url.toString();
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* @param onSessionId - Told whenever the CONFIRMED id changes (a `config` frame
|
|
881
|
+
* with a new id, `forget()`), so the session can notify `useSessionId()`
|
|
882
|
+
* even when no snapshot field moved with it.
|
|
883
|
+
* @internal
|
|
884
|
+
*/
|
|
885
|
+
function createDialer(options, onSessionId) {
|
|
886
|
+
/**
|
|
887
|
+
* The session ID to resume: seeded from `options.resumeSessionId`, else from
|
|
888
|
+
* what a previous LOAD of this page stored, then kept current from every
|
|
889
|
+
* `config` frame. Reconnect URLs carry it as `?sessionId=<id>` so the server
|
|
890
|
+
* re-registers the SAME session id — that key is what the session's slot state
|
|
891
|
+
* and event log live under, so an attempt that omits it gets a fresh session
|
|
892
|
+
* with none of the agent's context.
|
|
893
|
+
*
|
|
894
|
+
* Reading it from storage is what makes a page RELOAD resume, and so what makes
|
|
895
|
+
* the server's `syncState` push reach a UI that would otherwise come back
|
|
896
|
+
* empty. See `session/resume-store.ts`.
|
|
897
|
+
*/
|
|
898
|
+
let sessionId = options.resumeSessionId ?? readStoredSessionId(options.platformUrl);
|
|
899
|
+
/**
|
|
900
|
+
* Whether `platformUrl`'s `client-config` says anything per ATTEMPT: a
|
|
901
|
+
* `sessionUrl` (a broker) or a `sessionToken` (a server minting its own
|
|
902
|
+
* client's tickets — `aai dev` with `AAI_SESSION_SECRET`). A server does or it
|
|
903
|
+
* doesn't — it never flips mid-session — so once one that says neither is
|
|
904
|
+
* observed, later reconnects skip the `client-config` re-fetch that would only
|
|
905
|
+
* fall through to `buildWsUrl` (every reconnect on `aai dev` / self-hosted
|
|
906
|
+
* otherwise pays a wasted GET). `undefined` until the first fetch settles.
|
|
907
|
+
*/
|
|
908
|
+
let configPerAttempt;
|
|
909
|
+
/**
|
|
910
|
+
* The last ticket this server's `client-config` issued, presented on the next
|
|
911
|
+
* lookup that RESUMES (`SESSION_TICKET_HEADER`). On the managed platform a
|
|
912
|
+
* ticket is bound to its session and possession is the resume credential, so
|
|
913
|
+
* it is stored beside the id: a reload that lost it would start over.
|
|
914
|
+
*/
|
|
915
|
+
let serverTicket = readStoredTicket(options.platformUrl);
|
|
916
|
+
/** What the last `config` frame said — see `Dialer.sessionId`. */
|
|
917
|
+
let confirmed;
|
|
918
|
+
function confirm(sid) {
|
|
919
|
+
if (sid === confirmed) return;
|
|
920
|
+
confirmed = sid;
|
|
921
|
+
onSessionId?.();
|
|
922
|
+
}
|
|
923
|
+
/**
|
|
924
|
+
* This attempt's `?location=`, `?phone=` and `?client=`, resolved NOW: a
|
|
925
|
+
* getter is asked on every attempt so a UI that changes one (a settings
|
|
926
|
+
* field) is heard on the next reconnect without a remount. Trimmed; an empty
|
|
927
|
+
* answer is none.
|
|
928
|
+
*/
|
|
929
|
+
function report() {
|
|
930
|
+
return {
|
|
931
|
+
location: resolveReported(options.location),
|
|
932
|
+
phone: resolveReported(options.phone),
|
|
933
|
+
client: resolveReported(options.client)
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* The WebSocket URL and subprotocols for the *next* connection attempt.
|
|
938
|
+
* Evaluated per attempt (partysocket takes async URL and protocol providers):
|
|
939
|
+
*
|
|
940
|
+
* - `GET client-config` is re-fetched every attempt. When it names a
|
|
941
|
+
* `sessionUrl` — the platform's broker pointing at the agent's live sandbox
|
|
942
|
+
* — the session connects DIRECTLY there. The URL changes when the sandbox is
|
|
943
|
+
* replaced (idle eviction, redeploy), which is exactly when a reconnect
|
|
944
|
+
* happens, so per-attempt brokering is what makes reconnects land on the
|
|
945
|
+
* replacement. Without one (`aai dev`, older servers), the same-origin
|
|
946
|
+
* `websocket` path is used.
|
|
947
|
+
* - Once the first `config` arrives, every reconnect carries `?sessionId=<id>`
|
|
948
|
+
* and the server resumes the SAME session (id, tool state) instead of minting
|
|
949
|
+
* a new one.
|
|
950
|
+
* - The session ticket (`session/ticket.ts`) is the `token` option's,
|
|
951
|
+
* asked for THIS attempt, else the `sessionToken` this attempt's
|
|
952
|
+
* `client-config` issued — to a lookup that presented the last one when
|
|
953
|
+
* the attempt resumes, so a broker binding tickets to sessions re-mints for
|
|
954
|
+
* the same session. A ticket bound to a session opens THAT session, so if
|
|
955
|
+
* the broker could not re-mint, the server starts a new one and its
|
|
956
|
+
* `config` frame says which.
|
|
957
|
+
*/
|
|
958
|
+
async function resolveAttempt() {
|
|
959
|
+
const ownToken = resolveSessionToken(options.token, { sessionId });
|
|
960
|
+
const presented = sessionId === void 0 ? void 0 : serverTicket;
|
|
961
|
+
const cfg = configPerAttempt === false ? null : await loadClientConfig(options.platformUrl, void 0, presented);
|
|
962
|
+
if (cfg) configPerAttempt = cfg.sessionUrl !== void 0 || cfg.sessionToken !== void 0;
|
|
963
|
+
const own = await ownToken;
|
|
964
|
+
if (own === void 0 && cfg?.sessionToken !== void 0) {
|
|
965
|
+
serverTicket = cfg.sessionToken;
|
|
966
|
+
writeStoredTicket(options.platformUrl, serverTicket);
|
|
967
|
+
}
|
|
968
|
+
const carriage = ticketCarriage(own ?? cfg?.sessionToken);
|
|
969
|
+
return {
|
|
970
|
+
url: withQueryToken(cfg?.sessionUrl ? buildBrokeredWsUrl(cfg.sessionUrl, sessionId, report()) : buildWsUrl(options.platformUrl, sessionId, report()), carriage),
|
|
971
|
+
protocols: carriage.protocols
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* The attempt partysocket is dialling. It calls its URL and protocol providers
|
|
976
|
+
* back to back for each attempt and awaits them together, so the URL provider
|
|
977
|
+
* STARTS the attempt and the protocol provider reads that same one: a ticket
|
|
978
|
+
* and the address it was fetched beside belong to one attempt.
|
|
979
|
+
*/
|
|
980
|
+
let current;
|
|
981
|
+
return {
|
|
982
|
+
open: () => {
|
|
983
|
+
if (options.WebSocket) {
|
|
984
|
+
const carriage = ticketCarriage(resolveSessionTokenSync(options.token, { sessionId }));
|
|
985
|
+
const target = withQueryToken(buildWsUrl(options.platformUrl, sessionId, report()), carriage);
|
|
986
|
+
return carriage.protocols ? new options.WebSocket(target, carriage.protocols) : new options.WebSocket(target);
|
|
987
|
+
}
|
|
988
|
+
return openReconnectingSocket(async () => {
|
|
989
|
+
current = resolveAttempt();
|
|
990
|
+
return (await current).url;
|
|
991
|
+
}, async () => (await (current ?? resolveAttempt())).protocols ?? null);
|
|
992
|
+
},
|
|
993
|
+
configured: (sid) => {
|
|
994
|
+
sessionId = sid;
|
|
995
|
+
writeStoredSessionId(options.platformUrl, sid);
|
|
996
|
+
confirm(sid);
|
|
997
|
+
},
|
|
998
|
+
adopt: (sid) => {
|
|
999
|
+
sessionId = sid;
|
|
1000
|
+
writeStoredSessionId(options.platformUrl, sid);
|
|
1001
|
+
},
|
|
1002
|
+
forget: () => {
|
|
1003
|
+
sessionId = void 0;
|
|
1004
|
+
clearStoredSessionId(options.platformUrl);
|
|
1005
|
+
serverTicket = void 0;
|
|
1006
|
+
clearStoredTicket(options.platformUrl);
|
|
1007
|
+
confirm(void 0);
|
|
1008
|
+
},
|
|
1009
|
+
sessionId: () => confirmed,
|
|
1010
|
+
ticket: () => {
|
|
1011
|
+
if (options.token !== void 0) return typeof options.token === "function" ? resolveSessionToken(options.token, { sessionId: void 0 }) : resolveSessionTokenSync(options.token, { sessionId: void 0 });
|
|
1012
|
+
if (configPerAttempt === false) return;
|
|
1013
|
+
return loadClientConfig(options.platformUrl).then((cfg) => {
|
|
1014
|
+
if (cfg) configPerAttempt = cfg.sessionUrl !== void 0 || cfg.sessionToken !== void 0;
|
|
1015
|
+
return cfg?.sessionToken;
|
|
1016
|
+
});
|
|
1017
|
+
}
|
|
1018
|
+
};
|
|
1019
|
+
}
|
|
1020
|
+
//#endregion
|
|
1021
|
+
//#region src/session/handshake.ts
|
|
1022
|
+
/** What the session reports once the budget below is spent. */
|
|
1023
|
+
const HANDSHAKE_ERROR = {
|
|
1024
|
+
code: "connection",
|
|
1025
|
+
message: "Agent did not complete the session handshake",
|
|
1026
|
+
fatal: false
|
|
1027
|
+
};
|
|
1028
|
+
/** How long an OPEN socket may go without a `config` frame. */
|
|
1029
|
+
const HANDSHAKE_TIMEOUT_MS = 1e4;
|
|
1030
|
+
/**
|
|
1031
|
+
* How many consecutive handshake timeouts to ride out before giving up.
|
|
1032
|
+
*
|
|
1033
|
+
* `forceReconnect` restarts partysocket's own retry budget, so this is the
|
|
1034
|
+
* budget for this failure mode — without one, a permanently wedged peer would
|
|
1035
|
+
* be re-dialed every ~10s forever, which is the unbounded retry loop
|
|
1036
|
+
* `RECONNECT_OPTIONS.maxRetries` exists to prevent.
|
|
1037
|
+
*
|
|
1038
|
+
* CONSECUTIVE is the whole of it, and only `succeeded()` says so — see its doc.
|
|
1039
|
+
*/
|
|
1040
|
+
const MAX_HANDSHAKE_TIMEOUTS = 3;
|
|
1041
|
+
/**
|
|
1042
|
+
* Watch one connection's handshake.
|
|
1043
|
+
*
|
|
1044
|
+
* `onRetry` fires when a timed-out attempt has been re-dialed (the sandbox
|
|
1045
|
+
* behind the endpoint may have been replaced, and the URL provider re-brokers
|
|
1046
|
+
* on the next attempt); `onExhausted` when the budget is spent, or when the
|
|
1047
|
+
* socket has no reconnect machinery at all (an injected `options.WebSocket`).
|
|
1048
|
+
*
|
|
1049
|
+
* @internal
|
|
1050
|
+
*/
|
|
1051
|
+
function createHandshakeGuard(opts) {
|
|
1052
|
+
let timer;
|
|
1053
|
+
let timeouts = 0;
|
|
1054
|
+
function disarm() {
|
|
1055
|
+
if (timer === void 0) return;
|
|
1056
|
+
clearTimeout(timer);
|
|
1057
|
+
timer = void 0;
|
|
1058
|
+
}
|
|
1059
|
+
function fire() {
|
|
1060
|
+
timer = void 0;
|
|
1061
|
+
timeouts += 1;
|
|
1062
|
+
if (timeouts < MAX_HANDSHAKE_TIMEOUTS && forceReconnect(opts.socket)) {
|
|
1063
|
+
opts.onRetry();
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
opts.onExhausted();
|
|
1067
|
+
}
|
|
1068
|
+
opts.signal.addEventListener("abort", disarm);
|
|
1069
|
+
return {
|
|
1070
|
+
arm() {
|
|
1071
|
+
disarm();
|
|
1072
|
+
timer = setTimeout(fire, HANDSHAKE_TIMEOUT_MS);
|
|
1073
|
+
},
|
|
1074
|
+
disarm,
|
|
1075
|
+
succeeded() {
|
|
1076
|
+
disarm();
|
|
1077
|
+
timeouts = 0;
|
|
1078
|
+
}
|
|
1079
|
+
};
|
|
1080
|
+
}
|
|
1081
|
+
//#endregion
|
|
1082
|
+
//#region src/session/messages.ts
|
|
1083
|
+
/**
|
|
1084
|
+
* Incoming-message handling for the voice session core.
|
|
1085
|
+
*
|
|
1086
|
+
* Split out of `session/browser-session.ts`: this module owns the interpretation of
|
|
1087
|
+
* server→client frames (audio chunks + JSON {@link SessionEvent}s) and the
|
|
1088
|
+
* turn-boundary epoch, while `session/browser-session.ts` owns the state
|
|
1089
|
+
* store and connection lifecycle. The handlers read and mutate session state
|
|
1090
|
+
* exclusively through the injected `getSnapshot`/`updateState` deps.
|
|
1091
|
+
*/
|
|
1092
|
+
/** Cap on `customEvents` retained in the session snapshot to avoid unbounded growth. */
|
|
1093
|
+
const MAX_CUSTOM_EVENTS = 200;
|
|
1094
|
+
/** Cap on `messages` retained in the session snapshot; the `history.restored` frame's own. */
|
|
1095
|
+
const MAX_MESSAGES = MAX_CLIENT_MESSAGES;
|
|
1096
|
+
/**
|
|
1097
|
+
* Snapshot fields cleared when a session's conversation state is wiped —
|
|
1098
|
+
* shared by the initial snapshot, `resetState()`, and `session.reset`.
|
|
1099
|
+
* The empty arrays are safe to share: snapshot collections are never mutated
|
|
1100
|
+
* in place, only replaced.
|
|
1101
|
+
*/
|
|
1102
|
+
const CLEARED_SESSION_STATE = {
|
|
1103
|
+
messages: [],
|
|
1104
|
+
toolCalls: [],
|
|
1105
|
+
customEvents: [],
|
|
1106
|
+
agentState: null,
|
|
1107
|
+
userTranscript: null,
|
|
1108
|
+
agentTranscript: null,
|
|
1109
|
+
error: null
|
|
1110
|
+
};
|
|
1111
|
+
function appendCapped(list, item, cap) {
|
|
1112
|
+
if (list.length < cap) return [...list, item];
|
|
1113
|
+
const next = list.slice(list.length - cap + 1);
|
|
1114
|
+
next.push(item);
|
|
1115
|
+
return next;
|
|
1116
|
+
}
|
|
1117
|
+
/**
|
|
1118
|
+
* Create the server→client message handlers for one session core.
|
|
1119
|
+
*
|
|
1120
|
+
* Encapsulates the per-session dedup counters (`customEventSeq`,
|
|
1121
|
+
* `messageSeq`, `toolCallSeq`) that previously lived as closure locals in
|
|
1122
|
+
* `createBrowserSession`. The turn-boundary epoch is NOT one of them: it lives
|
|
1123
|
+
* on `conn` because the session core bumps it on teardown too (see
|
|
1124
|
+
* `ConnState.turn`).
|
|
1125
|
+
*/
|
|
1126
|
+
function createMessageHandlers(deps) {
|
|
1127
|
+
const { getSnapshot, updateState, conn, agentState, audio } = deps;
|
|
1128
|
+
/** Monotonically increasing counter for custom events -- used by useEvent to deduplicate. */
|
|
1129
|
+
let customEventSeq = 0;
|
|
1130
|
+
/** Monotonically increasing counter for chat messages -- stable render keys
|
|
1131
|
+
* and tool-call anchoring that survive the sliding message window. */
|
|
1132
|
+
let messageSeq = 0;
|
|
1133
|
+
/** Monotonically increasing counter for tool calls -- used by the tool-call
|
|
1134
|
+
* hooks to iterate only the unprocessed tail. */
|
|
1135
|
+
let toolCallSeq = 0;
|
|
1136
|
+
function appendCustomEvent(name, data) {
|
|
1137
|
+
updateState({ customEvents: appendCapped(getSnapshot().customEvents, {
|
|
1138
|
+
id: ++customEventSeq,
|
|
1139
|
+
event: name,
|
|
1140
|
+
data
|
|
1141
|
+
}, MAX_CUSTOM_EVENTS) });
|
|
1142
|
+
}
|
|
1143
|
+
function handleUserTranscriptEvent(text) {
|
|
1144
|
+
conn.turn.bump();
|
|
1145
|
+
updateState({
|
|
1146
|
+
userTranscript: null,
|
|
1147
|
+
messages: appendCapped(getSnapshot().messages, {
|
|
1148
|
+
id: ++messageSeq,
|
|
1149
|
+
role: "user",
|
|
1150
|
+
content: text
|
|
1151
|
+
}, MAX_MESSAGES),
|
|
1152
|
+
...agentState.apply({ type: "THINK" })
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
/**
|
|
1156
|
+
* The `agentTranscript.*` events carry the reply's text so far as a full-replacement
|
|
1157
|
+
* snapshot (see the protocol schema), so it renders as the live assistant
|
|
1158
|
+
* bubble and only becomes a message when the reply closes. Pipeline mode sends
|
|
1159
|
+
* one per piece of speech, so appending each would break a single reply into a
|
|
1160
|
+
* message per sentence.
|
|
1161
|
+
*
|
|
1162
|
+
* Replace rather than merge: a snapshot is not append-only. A pipeline reply's
|
|
1163
|
+
* closing snapshot drops the dead-air filler the interim ones carried, so it
|
|
1164
|
+
* can be shorter than its predecessor and differ mid-string.
|
|
1165
|
+
*/
|
|
1166
|
+
function handleAgentTranscriptEvent(text) {
|
|
1167
|
+
updateState({ agentTranscript: text });
|
|
1168
|
+
}
|
|
1169
|
+
/**
|
|
1170
|
+
* The reply is over (`reply.completed`, or `reply.cancelled` for a barge-in): move
|
|
1171
|
+
* whatever was spoken into the conversation. A cancelled reply still keeps its
|
|
1172
|
+
* text — the caller heard that much, and dropping it would leave the
|
|
1173
|
+
* transcript claiming the agent never spoke.
|
|
1174
|
+
*/
|
|
1175
|
+
function commitAgentTranscript() {
|
|
1176
|
+
const text = getSnapshot().agentTranscript;
|
|
1177
|
+
if (text === null || text.length === 0) {
|
|
1178
|
+
updateState({ agentTranscript: null });
|
|
1179
|
+
return;
|
|
1180
|
+
}
|
|
1181
|
+
updateState({
|
|
1182
|
+
agentTranscript: null,
|
|
1183
|
+
messages: appendCapped(getSnapshot().messages, {
|
|
1184
|
+
id: ++messageSeq,
|
|
1185
|
+
role: "assistant",
|
|
1186
|
+
content: text
|
|
1187
|
+
}, MAX_MESSAGES)
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
1190
|
+
/** Clear error state when a non-error event arrives — proves the session
|
|
1191
|
+
* is functional (e.g. audio init failed but WebSocket still works).
|
|
1192
|
+
*
|
|
1193
|
+
* A FATAL session is exempt, and that exemption is the whole reason the
|
|
1194
|
+
* `fatal` region exists: the host's teardown emits, so the frames that
|
|
1195
|
+
* follow a fatal error are a consequence of it rather than evidence
|
|
1196
|
+
* against it. Recovering on them left the one message that says what to
|
|
1197
|
+
* fix — a missing provider key — on screen for a fraction of a second,
|
|
1198
|
+
* over a session that could no longer hear anyone. */
|
|
1199
|
+
function clearRecoveredError() {
|
|
1200
|
+
updateState(agentState.apply({ type: "ACTIVITY" }));
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* Return to "listening" at a turn boundary — unless the session is over.
|
|
1204
|
+
*
|
|
1205
|
+
* `reply.completed`, `reply.cancelled` and `session.reset` each wrote
|
|
1206
|
+
* `state: "listening"` unconditionally, which is the second half of the same
|
|
1207
|
+
* bug `clearRecoveredError` covers: the host's fatal paths all call
|
|
1208
|
+
* `terminate()`, and terminating emits `onCancelled()`. So the frame that
|
|
1209
|
+
* ANNOUNCES the session's death was also the frame that painted a live-mic
|
|
1210
|
+
* state over the error it had just reported.
|
|
1211
|
+
*
|
|
1212
|
+
* The exemption is not restated here: `LISTEN` is declined while the `fatal`
|
|
1213
|
+
* region says so, which is what makes this the whole of the rule rather than
|
|
1214
|
+
* one of four sites that had to remember it.
|
|
1215
|
+
*/
|
|
1216
|
+
function toListening(extra = {}) {
|
|
1217
|
+
updateState({
|
|
1218
|
+
...extra,
|
|
1219
|
+
...agentState.apply({ type: "LISTEN" })
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
function handleErrorEvent(e) {
|
|
1223
|
+
console.error("Agent error:", e.message);
|
|
1224
|
+
const error = {
|
|
1225
|
+
code: e.code,
|
|
1226
|
+
message: e.message,
|
|
1227
|
+
fatal: e.fatal
|
|
1228
|
+
};
|
|
1229
|
+
if (!e.fatal) updateState(agentState.apply({
|
|
1230
|
+
type: "TURN_ERROR",
|
|
1231
|
+
error
|
|
1232
|
+
}));
|
|
1233
|
+
else {
|
|
1234
|
+
audio.teardown();
|
|
1235
|
+
updateState({
|
|
1236
|
+
...agentState.apply({
|
|
1237
|
+
type: "FATAL",
|
|
1238
|
+
error
|
|
1239
|
+
}),
|
|
1240
|
+
...STOPPED
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
/** Single entry point for all server->client session events. */
|
|
1245
|
+
function handleEvent(e) {
|
|
1246
|
+
if (e.type !== "error.reported") clearRecoveredError();
|
|
1247
|
+
switch (e.type) {
|
|
1248
|
+
case "speech.started":
|
|
1249
|
+
updateState({ userTranscript: "" });
|
|
1250
|
+
break;
|
|
1251
|
+
case "speech.stopped": break;
|
|
1252
|
+
case "userTranscript.committed":
|
|
1253
|
+
handleUserTranscriptEvent(e.text);
|
|
1254
|
+
break;
|
|
1255
|
+
case "userTranscript.updated":
|
|
1256
|
+
updateState({ userTranscript: e.text });
|
|
1257
|
+
break;
|
|
1258
|
+
case "agentTranscript.updated":
|
|
1259
|
+
case "agentTranscript.committed":
|
|
1260
|
+
handleAgentTranscriptEvent(e.text);
|
|
1261
|
+
break;
|
|
1262
|
+
case "tool.called":
|
|
1263
|
+
updateState({ toolCalls: appendCapped(getSnapshot().toolCalls, {
|
|
1264
|
+
callId: e.toolCallId,
|
|
1265
|
+
name: e.toolName,
|
|
1266
|
+
args: toArgsRecord(e.args),
|
|
1267
|
+
status: "pending",
|
|
1268
|
+
seq: ++toolCallSeq,
|
|
1269
|
+
afterMessageId: getSnapshot().messages.at(-1)?.id ?? -1
|
|
1270
|
+
}, MAX_MESSAGES) });
|
|
1271
|
+
break;
|
|
1272
|
+
case "tool.completed": {
|
|
1273
|
+
const tcs = getSnapshot().toolCalls;
|
|
1274
|
+
const idx = tcs.findIndex((tc) => tc.callId === e.toolCallId);
|
|
1275
|
+
const existing = tcs[idx];
|
|
1276
|
+
if (!existing) break;
|
|
1277
|
+
const updated = [...tcs];
|
|
1278
|
+
updated[idx] = {
|
|
1279
|
+
...existing,
|
|
1280
|
+
status: "done",
|
|
1281
|
+
result: e.result
|
|
1282
|
+
};
|
|
1283
|
+
updateState({ toolCalls: updated });
|
|
1284
|
+
break;
|
|
1285
|
+
}
|
|
1286
|
+
case "reply.completed":
|
|
1287
|
+
commitAgentTranscript();
|
|
1288
|
+
toListening();
|
|
1289
|
+
break;
|
|
1290
|
+
case "reply.cancelled":
|
|
1291
|
+
bargeIn(conn, audio);
|
|
1292
|
+
commitAgentTranscript();
|
|
1293
|
+
toListening({ userTranscript: null });
|
|
1294
|
+
break;
|
|
1295
|
+
case "session.reset": {
|
|
1296
|
+
bargeIn(conn, audio);
|
|
1297
|
+
const next = agentState.apply({ type: "RESET" });
|
|
1298
|
+
updateState(agentState.fatal() ? next : {
|
|
1299
|
+
...CLEARED_SESSION_STATE,
|
|
1300
|
+
...next
|
|
1301
|
+
});
|
|
1302
|
+
break;
|
|
1303
|
+
}
|
|
1304
|
+
case "custom.emitted":
|
|
1305
|
+
appendCustomEvent(e.event, e.data);
|
|
1306
|
+
break;
|
|
1307
|
+
case "state.updated":
|
|
1308
|
+
updateState({ agentState: shareUnchangedSlots(getSnapshot().agentState, e.state) });
|
|
1309
|
+
break;
|
|
1310
|
+
case "history.restored": {
|
|
1311
|
+
messageSeq = 0;
|
|
1312
|
+
toolCallSeq = 0;
|
|
1313
|
+
const restored = e.messages.slice(-MAX_MESSAGES).map((m) => ({
|
|
1314
|
+
id: ++messageSeq,
|
|
1315
|
+
role: m.role,
|
|
1316
|
+
content: m.content
|
|
1317
|
+
}));
|
|
1318
|
+
updateState({
|
|
1319
|
+
messages: restored,
|
|
1320
|
+
toolCalls: e.toolCalls.slice(-MAX_MESSAGES).map((tc) => ({
|
|
1321
|
+
callId: tc.callId,
|
|
1322
|
+
name: tc.name,
|
|
1323
|
+
args: toArgsRecord(tc.args),
|
|
1324
|
+
status: tc.status,
|
|
1325
|
+
...omitUndefined({ result: tc.result }),
|
|
1326
|
+
seq: ++toolCallSeq,
|
|
1327
|
+
afterMessageId: restored[tc.afterMessageIndex]?.id ?? -1
|
|
1328
|
+
}))
|
|
1329
|
+
});
|
|
1330
|
+
break;
|
|
1331
|
+
}
|
|
1332
|
+
case "error.reported":
|
|
1333
|
+
handleErrorEvent(e);
|
|
1334
|
+
break;
|
|
1335
|
+
case "session.timedOut": conn.retiredByServer = true;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
/** Enqueue a PCM16 audio chunk for playback. Transitions state to `"speaking"` on the first chunk. */
|
|
1339
|
+
function playAudioChunk(chunk) {
|
|
1340
|
+
updateState(agentState.apply({ type: "SPEAK" }));
|
|
1341
|
+
audio.enqueue(chunk);
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* Signal that the server has finished sending audio for this turn.
|
|
1345
|
+
*
|
|
1346
|
+
* A path that is up waits for its queue to drain before going back to
|
|
1347
|
+
* listening; one still coming up records the done and answers optimistically,
|
|
1348
|
+
* then replays it once the buffered greeting has somewhere to go. Both arms
|
|
1349
|
+
* live in `session/audio-state.ts`.
|
|
1350
|
+
*/
|
|
1351
|
+
function playAudioDone() {
|
|
1352
|
+
audio.done();
|
|
1353
|
+
}
|
|
1354
|
+
function handleMessage(data) {
|
|
1355
|
+
if (data instanceof ArrayBuffer) {
|
|
1356
|
+
playAudioChunk(data);
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
if (typeof data !== "string") {
|
|
1360
|
+
console.warn("session-core: non-string, non-binary frame received; dropping");
|
|
1361
|
+
return;
|
|
1362
|
+
}
|
|
1363
|
+
const raw = safeJsonParse(data);
|
|
1364
|
+
if (raw === void 0) {
|
|
1365
|
+
console.warn("session-core: invalid JSON; dropping");
|
|
1366
|
+
return;
|
|
1367
|
+
}
|
|
1368
|
+
const parsed = lenientParse(SessionEventSchema, raw);
|
|
1369
|
+
if (!parsed.ok) {
|
|
1370
|
+
if (parsed.malformed) console.warn("session-core: malformed server message", parsed.error);
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
const msg = parsed.data;
|
|
1374
|
+
if (msg.type === "session.configured") {
|
|
1375
|
+
agentState.apply({ type: "HANDSHAKE_COMPLETE" });
|
|
1376
|
+
return {
|
|
1377
|
+
sampleRate: msg.sampleRate,
|
|
1378
|
+
ttsSampleRate: msg.ttsSampleRate,
|
|
1379
|
+
sid: msg.sessionId
|
|
1380
|
+
};
|
|
1381
|
+
}
|
|
1382
|
+
if (msg.type === "audio.completed") {
|
|
1383
|
+
playAudioDone();
|
|
1384
|
+
return;
|
|
1385
|
+
}
|
|
1386
|
+
handleEvent(msg);
|
|
1387
|
+
}
|
|
1388
|
+
return { handleMessage };
|
|
1389
|
+
}
|
|
1390
|
+
/**
|
|
1391
|
+
* The new frame, with every slot whose view did not change keeping the OLD
|
|
1392
|
+
* value object — so `useAgentState(slot)` re-renders only for its own slot.
|
|
1393
|
+
* Compared by serialization, which is how the server decided to push at all.
|
|
1394
|
+
*/
|
|
1395
|
+
function shareUnchangedSlots(previous, next) {
|
|
1396
|
+
if (previous === null) return next;
|
|
1397
|
+
const frame = {};
|
|
1398
|
+
for (const [slot, value] of Object.entries(next)) {
|
|
1399
|
+
const before = previous[slot];
|
|
1400
|
+
frame[slot] = slot in previous && JSON.stringify(before) === JSON.stringify(value) ? before : value;
|
|
1401
|
+
}
|
|
1402
|
+
return frame;
|
|
1403
|
+
}
|
|
1404
|
+
//#endregion
|
|
1405
|
+
//#region src/session/mic.ts
|
|
1406
|
+
/**
|
|
1407
|
+
* The microphone's way onto the wire — backpressure, and the caller's MUTE.
|
|
1408
|
+
*
|
|
1409
|
+
* Split out of `session/browser-session.ts` at the source-length cap, along the seam the
|
|
1410
|
+
* two share: both decide what a captured frame becomes on the socket, and
|
|
1411
|
+
* nothing else in the core reads either.
|
|
1412
|
+
*
|
|
1413
|
+
* ## A muted frame is SILENCE, not an absent frame
|
|
1414
|
+
*
|
|
1415
|
+
* `setMicMuted(true)` keeps sending every frame the capture worklet produces,
|
|
1416
|
+
* at the same length and cadence, zero-filled. Sending nothing would look
|
|
1417
|
+
* cheaper and is wrong on three counts:
|
|
1418
|
+
*
|
|
1419
|
+
* - **Endpointing.** Hold-to-talk over an agent with AUTOMATIC turn detection
|
|
1420
|
+
* is mute-until-pressed, unmute, speak, mute on release. The transcriber
|
|
1421
|
+
* ends the turn on a stretch of silence; a stream that simply stops gives it
|
|
1422
|
+
* no silence to measure, so the last utterance is answered late or not at
|
|
1423
|
+
* all.
|
|
1424
|
+
* - **The stream clock.** The transcriber timestamps by samples received. The
|
|
1425
|
+
* runtime's own push-to-talk silences the mic the same way, for the same
|
|
1426
|
+
* reason (`aai-runtime/src/transports/pipeline/commands.ts`, "sent as SILENCE rather than
|
|
1427
|
+
* withheld").
|
|
1428
|
+
* - **Idle.** Nothing is gained on the server side by going quiet: the session
|
|
1429
|
+
* idle deadline deliberately does not count user audio (`ServerSession.onAudio`
|
|
1430
|
+
* in the runtime), so a muted caller is retired exactly as a silent one is —
|
|
1431
|
+
* while a provider that closes a socket fed no audio at all would see a gap.
|
|
1432
|
+
*
|
|
1433
|
+
* ## It is UI state, not connection state
|
|
1434
|
+
*
|
|
1435
|
+
* The flag lives on the snapshot (`micMuted`) and nothing that tears a
|
|
1436
|
+
* connection down touches it — not a reconnect, a resume, `disconnect()`,
|
|
1437
|
+
* `reset()` or `end()` — and it can be set before the first `connect()`, so a
|
|
1438
|
+
* UI can open a typed-only session already muted. Muting sends no command and
|
|
1439
|
+
* interrupts nothing; it only changes what the next frame carries.
|
|
1440
|
+
*
|
|
1441
|
+
* It sits AFTER the capture worklet, so the worklet's own digital-silence
|
|
1442
|
+
* probe (`onMicSilent`, `MIC_SILENCE_PROBE_MS` in `audio/voice-io.ts`) still measures
|
|
1443
|
+
* the real input: zeros this module writes can never read as a dead mic.
|
|
1444
|
+
*
|
|
1445
|
+
* @module
|
|
1446
|
+
*/
|
|
1447
|
+
/** Build one session's mic sender. @internal */
|
|
1448
|
+
function createMicSender(deps) {
|
|
1449
|
+
let silence = /* @__PURE__ */ new ArrayBuffer(0);
|
|
1450
|
+
const silent = (length) => {
|
|
1451
|
+
if (silence.byteLength !== length) silence = new ArrayBuffer(length);
|
|
1452
|
+
return silence;
|
|
1453
|
+
};
|
|
1454
|
+
return {
|
|
1455
|
+
sendAudio(bytes) {
|
|
1456
|
+
const ws = deps.conn.ws;
|
|
1457
|
+
if (ws?.readyState !== WS_OPEN) return;
|
|
1458
|
+
if (ws.bufferedAmount > MIC_SEND_MAX_BUFFERED_BYTES) return;
|
|
1459
|
+
ws.send(deps.snapshot().micMuted ? silent(bytes.byteLength) : bytes);
|
|
1460
|
+
},
|
|
1461
|
+
sendBuffered(chunks) {
|
|
1462
|
+
const ws = deps.conn.ws;
|
|
1463
|
+
if (ws?.readyState !== WS_OPEN) return;
|
|
1464
|
+
const muted = deps.snapshot().micMuted;
|
|
1465
|
+
for (const bytes of chunks) ws.send(muted ? silent(bytes.byteLength) : bytes);
|
|
1466
|
+
},
|
|
1467
|
+
setMicMuted(muted) {
|
|
1468
|
+
deps.updateState({ micMuted: muted });
|
|
1469
|
+
}
|
|
1470
|
+
};
|
|
1471
|
+
}
|
|
1472
|
+
//#endregion
|
|
1473
|
+
//#region src/session/preconnect.ts
|
|
1474
|
+
/**
|
|
1475
|
+
* The rate the capture guesses: the SDK's default STT rate
|
|
1476
|
+
* (`DEFAULT_STT_SAMPLE_RATE`), which is also the only rate an AssemblyAI S2S
|
|
1477
|
+
* agent accepts. A miss costs a resample of the buffer, not the audio.
|
|
1478
|
+
*/
|
|
1479
|
+
const PRE_CONNECT_SAMPLE_RATE = 16e3;
|
|
1480
|
+
/** Build one session's pre-connect capture holder. @internal */
|
|
1481
|
+
function createPreConnectAudio(enabled) {
|
|
1482
|
+
let held = null;
|
|
1483
|
+
return {
|
|
1484
|
+
begin() {
|
|
1485
|
+
loadAudioModules().catch(() => {});
|
|
1486
|
+
if (!enabled || held !== null) return;
|
|
1487
|
+
held = Promise.all([import("./audio/index.js"), loadAudioModules()]).then(([{ openPreConnectCapture }, [, captureWorklet]]) => openPreConnectCapture({
|
|
1488
|
+
sampleRate: PRE_CONNECT_SAMPLE_RATE,
|
|
1489
|
+
captureWorkletSrc: captureWorklet,
|
|
1490
|
+
maxSeconds: 10
|
|
1491
|
+
})).catch(() => null);
|
|
1492
|
+
},
|
|
1493
|
+
take() {
|
|
1494
|
+
const taken = held;
|
|
1495
|
+
held = null;
|
|
1496
|
+
return taken;
|
|
1497
|
+
},
|
|
1498
|
+
release() {
|
|
1499
|
+
const taken = held;
|
|
1500
|
+
held = null;
|
|
1501
|
+
taken?.then((capture) => capture?.close());
|
|
1502
|
+
}
|
|
1503
|
+
};
|
|
1504
|
+
}
|
|
1505
|
+
//#endregion
|
|
1506
|
+
//#region src/session/state.ts
|
|
1507
|
+
/**
|
|
1508
|
+
* The browser session's {@link AgentState}, and the error beside it, as a
|
|
1509
|
+
* statechart.
|
|
1510
|
+
*
|
|
1511
|
+
* These two were written independently from thirteen sites across
|
|
1512
|
+
* `session/browser-session.ts`, `session/messages.ts` and
|
|
1513
|
+
* `session/audio-setup.ts`, each one deciding for itself whether its write
|
|
1514
|
+
* was legal by reading the snapshot back first. Three shipped bugs came out of
|
|
1515
|
+
* that, and all three are the same shape — a transition nothing forbade:
|
|
1516
|
+
*
|
|
1517
|
+
* - **A straggler audio chunk flipped an errored session to `"speaking"`.**
|
|
1518
|
+
* Guarded by hand at the call site
|
|
1519
|
+
* (`if (snap.state === "error" || (snap.state === "disconnected" && …))`).
|
|
1520
|
+
* Here `error` simply does not handle `SPEAK`.
|
|
1521
|
+
* - **The frame that announced a session's death also wiped the banner
|
|
1522
|
+
* reporting it.** Every fatal path in the host tears the transport down, and
|
|
1523
|
+
* tearing down EMITS — so `reply.cancelled` arrived right behind the error and
|
|
1524
|
+
* `toListening()` painted a live-mic state over it. A missing provider key is
|
|
1525
|
+
* the case that made it visible: the one message that says exactly what to fix
|
|
1526
|
+
* was on screen for a few hundred milliseconds and left a session that looked
|
|
1527
|
+
* live and was deaf.
|
|
1528
|
+
* - **A later frame RECOVERED the state.** `clearRecoveredError` reads a
|
|
1529
|
+
* non-error frame as proof the session works, which is right for a
|
|
1530
|
+
* turn-level failure and wrong for a server error that ended the call.
|
|
1531
|
+
*
|
|
1532
|
+
* The second and third were fixed with a `conn.fatalError` boolean that every
|
|
1533
|
+
* writer had to remember to consult. It is the `fatal` region here, so
|
|
1534
|
+
* forgetting is not available: `LISTEN` and `ACTIVITY` are declined in one
|
|
1535
|
+
* place rather than at each of their five call sites.
|
|
1536
|
+
*
|
|
1537
|
+
* ## Two regions, because the fatal latch OUTLIVES the error state
|
|
1538
|
+
*
|
|
1539
|
+
* `fatal` is not a substate of `error`, tempting as that looks. It is cleared by
|
|
1540
|
+
* exactly one thing — the next `config` frame, i.e. a completed handshake — and
|
|
1541
|
+
* that is per CONNECTION rather than per session, so a reconnect that really
|
|
1542
|
+
* works is not pinned to a dead session's banner. Between the error and that
|
|
1543
|
+
* frame the phase runs `error → connecting → ready` while the latch stays set,
|
|
1544
|
+
* which a substate cannot express.
|
|
1545
|
+
*
|
|
1546
|
+
* ## The published surface is unchanged
|
|
1547
|
+
*
|
|
1548
|
+
* {@link AgentState} is a public type in the versioned `aai-ui:session`
|
|
1549
|
+
* capability, so {@link AgentStateSnapshot.state} is one of its seven names and
|
|
1550
|
+
* nothing here widens it. The internal distinctions live in the second region
|
|
1551
|
+
* and in context, not in the projection.
|
|
1552
|
+
*/
|
|
1553
|
+
/**
|
|
1554
|
+
* Not fatally over — so a working state may be painted.
|
|
1555
|
+
*
|
|
1556
|
+
* `stateIn` reads the sibling REGION rather than a mirror of it in context,
|
|
1557
|
+
* which is what keeps `fatal` the one place the latch lives. The three events
|
|
1558
|
+
* this guards are the three that used to consult `conn.fatalError` by hand at
|
|
1559
|
+
* five call sites; `THINK` is a fourth that did not, and should have — the doc
|
|
1560
|
+
* on that flag says outright that "no later frame may take its banner off the
|
|
1561
|
+
* screen", and a `userTranscript.committed` arriving behind a fatal error
|
|
1562
|
+
* painted `"thinking"` over it.
|
|
1563
|
+
*/
|
|
1564
|
+
const NOT_FATAL = not(stateIn({ fatal: "yes" }));
|
|
1565
|
+
const sessionStateMachine = setup({
|
|
1566
|
+
types: {},
|
|
1567
|
+
guards: {
|
|
1568
|
+
/** A banner is still up from a failure the session survived. */
|
|
1569
|
+
hasError: ({ context }) => context.error !== null },
|
|
1570
|
+
actions: {
|
|
1571
|
+
clearError: assign({ error: null }),
|
|
1572
|
+
setError: assign({ error: ({ context, event }) => event.type === "TURN_ERROR" || event.type === "FATAL" || event.type === "FAILED" ? event.error : context.error })
|
|
1573
|
+
}
|
|
1574
|
+
}).createMachine({
|
|
1575
|
+
id: "sessionState",
|
|
1576
|
+
type: "parallel",
|
|
1577
|
+
context: { error: null },
|
|
1578
|
+
states: {
|
|
1579
|
+
/** The seven names {@link AgentState} publishes. */
|
|
1580
|
+
phase: {
|
|
1581
|
+
initial: "disconnected",
|
|
1582
|
+
on: {
|
|
1583
|
+
CONNECT: {
|
|
1584
|
+
target: ".connecting",
|
|
1585
|
+
actions: "clearError"
|
|
1586
|
+
},
|
|
1587
|
+
CLOSED: {
|
|
1588
|
+
target: ".disconnected",
|
|
1589
|
+
actions: "clearError"
|
|
1590
|
+
},
|
|
1591
|
+
DISCONNECT: ".disconnected",
|
|
1592
|
+
END: {
|
|
1593
|
+
target: ".disconnected",
|
|
1594
|
+
actions: "clearError"
|
|
1595
|
+
},
|
|
1596
|
+
TURN_ERROR: { actions: "setError" },
|
|
1597
|
+
FATAL: {
|
|
1598
|
+
target: ".error",
|
|
1599
|
+
actions: "setError"
|
|
1600
|
+
},
|
|
1601
|
+
FAILED: {
|
|
1602
|
+
target: ".error",
|
|
1603
|
+
actions: "setError"
|
|
1604
|
+
},
|
|
1605
|
+
LISTEN: {
|
|
1606
|
+
guard: NOT_FATAL,
|
|
1607
|
+
target: ".listening"
|
|
1608
|
+
},
|
|
1609
|
+
RESET: {
|
|
1610
|
+
guard: NOT_FATAL,
|
|
1611
|
+
target: ".listening",
|
|
1612
|
+
actions: "clearError"
|
|
1613
|
+
},
|
|
1614
|
+
THINK: {
|
|
1615
|
+
guard: NOT_FATAL,
|
|
1616
|
+
target: ".thinking"
|
|
1617
|
+
},
|
|
1618
|
+
ACTIVITY: {
|
|
1619
|
+
guard: and([NOT_FATAL, "hasError"]),
|
|
1620
|
+
actions: "clearError"
|
|
1621
|
+
}
|
|
1622
|
+
},
|
|
1623
|
+
states: {
|
|
1624
|
+
disconnected: { on: { SPEAK: {
|
|
1625
|
+
guard: not("hasError"),
|
|
1626
|
+
target: "speaking"
|
|
1627
|
+
} } },
|
|
1628
|
+
connecting: { on: { SOCKET_OPEN: "ready" } },
|
|
1629
|
+
ready: { on: { SPEAK: "speaking" } },
|
|
1630
|
+
listening: { on: { SPEAK: "speaking" } },
|
|
1631
|
+
thinking: { on: { SPEAK: "speaking" } },
|
|
1632
|
+
speaking: {},
|
|
1633
|
+
/**
|
|
1634
|
+
* A failure is on screen.
|
|
1635
|
+
*
|
|
1636
|
+
* `SPEAK` is absent deliberately — that is half the guard
|
|
1637
|
+
* `playAudioChunk` used to spell, and the other half is `disconnected`'s
|
|
1638
|
+
* above. `ACTIVITY` recovers to `listening` rather than merely clearing
|
|
1639
|
+
* the banner: the socket is demonstrably open (we are handling a server
|
|
1640
|
+
* event), so `disconnected` would misreport a live session.
|
|
1641
|
+
*/
|
|
1642
|
+
error: { on: {
|
|
1643
|
+
ACTIVITY: {
|
|
1644
|
+
guard: NOT_FATAL,
|
|
1645
|
+
target: "listening",
|
|
1646
|
+
actions: "clearError"
|
|
1647
|
+
},
|
|
1648
|
+
CLOSED: {}
|
|
1649
|
+
} }
|
|
1650
|
+
}
|
|
1651
|
+
},
|
|
1652
|
+
/**
|
|
1653
|
+
* Whether the session is fatally over.
|
|
1654
|
+
*
|
|
1655
|
+
* Separate from `phase` because it OUTLIVES the `error` state: only a
|
|
1656
|
+
* completed handshake clears it, and the phase moves through `connecting`
|
|
1657
|
+
* and `ready` on the way to one. A substate of `error` could not say that.
|
|
1658
|
+
*/
|
|
1659
|
+
fatal: {
|
|
1660
|
+
initial: "no",
|
|
1661
|
+
states: {
|
|
1662
|
+
no: { on: { FATAL: "yes" } },
|
|
1663
|
+
yes: { on: { HANDSHAKE_COMPLETE: "no" } }
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
});
|
|
1668
|
+
/** Create the state machine for one browser session. */
|
|
1669
|
+
function createSessionStateMachine() {
|
|
1670
|
+
const actor = createActor(sessionStateMachine).start();
|
|
1671
|
+
function snapshot() {
|
|
1672
|
+
const at = actor.getSnapshot();
|
|
1673
|
+
return {
|
|
1674
|
+
state: at.value.phase,
|
|
1675
|
+
error: at.context.error
|
|
1676
|
+
};
|
|
1677
|
+
}
|
|
1678
|
+
return {
|
|
1679
|
+
snapshot,
|
|
1680
|
+
apply(event) {
|
|
1681
|
+
actor.send(event);
|
|
1682
|
+
return snapshot();
|
|
1683
|
+
},
|
|
1684
|
+
fatal: () => actor.getSnapshot().matches({ fatal: "yes" })
|
|
1685
|
+
};
|
|
1686
|
+
}
|
|
1687
|
+
//#endregion
|
|
1688
|
+
//#region src/session/user-turn.ts
|
|
1689
|
+
/**
|
|
1690
|
+
* The local half of an interruption: the agent stops here at once, the same
|
|
1691
|
+
* barge-in `cancel()` makes — only when there is something to stop, so a
|
|
1692
|
+
* press or a message into silence leaves a "listening" state alone. The
|
|
1693
|
+
* server's `reply.cancelled` follows and finds nothing left to flush.
|
|
1694
|
+
*/
|
|
1695
|
+
function interruptLocally(deps) {
|
|
1696
|
+
const { state } = deps.snapshot();
|
|
1697
|
+
if (state === "speaking" || state === "thinking") {
|
|
1698
|
+
deps.bargeIn();
|
|
1699
|
+
deps.updateState(deps.agentState.apply({ type: "LISTEN" }));
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
/**
|
|
1703
|
+
* Everything a caller's turn can be started by: push-to-talk's three edges
|
|
1704
|
+
* and a TYPED turn — `session.userTurn` and `session.sendText`. One builder
|
|
1705
|
+
* over one set of deps, because both are the same kind of thing (a user turn
|
|
1706
|
+
* the client, not the transcriber, delimits) and both interrupt the same way.
|
|
1707
|
+
*
|
|
1708
|
+
* @internal
|
|
1709
|
+
*/
|
|
1710
|
+
function createUserInput(deps) {
|
|
1711
|
+
function send(trimmed) {
|
|
1712
|
+
interruptLocally(deps);
|
|
1713
|
+
deps.sendJson({
|
|
1714
|
+
type: "user_text",
|
|
1715
|
+
text: trimmed
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
return {
|
|
1719
|
+
userTurn: createUserTurnActions(deps),
|
|
1720
|
+
sendText(text, options) {
|
|
1721
|
+
const trimmed = text.trim();
|
|
1722
|
+
if (trimmed === "") return;
|
|
1723
|
+
if (deps.connected() && deps.queued.length === 0) {
|
|
1724
|
+
send(trimmed);
|
|
1725
|
+
return;
|
|
1726
|
+
}
|
|
1727
|
+
if (deps.queued.length === 0 && options?.connect !== true) return;
|
|
1728
|
+
deps.queued.push(trimmed);
|
|
1729
|
+
deps.open();
|
|
1730
|
+
},
|
|
1731
|
+
flushQueued() {
|
|
1732
|
+
if (!deps.connected()) return;
|
|
1733
|
+
for (const text of deps.queued.splice(0)) send(text);
|
|
1734
|
+
},
|
|
1735
|
+
sendToolResult(toolCallId, outcome) {
|
|
1736
|
+
if (!deps.connected()) return;
|
|
1737
|
+
deps.sendJson({
|
|
1738
|
+
type: "tool_result",
|
|
1739
|
+
toolCallId,
|
|
1740
|
+
...encodeOutcome(outcome)
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
};
|
|
1744
|
+
}
|
|
1745
|
+
/**
|
|
1746
|
+
* The frame's two fields. A result `JSON.stringify` cannot encode (a cycle, a
|
|
1747
|
+
* `BigInt`, a bare function) fails the call naming why, rather than throwing
|
|
1748
|
+
* into the handler that produced it or leaving the server waiting.
|
|
1749
|
+
*/
|
|
1750
|
+
function encodeOutcome(outcome) {
|
|
1751
|
+
if ("error" in outcome) return {
|
|
1752
|
+
result: "",
|
|
1753
|
+
error: outcome.error
|
|
1754
|
+
};
|
|
1755
|
+
try {
|
|
1756
|
+
return { result: JSON.stringify(outcome.result ?? null) ?? "null" };
|
|
1757
|
+
} catch (err) {
|
|
1758
|
+
return {
|
|
1759
|
+
result: "",
|
|
1760
|
+
error: `Tool result is not JSON-serializable: ${String(err)}`
|
|
1761
|
+
};
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
/** Build the three push-to-talk edges. @internal */
|
|
1765
|
+
function createUserTurnActions(deps) {
|
|
1766
|
+
return {
|
|
1767
|
+
start() {
|
|
1768
|
+
if (!deps.connected()) return;
|
|
1769
|
+
interruptLocally(deps);
|
|
1770
|
+
deps.sendJson({ type: "user_turn_start" });
|
|
1771
|
+
},
|
|
1772
|
+
commit() {
|
|
1773
|
+
if (!deps.connected()) return;
|
|
1774
|
+
deps.sendJson({ type: "user_turn_commit" });
|
|
1775
|
+
},
|
|
1776
|
+
clear() {
|
|
1777
|
+
if (!deps.connected()) return;
|
|
1778
|
+
deps.updateState({ userTranscript: null });
|
|
1779
|
+
deps.sendJson({ type: "user_turn_clear" });
|
|
1780
|
+
}
|
|
1781
|
+
};
|
|
1782
|
+
}
|
|
1783
|
+
//#endregion
|
|
1784
|
+
//#region src/session/browser-session.ts
|
|
1785
|
+
/**
|
|
1786
|
+
* Framework-agnostic voice session core.
|
|
1787
|
+
*
|
|
1788
|
+
* Manages WebSocket communication, audio capture/playback, and agent state
|
|
1789
|
+
* transitions using a subscribe/getSnapshot pattern compatible with React's
|
|
1790
|
+
* `useSyncExternalStore` and other external store consumers.
|
|
1791
|
+
*
|
|
1792
|
+
* Server→client message interpretation lives in `session/messages.ts`;
|
|
1793
|
+
* the public/internal type declarations live in `session/types.ts`.
|
|
1794
|
+
*
|
|
1795
|
+
* No dependency on React, Preact, or any UI framework.
|
|
1796
|
+
*/
|
|
1797
|
+
/**
|
|
1798
|
+
* Create a framework-agnostic voice session core that connects to an AAI
|
|
1799
|
+
* server via WebSocket.
|
|
1800
|
+
*
|
|
1801
|
+
* Uses a subscribe/getSnapshot pattern for state management, compatible with
|
|
1802
|
+
* React's `useSyncExternalStore` and other external store integrations.
|
|
1803
|
+
*
|
|
1804
|
+
* Most clients never call this: `mountClient()` creates a core and installs it in
|
|
1805
|
+
* React context for the hooks. Reach for it directly when building a
|
|
1806
|
+
* non-React UI (or wiring the session into another framework's store).
|
|
1807
|
+
*
|
|
1808
|
+
* @example
|
|
1809
|
+
* ```ts
|
|
1810
|
+
* import { createBrowserSession, type SessionSnapshot } from "@alexkroman1/aai-ui";
|
|
1811
|
+
*
|
|
1812
|
+
* declare function render(snapshot: SessionSnapshot): void;
|
|
1813
|
+
*
|
|
1814
|
+
* const session = createBrowserSession({ platformUrl: "https://host/my-agent/" });
|
|
1815
|
+
* session.subscribe(() => render(session.getSnapshot()));
|
|
1816
|
+
* session.start();
|
|
1817
|
+
* ```
|
|
1818
|
+
*
|
|
1819
|
+
* @param options - Session configuration including the platform server URL.
|
|
1820
|
+
* @returns A {@link BrowserSession} handle for controlling the session.
|
|
1821
|
+
*
|
|
1822
|
+
* @public
|
|
1823
|
+
*/
|
|
1824
|
+
function createBrowserSession(options) {
|
|
1825
|
+
return createBrowserSessionWith(options, {});
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* {@link createBrowserSession} with its audio bring-up injectable. Not on the
|
|
1829
|
+
* barrel: it is the seam a spec uses to hold an audio path open by hand rather
|
|
1830
|
+
* than replacing the audio module.
|
|
1831
|
+
*
|
|
1832
|
+
* @internal
|
|
1833
|
+
*/
|
|
1834
|
+
function createBrowserSessionWith(options, internals) {
|
|
1835
|
+
let currentSnapshot = {
|
|
1836
|
+
...CLEARED_SESSION_STATE,
|
|
1837
|
+
state: "disconnected",
|
|
1838
|
+
contentVersion: 0,
|
|
1839
|
+
started: false,
|
|
1840
|
+
running: false,
|
|
1841
|
+
recording: false,
|
|
1842
|
+
micMuted: false,
|
|
1843
|
+
apiUrl: buildWsUrl(options.platformUrl).toString()
|
|
1844
|
+
};
|
|
1845
|
+
const subscribers = /* @__PURE__ */ new Set();
|
|
1846
|
+
function notify() {
|
|
1847
|
+
for (const sub of subscribers) sub();
|
|
1848
|
+
}
|
|
1849
|
+
/** Snapshot fields whose changes bump `contentVersion` (rendered conversation content). */
|
|
1850
|
+
const contentKeys = [
|
|
1851
|
+
"messages",
|
|
1852
|
+
"toolCalls",
|
|
1853
|
+
"userTranscript",
|
|
1854
|
+
"agentTranscript"
|
|
1855
|
+
];
|
|
1856
|
+
/** Does `partial` leave this field exactly as it already is? */
|
|
1857
|
+
function isUnchanged(key, partial) {
|
|
1858
|
+
return partial[key] === currentSnapshot[key];
|
|
1859
|
+
}
|
|
1860
|
+
/** `sendText(…, { connect: true })` waiting for a session — see `UserTurnDeps.queued`. */
|
|
1861
|
+
const queuedText = [];
|
|
1862
|
+
function updateState(partial) {
|
|
1863
|
+
if (partial.running === false) queuedText.length = 0;
|
|
1864
|
+
if (Object.keys(partial).every((key) => isUnchanged(key, partial))) return;
|
|
1865
|
+
currentSnapshot = contentKeys.some((key) => key in partial && partial[key] !== currentSnapshot[key]) ? {
|
|
1866
|
+
...currentSnapshot,
|
|
1867
|
+
...partial,
|
|
1868
|
+
contentVersion: currentSnapshot.contentVersion + 1
|
|
1869
|
+
} : {
|
|
1870
|
+
...currentSnapshot,
|
|
1871
|
+
...partial
|
|
1872
|
+
};
|
|
1873
|
+
notify();
|
|
1874
|
+
}
|
|
1875
|
+
function getSnapshot() {
|
|
1876
|
+
return currentSnapshot;
|
|
1877
|
+
}
|
|
1878
|
+
function subscribe(callback) {
|
|
1879
|
+
subscribers.add(callback);
|
|
1880
|
+
return () => {
|
|
1881
|
+
subscribers.delete(callback);
|
|
1882
|
+
};
|
|
1883
|
+
}
|
|
1884
|
+
/**
|
|
1885
|
+
* The session's `state` and `error`, as one fact rather than two fields
|
|
1886
|
+
* thirteen call sites wrote independently — see `session/state.ts`,
|
|
1887
|
+
* which carries the three shipped bugs that arrangement produced.
|
|
1888
|
+
*/
|
|
1889
|
+
const agentState = createSessionStateMachine();
|
|
1890
|
+
const conn = {
|
|
1891
|
+
ws: null,
|
|
1892
|
+
retiredByServer: false,
|
|
1893
|
+
turn: createEpoch()
|
|
1894
|
+
};
|
|
1895
|
+
let connectionController = null;
|
|
1896
|
+
const identity = createSessionIdentity(options, () => dialer);
|
|
1897
|
+
const dialer = createDialer({
|
|
1898
|
+
...options,
|
|
1899
|
+
client: identity.clientId
|
|
1900
|
+
}, notify);
|
|
1901
|
+
function resetState() {
|
|
1902
|
+
updateState(CLEARED_SESSION_STATE);
|
|
1903
|
+
}
|
|
1904
|
+
/** The socket if it can carry a frame — a socket, not a boolean, so it narrows `conn.ws`. */
|
|
1905
|
+
function openSocket() {
|
|
1906
|
+
return conn.ws?.readyState === WS_OPEN ? conn.ws : null;
|
|
1907
|
+
}
|
|
1908
|
+
function sendJson(msg) {
|
|
1909
|
+
openSocket()?.send(JSON.stringify(msg));
|
|
1910
|
+
}
|
|
1911
|
+
const mic = createMicSender({
|
|
1912
|
+
conn,
|
|
1913
|
+
snapshot: () => currentSnapshot,
|
|
1914
|
+
updateState
|
|
1915
|
+
});
|
|
1916
|
+
/**
|
|
1917
|
+
* The microphone, the worklets and the buffer in front of them, as a
|
|
1918
|
+
* statechart — see `session/audio-state.ts`, which carries what the
|
|
1919
|
+
* in-flight latch, the generation epoch and the two pre-init fields on
|
|
1920
|
+
* `ConnState` cost. `session/audio-effects.ts` is the other half: every
|
|
1921
|
+
* frame and snapshot write the machine decides on but cannot make.
|
|
1922
|
+
*/
|
|
1923
|
+
const preConnect = createPreConnectAudio(options.preConnectAudio !== false);
|
|
1924
|
+
const audio = createAudioPath(createAudioEffects({
|
|
1925
|
+
conn,
|
|
1926
|
+
updateState,
|
|
1927
|
+
agentState,
|
|
1928
|
+
sendJson,
|
|
1929
|
+
mic,
|
|
1930
|
+
preConnect,
|
|
1931
|
+
openAudioPath: internals.openAudioPath
|
|
1932
|
+
}));
|
|
1933
|
+
const { handleMessage } = createMessageHandlers({
|
|
1934
|
+
getSnapshot,
|
|
1935
|
+
updateState,
|
|
1936
|
+
conn,
|
|
1937
|
+
agentState,
|
|
1938
|
+
audio
|
|
1939
|
+
});
|
|
1940
|
+
/** Abort the in-flight connection and release audio + WebSocket resources. */
|
|
1941
|
+
function teardownConnection() {
|
|
1942
|
+
connectionController?.abort();
|
|
1943
|
+
connectionController = null;
|
|
1944
|
+
preConnect.release();
|
|
1945
|
+
audio.teardown();
|
|
1946
|
+
conn.ws?.close();
|
|
1947
|
+
conn.ws = null;
|
|
1948
|
+
}
|
|
1949
|
+
/**
|
|
1950
|
+
* React to the server's `session.configured` frame: record it and set up the
|
|
1951
|
+
* session's audio path. **It no longer replays history, and that is the
|
|
1952
|
+
* point**: a reconnect used to push `messages` back, making the CLIENT the
|
|
1953
|
+
* authority on the agent's memory; the server restores the conversation from
|
|
1954
|
+
* its own event stream now, covering what a client cannot (a second tab, a
|
|
1955
|
+
* replacement sandbox, a reopened tab). The transcript on screen is untouched.
|
|
1956
|
+
*/
|
|
1957
|
+
function onServerConfig(config) {
|
|
1958
|
+
dialer.configured(config.sid);
|
|
1959
|
+
options.onSessionId?.(config.sid);
|
|
1960
|
+
flushQueued();
|
|
1961
|
+
audio.start(config);
|
|
1962
|
+
}
|
|
1963
|
+
function connect(opts) {
|
|
1964
|
+
if (opts?.signal?.aborted) {
|
|
1965
|
+
disconnect();
|
|
1966
|
+
return;
|
|
1967
|
+
}
|
|
1968
|
+
updateState(agentState.apply({ type: "CONNECT" }));
|
|
1969
|
+
teardownConnection();
|
|
1970
|
+
preConnect.begin();
|
|
1971
|
+
conn.retiredByServer = false;
|
|
1972
|
+
const controller = new AbortController();
|
|
1973
|
+
connectionController = controller;
|
|
1974
|
+
const { signal: sig } = controller;
|
|
1975
|
+
if (opts?.signal) opts.signal.addEventListener("abort", () => disconnect(), { signal: sig });
|
|
1976
|
+
const socket = dialer.open();
|
|
1977
|
+
socket.binaryType = "arraybuffer";
|
|
1978
|
+
conn.ws = socket;
|
|
1979
|
+
let socketErrored = false;
|
|
1980
|
+
const handshake = createHandshakeGuard({
|
|
1981
|
+
socket,
|
|
1982
|
+
signal: sig,
|
|
1983
|
+
onRetry: () => {
|
|
1984
|
+
audio.teardown();
|
|
1985
|
+
updateState({
|
|
1986
|
+
...agentState.apply({ type: "CONNECT" }),
|
|
1987
|
+
recording: false
|
|
1988
|
+
});
|
|
1989
|
+
},
|
|
1990
|
+
onExhausted: () => {
|
|
1991
|
+
preConnect.release();
|
|
1992
|
+
audio.teardown();
|
|
1993
|
+
controller.abort();
|
|
1994
|
+
socket.close();
|
|
1995
|
+
conn.ws = null;
|
|
1996
|
+
updateState({
|
|
1997
|
+
...agentState.apply({
|
|
1998
|
+
type: "FAILED",
|
|
1999
|
+
error: HANDSHAKE_ERROR
|
|
2000
|
+
}),
|
|
2001
|
+
...STOPPED
|
|
2002
|
+
});
|
|
2003
|
+
}
|
|
2004
|
+
});
|
|
2005
|
+
socket.addEventListener("open", () => {
|
|
2006
|
+
updateState(agentState.apply({ type: "SOCKET_OPEN" }));
|
|
2007
|
+
handshake.arm();
|
|
2008
|
+
}, { signal: sig });
|
|
2009
|
+
socket.addEventListener("message", (event) => {
|
|
2010
|
+
const config = handleMessage(event.data);
|
|
2011
|
+
if (!config) return;
|
|
2012
|
+
handshake.succeeded();
|
|
2013
|
+
onServerConfig(config);
|
|
2014
|
+
}, { signal: sig });
|
|
2015
|
+
socket.addEventListener("error", () => {
|
|
2016
|
+
socketErrored = true;
|
|
2017
|
+
}, { signal: sig });
|
|
2018
|
+
socket.addEventListener("close", (event) => {
|
|
2019
|
+
if (sig.aborted) return;
|
|
2020
|
+
handshake.disarm();
|
|
2021
|
+
audio.teardown();
|
|
2022
|
+
if (!(conn.retiredByServer || agentState.fatal()) && reconnectPending(socket)) {
|
|
2023
|
+
socketErrored = false;
|
|
2024
|
+
updateState({
|
|
2025
|
+
...agentState.apply({ type: "CONNECT" }),
|
|
2026
|
+
recording: false
|
|
2027
|
+
});
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
2030
|
+
preConnect.release();
|
|
2031
|
+
controller.abort();
|
|
2032
|
+
socket.close();
|
|
2033
|
+
conn.ws = null;
|
|
2034
|
+
const failure = closeFailure(event, socketErrored);
|
|
2035
|
+
updateState({
|
|
2036
|
+
...failure === null ? agentState.apply({ type: "CLOSED" }) : agentState.apply({
|
|
2037
|
+
type: "FAILED",
|
|
2038
|
+
error: {
|
|
2039
|
+
code: "connection",
|
|
2040
|
+
message: failure,
|
|
2041
|
+
fatal: false
|
|
2042
|
+
}
|
|
2043
|
+
}),
|
|
2044
|
+
...STOPPED
|
|
2045
|
+
});
|
|
2046
|
+
}, { signal: sig });
|
|
2047
|
+
}
|
|
2048
|
+
function cancel() {
|
|
2049
|
+
if (!openSocket()) return;
|
|
2050
|
+
bargeIn(conn, audio);
|
|
2051
|
+
updateState(agentState.apply({ type: "LISTEN" }));
|
|
2052
|
+
sendJson({ type: "cancel" });
|
|
2053
|
+
}
|
|
2054
|
+
const { userTurn, sendText, flushQueued, sendToolResult } = createUserInput({
|
|
2055
|
+
snapshot: () => currentSnapshot,
|
|
2056
|
+
connected: () => openSocket() !== null,
|
|
2057
|
+
queued: queuedText,
|
|
2058
|
+
open: () => {
|
|
2059
|
+
if (!currentSnapshot.running) (currentSnapshot.started ? toggle : start)();
|
|
2060
|
+
},
|
|
2061
|
+
bargeIn: () => bargeIn(conn, audio),
|
|
2062
|
+
agentState,
|
|
2063
|
+
updateState,
|
|
2064
|
+
sendJson
|
|
2065
|
+
});
|
|
2066
|
+
function reset() {
|
|
2067
|
+
bargeIn(conn, audio);
|
|
2068
|
+
if (openSocket()) {
|
|
2069
|
+
sendJson({ type: "reset" });
|
|
2070
|
+
return;
|
|
2071
|
+
}
|
|
2072
|
+
end();
|
|
2073
|
+
start();
|
|
2074
|
+
}
|
|
2075
|
+
function disconnect() {
|
|
2076
|
+
teardownConnection();
|
|
2077
|
+
updateState({
|
|
2078
|
+
...agentState.apply({ type: "DISCONNECT" }),
|
|
2079
|
+
...STOPPED
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
function start() {
|
|
2083
|
+
updateState({
|
|
2084
|
+
started: true,
|
|
2085
|
+
running: true
|
|
2086
|
+
});
|
|
2087
|
+
connect();
|
|
2088
|
+
}
|
|
2089
|
+
function toggle() {
|
|
2090
|
+
if (currentSnapshot.running) disconnect();
|
|
2091
|
+
else {
|
|
2092
|
+
updateState({ running: true });
|
|
2093
|
+
connect();
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
function end() {
|
|
2097
|
+
teardownConnection();
|
|
2098
|
+
dialer.forget();
|
|
2099
|
+
updateState({
|
|
2100
|
+
...CLEARED_SESSION_STATE,
|
|
2101
|
+
...agentState.apply({ type: "END" }),
|
|
2102
|
+
started: false,
|
|
2103
|
+
...STOPPED
|
|
2104
|
+
});
|
|
2105
|
+
}
|
|
2106
|
+
function restart() {
|
|
2107
|
+
end();
|
|
2108
|
+
start();
|
|
2109
|
+
}
|
|
2110
|
+
function resume(sessionId) {
|
|
2111
|
+
if (!RESUME_ID_RE.test(sessionId)) throw new RangeError(`resume: "${sessionId}" is not a session id (${RESUME_ID_RE})`);
|
|
2112
|
+
end();
|
|
2113
|
+
dialer.adopt(sessionId);
|
|
2114
|
+
start();
|
|
2115
|
+
}
|
|
2116
|
+
return {
|
|
2117
|
+
getSnapshot,
|
|
2118
|
+
subscribe,
|
|
2119
|
+
connect,
|
|
2120
|
+
cancel,
|
|
2121
|
+
userTurn: Object.freeze(userTurn),
|
|
2122
|
+
identity,
|
|
2123
|
+
sendText,
|
|
2124
|
+
sendToolResult,
|
|
2125
|
+
setMicMuted: mic.setMicMuted,
|
|
2126
|
+
resetState,
|
|
2127
|
+
reset,
|
|
2128
|
+
disconnect,
|
|
2129
|
+
start,
|
|
2130
|
+
toggle,
|
|
2131
|
+
end,
|
|
2132
|
+
restart,
|
|
2133
|
+
resume,
|
|
2134
|
+
[Symbol.dispose]() {
|
|
2135
|
+
disconnect();
|
|
2136
|
+
}
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
2139
|
+
//#endregion
|
|
2140
|
+
export { ticketCarriage as a, resolveReported as c, storageSet as d, urlSlot as f, resolveSessionTokenSync as i, storageGet as l, CLEARED_SESSION_STATE as n, loadAudioModules as o, resolveSessionToken as r, browserClientId as s, createBrowserSession as t, storageRemove as u };
|