@iloveagents/foundry-web-voice 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +157 -0
  3. package/dist/adapter/half-duplex.d.ts +42 -0
  4. package/dist/adapter/half-duplex.js +65 -0
  5. package/dist/adapter/session-config.d.ts +55 -0
  6. package/dist/adapter/session-config.js +148 -0
  7. package/dist/adapter/speech-queue.d.ts +95 -0
  8. package/dist/adapter/speech-queue.js +344 -0
  9. package/dist/adapter/tool-bridge.d.ts +25 -0
  10. package/dist/adapter/tool-bridge.js +37 -0
  11. package/dist/adapter/tool-sync.d.ts +46 -0
  12. package/dist/adapter/tool-sync.js +55 -0
  13. package/dist/adapter/types.d.ts +91 -0
  14. package/dist/adapter/types.js +8 -0
  15. package/dist/adapter/voice-bridge.d.ts +214 -0
  16. package/dist/adapter/voice-bridge.js +539 -0
  17. package/dist/index.d.ts +28 -0
  18. package/dist/index.js +30 -0
  19. package/dist/react/audio-ownership.d.ts +41 -0
  20. package/dist/react/audio-ownership.js +37 -0
  21. package/dist/react/install.d.ts +62 -0
  22. package/dist/react/install.js +100 -0
  23. package/dist/react/relay-answer-watcher.d.ts +16 -0
  24. package/dist/react/relay-answer-watcher.js +45 -0
  25. package/dist/react/use-direct-audio-output.d.ts +23 -0
  26. package/dist/react/use-direct-audio-output.js +44 -0
  27. package/dist/react/voice-audio-sink.d.ts +21 -0
  28. package/dist/react/voice-audio-sink.js +55 -0
  29. package/dist/react/voice-avatar.d.ts +61 -0
  30. package/dist/react/voice-avatar.js +76 -0
  31. package/dist/react/voice-launcher-badge.d.ts +14 -0
  32. package/dist/react/voice-launcher-badge.js +36 -0
  33. package/dist/react/voice-mic-button.d.ts +19 -0
  34. package/dist/react/voice-mic-button.js +53 -0
  35. package/dist/react/voice-module.d.ts +34 -0
  36. package/dist/react/voice-module.js +25 -0
  37. package/dist/react/voice-stage.d.ts +20 -0
  38. package/dist/react/voice-stage.js +64 -0
  39. package/dist/react/voice-status-strip.d.ts +10 -0
  40. package/dist/react/voice-status-strip.js +31 -0
  41. package/dist/react/voice-surface.d.ts +27 -0
  42. package/dist/react/voice-surface.js +289 -0
  43. package/dist/react/voice-ui-store.d.ts +58 -0
  44. package/dist/react/voice-ui-store.js +46 -0
  45. package/dist/react/voice-visualizer.d.ts +37 -0
  46. package/dist/react/voice-visualizer.js +222 -0
  47. package/package.json +71 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 iLoveAgents, a brand of Leitwolf GmbH
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,157 @@
1
+ # @iloveagents/foundry-web-voice
2
+
3
+ Spoken conversation for [Foundry UI](https://github.com/iLoveAgents/foundry-ui), backed by
4
+ [Microsoft Foundry Voice Live](https://learn.microsoft.com/azure/ai-services/speech-service/voice-live).
5
+
6
+ Optional by construction: no other `@iloveagents/foundry-*` package imports this one, so an app
7
+ that never installs it has an unchanged composer and an unchanged runtime.
8
+
9
+ ```bash
10
+ npm install @iloveagents/foundry-web-voice
11
+ ```
12
+
13
+ ```ts
14
+ import { bootstrapShell } from "@iloveagents/foundry-web-shell";
15
+ import { createVoiceModule } from "@iloveagents/foundry-web-voice";
16
+
17
+ bootstrapShell({
18
+ modules: [
19
+ createVoiceModule({
20
+ connection: { proxyUrl: import.meta.env.VITE_VOICE_PROXY_URL },
21
+ }),
22
+ ],
23
+ });
24
+ ```
25
+
26
+ That is the whole integration. A voice button appears in the composer (holding the Send corner
27
+ until there is something to send), and while a call is live the conversation is replaced by a
28
+ theme-aware visualiser — or the avatar, if you configure one — and comes straight back when the
29
+ call ends. Pass `stage: false` to keep the transcript visible instead.
30
+
31
+ ## The two modes
32
+
33
+ | | `relay` (default) | `realtime` |
34
+ | ----------------------- | ----------------------------------------------------- | ------------------------------- |
35
+ | Who answers | **your AG-UI agent** — retrieval, tools, citations | the Voice Live model |
36
+ | Voice Live's job | speech in, speech out (and the avatar) | the whole conversation |
37
+ | Turns in the thread | ordinary turns — tool cards, citations, **persisted** | live transcript, session-scoped |
38
+ | First word of the reply | as fast as your agent | sub-second |
39
+
40
+ **`relay`** keeps the answer yours. Voice Live transcribes what the user said, the transcript goes
41
+ through the composer exactly as if it had been typed, your agent answers with its full tool and
42
+ retrieval surface, and the reply is spoken back in the Azure voice you configured. Under the hood
43
+ that is `turn_detection.create_response: false` plus `pre_generated_assistant_message` — both
44
+ documented Voice Live features, not tricks.
45
+
46
+ **`realtime`** hands the conversation to the Voice Live model for the lowest possible latency. It
47
+ can still call your client-side tools: the schemas in `clientToolRegistry` are sent with the
48
+ session, and a call is dispatched through the same registry the typed chat uses, so `ui_navigate`
49
+ and your `page_*` tools work by voice with no second declaration.
50
+
51
+ ```ts
52
+ createVoiceModule({
53
+ mode: "realtime",
54
+ connection: { proxyUrl: VOICE_PROXY_URL },
55
+ session: { instructions: "You are a helpful assistant. Keep answers to one or two sentences." },
56
+ });
57
+ ```
58
+
59
+ ### One thing to know about `realtime` and history
60
+
61
+ assistant-ui merges realtime transcripts into the thread while the session is live and clears them
62
+ on disconnect (`disconnectVoice()` empties `_voiceMessages`). They are never handed to a
63
+ `ThreadHistoryAdapter`. That is the framework's design for realtime voice, not a gap here — and it
64
+ is why `relay` is the default. Pass `onTranscript` if you want to persist them yourself.
65
+
66
+ ## An avatar
67
+
68
+ Configure one on the session and the panel renders itself once the video track arrives:
69
+
70
+ ```ts
71
+ import { sessionConfig } from "@iloveagents/foundry-voice-live-react";
72
+
73
+ createVoiceModule({
74
+ connection: { proxyUrl: VOICE_PROXY_URL },
75
+ session: sessionConfig()
76
+ .hdVoice("en-US-Ava:DragonHDLatestNeural")
77
+ .avatar("lisa", "casual-sitting", { codec: "h264" })
78
+ .build(),
79
+ });
80
+ ```
81
+
82
+ In `relay` mode the avatar lip-syncs your own agent's answer, because the words are sent as a
83
+ pre-generated assistant message rather than generated by the Voice Live model.
84
+
85
+ ## Options that matter in a room
86
+
87
+ - `halfDuplex: true` — mute the microphone while the assistant speaks, so on laptop speakers it
88
+ cannot hear itself and answer its own voice. Costs barge-in; right on speakerphones, wrong with
89
+ headsets. It never overrides a mute the user set themselves.
90
+ - `chromaKey` — tune the avatar's green-screen removal. The default cuts a _tighter_ edge than the
91
+ SDK's, which left a green fringe on the silhouette in live runs. Tighter, not softer: the subject
92
+ sits closer to the key colour than it looks, so widening the blend band does not soften the edge —
93
+ it turns the whole figure translucent.
94
+ - `stage: false` — keep the thread visible during a call instead of replacing it.
95
+
96
+ ## You need a proxy
97
+
98
+ A browser cannot set an `Authorization` header on a WebSocket, so the credential has to live
99
+ somewhere else. Run the SDK's proxy and point `proxyUrl` at it:
100
+
101
+ ```bash
102
+ docker run -p 8080:8080 \
103
+ -e FOUNDRY_RESOURCE_NAME=your-foundry-resource \
104
+ -e FOUNDRY_API_KEY="…" \
105
+ -e ALLOWED_ORIGINS="https://your-app.example.com" \
106
+ ghcr.io/iloveagents/foundry-voice-live-proxy:latest
107
+ ```
108
+
109
+ `connection` accepts everything
110
+ [`@iloveagents/foundry-voice-live-react`](https://www.npmjs.com/package/@iloveagents/foundry-voice-live-react)
111
+ does, including a per-user MSAL token and Foundry Agent Service. An `apiKey` in the browser is for
112
+ local development only.
113
+
114
+ > The default model (`gpt-realtime`) is available in **East US 2** and **Sweden Central**.
115
+
116
+ ## Without the shell
117
+
118
+ If you compose the runtime yourself, render `FoundryVoice` anywhere inside `AGUIRuntimeProvider`:
119
+
120
+ ```tsx
121
+ <AGUIRuntimeProvider>
122
+ <FoundryVoice connection={{ proxyUrl: VOICE_PROXY_URL }} />
123
+ <ChatContent />
124
+ </AGUIRuntimeProvider>
125
+ ```
126
+
127
+ Pass `chrome: false` to keep the composer untouched and place `VoiceMicButton` /
128
+ `VoiceStatusStrip` wherever you want them.
129
+
130
+ ## How it fits together
131
+
132
+ assistant-ui 0.15 already models realtime voice: `useLocalRuntime` accepts a
133
+ `RealtimeVoiceAdapter`, the thread runtime merges the transcripts it emits into `thread.messages`,
134
+ and `useVoiceControls()` / `useVoiceState()` / `useVoiceVolume()` expose the session. This package
135
+ implements that one interface over Voice Live — which is why the mic button is ordinary
136
+ assistant-ui code and would work for any other voice backend registered the same way.
137
+
138
+ `foundry-web-ui` contributes only the seams: `registerVoiceAdapter()` and a `composerActions` chat
139
+ slot. Neither mentions Voice Live.
140
+
141
+ ## API
142
+
143
+ | Export | What it is |
144
+ | -------------------------------------------------------- | ------------------------------------------------------------------------------- |
145
+ | `createVoiceModule(config)` | a `bootstrapShell` module |
146
+ | `FoundryVoice` | the same thing as a component, for shell-less apps |
147
+ | `installVoice(config)` | `{ bridge, Host, dispose }` for full control |
148
+ | `VoiceMicButton`, `VoiceStatusStrip`, `VoiceAvatarPanel` | the chrome, for `chrome: false` |
149
+ | `VoiceBridge` | the protocol layer — no components or hooks; driven by `VoiceTransportControls` |
150
+ | `buildVoiceSession`, `toVoiceLiveTools` | mode → Voice Live session; registry → function tools |
151
+ | `SentenceStream`, `SpeechQueue`, `toSpeakableText` | the markdown-to-speech pipeline |
152
+ | `VoiceStage`, `VoiceVisualizer` | the in-call surface and its theme-aware canvas |
153
+ | `HalfDuplexGate` | when the mic closes/opens around assistant speech (pure, tested) |
154
+
155
+ ## License
156
+
157
+ MIT
@@ -0,0 +1,42 @@
1
+ /**
2
+ * `HalfDuplexGate` — decides when the microphone opens and closes while the
3
+ * assistant is speaking.
4
+ *
5
+ * Half-duplex exists for speakers: the assistant's own voice trips the
6
+ * service's VAD and it starts answering itself. The naive version — "mute
7
+ * while speaking, unmute after" — has two failure modes that only show up
8
+ * with a person in the loop, which is why this is a pure machine with a test
9
+ * for each of them rather than a pair of booleans inside a React effect:
10
+ *
11
+ * - **The mic must never end up hot behind a muted UI.** If the user mutes
12
+ * *while* the auto-mute is engaged, a release that only remembers "I
13
+ * muted" un-mutes on their behalf: the runtime still shows muted, the SDK
14
+ * is capturing, and the user is being recorded while the button says they
15
+ * are not. Releasing therefore also requires that the user has not asked
16
+ * for mute in the meantime.
17
+ *
18
+ * - **An explicit unmute is a decision, not a glitch.** If the user unmutes
19
+ * mid-answer they are choosing barge-in over echo protection; re-muting
20
+ * them on the next tick makes interruption impossible. The gate engages
21
+ * only on the *transition* into speaking, so a mid-answer unmute stands
22
+ * until the next assistant turn.
23
+ */
24
+ export interface HalfDuplexInputs {
25
+ /** The feature switch (`VoiceConfig.halfDuplex`). */
26
+ enabled: boolean;
27
+ /** The assistant is currently speaking. */
28
+ speaking: boolean;
29
+ /** The SDK's capture-mute state (what the microphone is actually doing). */
30
+ sdkMuted: boolean;
31
+ /** The user's own intent, as expressed through the runtime's mute control. */
32
+ userMuted: boolean;
33
+ }
34
+ export type HalfDuplexAction = "mute" | "unmute" | "none";
35
+ export declare class HalfDuplexGate {
36
+ /** True while a mute this gate applied is outstanding. */
37
+ private autoMuted;
38
+ private wasSpeaking;
39
+ next({ enabled, speaking, sdkMuted, userMuted }: HalfDuplexInputs): HalfDuplexAction;
40
+ /** A session ended; whatever was outstanding no longer is. */
41
+ reset(): void;
42
+ }
@@ -0,0 +1,65 @@
1
+ /**
2
+ * `HalfDuplexGate` — decides when the microphone opens and closes while the
3
+ * assistant is speaking.
4
+ *
5
+ * Half-duplex exists for speakers: the assistant's own voice trips the
6
+ * service's VAD and it starts answering itself. The naive version — "mute
7
+ * while speaking, unmute after" — has two failure modes that only show up
8
+ * with a person in the loop, which is why this is a pure machine with a test
9
+ * for each of them rather than a pair of booleans inside a React effect:
10
+ *
11
+ * - **The mic must never end up hot behind a muted UI.** If the user mutes
12
+ * *while* the auto-mute is engaged, a release that only remembers "I
13
+ * muted" un-mutes on their behalf: the runtime still shows muted, the SDK
14
+ * is capturing, and the user is being recorded while the button says they
15
+ * are not. Releasing therefore also requires that the user has not asked
16
+ * for mute in the meantime.
17
+ *
18
+ * - **An explicit unmute is a decision, not a glitch.** If the user unmutes
19
+ * mid-answer they are choosing barge-in over echo protection; re-muting
20
+ * them on the next tick makes interruption impossible. The gate engages
21
+ * only on the *transition* into speaking, so a mid-answer unmute stands
22
+ * until the next assistant turn.
23
+ */
24
+ export class HalfDuplexGate {
25
+ constructor() {
26
+ /** True while a mute this gate applied is outstanding. */
27
+ this.autoMuted = false;
28
+ this.wasSpeaking = false;
29
+ }
30
+ next({ enabled, speaking, sdkMuted, userMuted }) {
31
+ if (!enabled) {
32
+ // Turning the feature off mid-session must not strand an auto-mute.
33
+ const release = this.autoMuted && sdkMuted && !userMuted;
34
+ this.autoMuted = false;
35
+ this.wasSpeaking = speaking;
36
+ return release ? "unmute" : "none";
37
+ }
38
+ const startedSpeaking = speaking && !this.wasSpeaking;
39
+ this.wasSpeaking = speaking;
40
+ if (speaking) {
41
+ // Edge-triggered: engage once per assistant turn. A level-triggered
42
+ // gate would instantly undo a user's mid-answer unmute, and with it
43
+ // any possibility of barging in.
44
+ if (startedSpeaking && !sdkMuted && !userMuted) {
45
+ this.autoMuted = true;
46
+ return "mute";
47
+ }
48
+ return "none";
49
+ }
50
+ if (this.autoMuted) {
51
+ this.autoMuted = false;
52
+ // Release only what this gate took. If the user asked for mute while
53
+ // the auto-mute was engaged, unmuting here would leave the microphone
54
+ // hot behind a UI that says muted.
55
+ if (!userMuted && sdkMuted)
56
+ return "unmute";
57
+ }
58
+ return "none";
59
+ }
60
+ /** A session ended; whatever was outstanding no longer is. */
61
+ reset() {
62
+ this.autoMuted = false;
63
+ this.wasSpeaking = false;
64
+ }
65
+ }
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Derive the Voice Live session from the chosen mode.
3
+ *
4
+ * Two settings carry the whole behavioural difference between the modes, and
5
+ * both are documented Voice Live features rather than anything clever:
6
+ *
7
+ * - `turn_detection.create_response` — "Enable or disable whether a
8
+ * response is generated (default: true)". `relay` sets it false, so the
9
+ * Voice Live model never answers; the app's AG-UI agent does, and the
10
+ * answer is spoken back with `pre_generated_assistant_message`.
11
+ * - `tools` — only `realtime` gets them, because only there is the Voice
12
+ * Live model the one deciding to call something.
13
+ *
14
+ * `input_audio_transcription` is forced on for both modes. The SDK's
15
+ * `DEFAULT_SESSION_CONFIG` omits it, and without it no user transcript ever
16
+ * arrives — which silently reduces `relay` to a microphone that does
17
+ * nothing and `realtime` to an assistant-only transcript.
18
+ */
19
+ import type { FunctionTool, Tool, VoiceLiveSessionConfig } from "@iloveagents/foundry-voice-live-react";
20
+ import type { ToolRegistry } from "@iloveagents/foundry-agent";
21
+ import type { VoiceMode } from "./types.ts";
22
+ /**
23
+ * AG-UI tool schemas → Voice Live function tools.
24
+ *
25
+ * The two shapes are the same object plus a discriminator, which is the
26
+ * point: a tool the typed chat can call is callable by voice with no second
27
+ * declaration and no drift between them.
28
+ */
29
+ export declare function toVoiceLiveTools(registry: ToolRegistry): FunctionTool[];
30
+ export interface BuildVoiceSessionOptions {
31
+ mode: VoiceMode;
32
+ /** Host overrides, applied last — an explicit value always wins. */
33
+ overrides?: VoiceLiveSessionConfig;
34
+ /** Present only when the host opted into exposing client tools in `realtime`. */
35
+ registry?: ToolRegistry;
36
+ }
37
+ /**
38
+ * Build the session config for a mode.
39
+ *
40
+ * Precedence is deliberate and shallow-per-key: mode defaults, then host
41
+ * overrides. `turnDetection` merges key-by-key so a host can retune VAD
42
+ * (threshold, eagerness, filler words) without having to re-state
43
+ * `createResponse` and silently break the mode.
44
+ */
45
+ export declare function buildVoiceSession({ mode, overrides, registry, }: BuildVoiceSessionOptions): VoiceLiveSessionConfig;
46
+ /**
47
+ * The tool catalogue a session carries: the registry's client tools merged
48
+ * with whatever the host declared.
49
+ *
50
+ * Exported because `session.update` REPLACES the catalogue rather than adding
51
+ * to it, so the live re-sync has to rebuild the very same list this file
52
+ * builds at connect time. Two places deriving it independently is precisely
53
+ * how the host's MCP tools went missing the moment a session became ready.
54
+ */
55
+ export declare function sessionTools(registry: ToolRegistry | undefined, hostTools: Tool[] | undefined): Tool[] | undefined;
@@ -0,0 +1,148 @@
1
+ /**
2
+ * Derive the Voice Live session from the chosen mode.
3
+ *
4
+ * Two settings carry the whole behavioural difference between the modes, and
5
+ * both are documented Voice Live features rather than anything clever:
6
+ *
7
+ * - `turn_detection.create_response` — "Enable or disable whether a
8
+ * response is generated (default: true)". `relay` sets it false, so the
9
+ * Voice Live model never answers; the app's AG-UI agent does, and the
10
+ * answer is spoken back with `pre_generated_assistant_message`.
11
+ * - `tools` — only `realtime` gets them, because only there is the Voice
12
+ * Live model the one deciding to call something.
13
+ *
14
+ * `input_audio_transcription` is forced on for both modes. The SDK's
15
+ * `DEFAULT_SESSION_CONFIG` omits it, and without it no user transcript ever
16
+ * arrives — which silently reduces `relay` to a microphone that does
17
+ * nothing and `realtime` to an assistant-only transcript.
18
+ */
19
+ /**
20
+ * Transcription model. `mai-transcribe` is the Voice Live model that works
21
+ * with every model and agent, rather than only the OpenAI-native ones.
22
+ */
23
+ const DEFAULT_TRANSCRIPTION_MODEL = "mai-transcribe";
24
+ /**
25
+ * Green screen, so the avatar can be keyed out and sit on the app's own
26
+ * background.
27
+ *
28
+ * `VoiceLiveAvatar` chroma-keys by default, but only the *session* can ask
29
+ * Azure to render a keyable background — without this the service composites
30
+ * the avatar onto opaque white, the chroma key finds no green, and the
31
+ * result is a white rectangle pasted over the page.
32
+ */
33
+ const GREEN_SCREEN = "#00FF00FF";
34
+ /**
35
+ * AG-UI tool schemas → Voice Live function tools.
36
+ *
37
+ * The two shapes are the same object plus a discriminator, which is the
38
+ * point: a tool the typed chat can call is callable by voice with no second
39
+ * declaration and no drift between them.
40
+ */
41
+ export function toVoiceLiveTools(registry) {
42
+ return registry.getActiveSchemas().map((schema) => ({
43
+ type: "function",
44
+ name: schema.name,
45
+ description: schema.description ?? "",
46
+ parameters: (schema.parameters ?? {}),
47
+ }));
48
+ }
49
+ /**
50
+ * Build the session config for a mode.
51
+ *
52
+ * Precedence is deliberate and shallow-per-key: mode defaults, then host
53
+ * overrides. `turnDetection` merges key-by-key so a host can retune VAD
54
+ * (threshold, eagerness, filler words) without having to re-state
55
+ * `createResponse` and silently break the mode.
56
+ */
57
+ export function buildVoiceSession({ mode, overrides, registry, }) {
58
+ const isRelay = mode === "relay";
59
+ const tools = registry ? toVoiceLiveTools(registry) : undefined;
60
+ const base = {
61
+ modalities: ["text", "audio"],
62
+ inputAudioTranscription: { model: DEFAULT_TRANSCRIPTION_MODEL },
63
+ turnDetection: {
64
+ type: "azure_semantic_vad",
65
+ // Barge-in, and keep the server's transcript honest about what was
66
+ // actually heard before the interruption.
67
+ interruptResponse: true,
68
+ autoTruncate: true,
69
+ createResponse: !isRelay,
70
+ },
71
+ };
72
+ // An empty tool list is not the same as "no tools": sending `tools: []`
73
+ // states there are none, which is right for relay and wrong to send when
74
+ // the host simply did not opt in.
75
+ if (tools !== undefined) {
76
+ base.tools = tools;
77
+ base.toolChoice = "auto";
78
+ }
79
+ if (!overrides)
80
+ return withKeyableAvatarBackground(base);
81
+ return withKeyableAvatarBackground({
82
+ ...base,
83
+ ...overrides,
84
+ // Tools MERGE rather than replace. A host declaring an MCP server or a
85
+ // Foundry agent tool is adding server-side capability, not withdrawing
86
+ // the browser's — and a plain spread would silently drop every client
87
+ // tool the moment one was declared, leaving the model unable to touch
88
+ // the UI with no error to explain why.
89
+ tools: sessionTools(registry, overrides.tools),
90
+ turnDetection: overrides.turnDetection === undefined
91
+ ? base.turnDetection
92
+ : overrides.turnDetection === null
93
+ ? null
94
+ : { ...base.turnDetection, ...overrides.turnDetection },
95
+ });
96
+ }
97
+ /**
98
+ * Give a configured avatar a keyable background unless the host chose one.
99
+ *
100
+ * A host that wants the avatar composited on a colour or an image says so and
101
+ * is left alone; the default is the one that composes with any theme.
102
+ */
103
+ function withKeyableAvatarBackground(session) {
104
+ const avatar = session.avatar;
105
+ if (!avatar || avatar.video?.background !== undefined)
106
+ return session;
107
+ return {
108
+ ...session,
109
+ avatar: {
110
+ ...avatar,
111
+ video: { ...avatar.video, background: { color: GREEN_SCREEN } },
112
+ },
113
+ };
114
+ }
115
+ /**
116
+ * The tool catalogue a session carries: the registry's client tools merged
117
+ * with whatever the host declared.
118
+ *
119
+ * Exported because `session.update` REPLACES the catalogue rather than adding
120
+ * to it, so the live re-sync has to rebuild the very same list this file
121
+ * builds at connect time. Two places deriving it independently is precisely
122
+ * how the host's MCP tools went missing the moment a session became ready.
123
+ */
124
+ export function sessionTools(registry, hostTools) {
125
+ return mergeTools(registry ? toVoiceLiveTools(registry) : undefined, hostTools);
126
+ }
127
+ /**
128
+ * Combine the registry's function tools with whatever the host declared.
129
+ *
130
+ * Function tools are keyed by name so a host can deliberately override one
131
+ * of the registry's; everything else (MCP servers, Foundry agent tools) is
132
+ * additive, since those are distinct capabilities rather than replacements.
133
+ */
134
+ function mergeTools(base, extra) {
135
+ if (!base)
136
+ return extra;
137
+ if (!extra)
138
+ return base;
139
+ const byName = new Map();
140
+ const others = [];
141
+ for (const tool of [...base, ...extra]) {
142
+ if (tool.type === "function")
143
+ byName.set(tool.name, tool);
144
+ else
145
+ others.push(tool);
146
+ }
147
+ return [...byName.values(), ...others];
148
+ }
@@ -0,0 +1,95 @@
1
+ /**
2
+ * Turning a streaming markdown answer into spoken audio.
3
+ *
4
+ * Three separate problems, kept separate:
5
+ *
6
+ * 1. **What to say.** The agent writes markdown for a reader. Read aloud
7
+ * verbatim it becomes "asterisk asterisk important asterisk asterisk",
8
+ * and foundry's `[1]` citation markers become "bracket one". `toSpeakableText`
9
+ * strips the notation and keeps the prose.
10
+ *
11
+ * 2. **When to say it.** Waiting for the whole answer wastes the seconds the
12
+ * agent spends writing it. `SentenceStream` takes cumulative text and
13
+ * hands back complete sentences as they finish, so speech starts on the
14
+ * first full sentence.
15
+ *
16
+ * 3. **One at a time.** Voice Live rejects overlapping responses. The SDK's
17
+ * own `ResponseGate` serializes them, but it deliberately collapses
18
+ * everything queued during one response into a *single* follow-up — the
19
+ * right behaviour for turns, fatal for a sentence queue, where
20
+ * collapsing three pending sentences into one would drop two of them.
21
+ * `SpeechQueue` therefore keeps its own FIFO and releases the next
22
+ * utterance only once the previous response is done.
23
+ */
24
+ /**
25
+ * Markdown → text a voice can read.
26
+ *
27
+ * Deliberately lossy: code blocks and table rules are dropped rather than
28
+ * spelled out, because hearing punctuation read aloud is worse than not
29
+ * hearing it at all. What survives is the prose.
30
+ */
31
+ export declare function toSpeakableText(markdown: string, options?: {
32
+ final?: boolean;
33
+ }): string;
34
+ /**
35
+ * Cuts cumulative streaming text into speakable utterances.
36
+ *
37
+ * Fed the *whole* answer so far on each call (which is what assistant-ui
38
+ * exposes — parts carry cumulative text, not deltas) and returns only what
39
+ * has not been emitted yet.
40
+ */
41
+ export declare class SentenceStream {
42
+ /** How much of the answer has been turned into emitted utterances. */
43
+ private consumed;
44
+ /** Exactly that prefix, kept so a rewrite of it can be detected. */
45
+ private consumedText;
46
+ /**
47
+ * Every position an utterance has ended at, 0 first.
48
+ *
49
+ * A rewrite inside already-spoken text can only be repaired at one of these.
50
+ * Rewinding to the raw point of divergence resumes mid-word — "…ly different
51
+ * answer" — because the divergence lands wherever the two strings happen to
52
+ * part, which is usually inside a word.
53
+ */
54
+ private boundaries;
55
+ /** The last cumulative answer seen, kept so `flush()` can finish it. */
56
+ private text;
57
+ /**
58
+ * @param cumulative the full answer so far, already run through `toSpeakableText`
59
+ * @returns complete utterances that became available with this update
60
+ */
61
+ push(cumulative: string): string[];
62
+ /** Emit whatever is left, complete sentence or not. Call when the answer ends. */
63
+ flush(): string[];
64
+ reset(): void;
65
+ private take;
66
+ private advanceTo;
67
+ }
68
+ /**
69
+ * A strict FIFO of utterances, one on the wire at a time.
70
+ *
71
+ * `send` puts one utterance on the wire. `onResponseDone` must be called
72
+ * when the service reports that response finished — that is the only thing
73
+ * that releases the next one.
74
+ */
75
+ export declare class SpeechQueue {
76
+ private readonly send;
77
+ private readonly pending;
78
+ private speaking;
79
+ constructor(send: (text: string) => void);
80
+ get isSpeaking(): boolean;
81
+ get pendingCount(): number;
82
+ enqueue(text: string): void;
83
+ /** The in-flight response finished; release the next utterance. */
84
+ onResponseDone(): void;
85
+ /**
86
+ * Barge-in or a new turn: forget everything still queued.
87
+ *
88
+ * `speaking` is cleared too. The caller cancels the in-flight response on
89
+ * the wire, and a cancelled response may never produce the `response.done`
90
+ * that would otherwise release the queue — leaving it wedged shut for the
91
+ * rest of the session.
92
+ */
93
+ cancel(): void;
94
+ private pump;
95
+ }