@iloveagents/foundry-web-voice 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -0
- package/dist/adapter/half-duplex.d.ts +42 -0
- package/dist/adapter/half-duplex.js +65 -0
- package/dist/adapter/session-config.d.ts +55 -0
- package/dist/adapter/session-config.js +148 -0
- package/dist/adapter/speech-queue.d.ts +95 -0
- package/dist/adapter/speech-queue.js +344 -0
- package/dist/adapter/tool-bridge.d.ts +25 -0
- package/dist/adapter/tool-bridge.js +37 -0
- package/dist/adapter/tool-sync.d.ts +46 -0
- package/dist/adapter/tool-sync.js +55 -0
- package/dist/adapter/types.d.ts +91 -0
- package/dist/adapter/types.js +8 -0
- package/dist/adapter/voice-bridge.d.ts +214 -0
- package/dist/adapter/voice-bridge.js +539 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +30 -0
- package/dist/react/audio-ownership.d.ts +41 -0
- package/dist/react/audio-ownership.js +37 -0
- package/dist/react/install.d.ts +62 -0
- package/dist/react/install.js +100 -0
- package/dist/react/relay-answer-watcher.d.ts +16 -0
- package/dist/react/relay-answer-watcher.js +45 -0
- package/dist/react/use-direct-audio-output.d.ts +23 -0
- package/dist/react/use-direct-audio-output.js +44 -0
- package/dist/react/voice-audio-sink.d.ts +21 -0
- package/dist/react/voice-audio-sink.js +55 -0
- package/dist/react/voice-avatar.d.ts +61 -0
- package/dist/react/voice-avatar.js +76 -0
- package/dist/react/voice-launcher-badge.d.ts +14 -0
- package/dist/react/voice-launcher-badge.js +36 -0
- package/dist/react/voice-mic-button.d.ts +19 -0
- package/dist/react/voice-mic-button.js +53 -0
- package/dist/react/voice-module.d.ts +34 -0
- package/dist/react/voice-module.js +25 -0
- package/dist/react/voice-stage.d.ts +20 -0
- package/dist/react/voice-stage.js +64 -0
- package/dist/react/voice-status-strip.d.ts +10 -0
- package/dist/react/voice-status-strip.js +31 -0
- package/dist/react/voice-surface.d.ts +27 -0
- package/dist/react/voice-surface.js +289 -0
- package/dist/react/voice-ui-store.d.ts +58 -0
- package/dist/react/voice-ui-store.js +46 -0
- package/dist/react/voice-visualizer.d.ts +37 -0
- package/dist/react/voice-visualizer.js +222 -0
- package/package.json +71 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 iLoveAgents, a brand of Leitwolf GmbH
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# @iloveagents/foundry-web-voice
|
|
2
|
+
|
|
3
|
+
Spoken conversation for [Foundry UI](https://github.com/iLoveAgents/foundry-ui), backed by
|
|
4
|
+
[Microsoft Foundry Voice Live](https://learn.microsoft.com/azure/ai-services/speech-service/voice-live).
|
|
5
|
+
|
|
6
|
+
Optional by construction: no other `@iloveagents/foundry-*` package imports this one, so an app
|
|
7
|
+
that never installs it has an unchanged composer and an unchanged runtime.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @iloveagents/foundry-web-voice
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { bootstrapShell } from "@iloveagents/foundry-web-shell";
|
|
15
|
+
import { createVoiceModule } from "@iloveagents/foundry-web-voice";
|
|
16
|
+
|
|
17
|
+
bootstrapShell({
|
|
18
|
+
modules: [
|
|
19
|
+
createVoiceModule({
|
|
20
|
+
connection: { proxyUrl: import.meta.env.VITE_VOICE_PROXY_URL },
|
|
21
|
+
}),
|
|
22
|
+
],
|
|
23
|
+
});
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
That is the whole integration. A voice button appears in the composer (holding the Send corner
|
|
27
|
+
until there is something to send), and while a call is live the conversation is replaced by a
|
|
28
|
+
theme-aware visualiser — or the avatar, if you configure one — and comes straight back when the
|
|
29
|
+
call ends. Pass `stage: false` to keep the transcript visible instead.
|
|
30
|
+
|
|
31
|
+
## The two modes
|
|
32
|
+
|
|
33
|
+
| | `relay` (default) | `realtime` |
|
|
34
|
+
| ----------------------- | ----------------------------------------------------- | ------------------------------- |
|
|
35
|
+
| Who answers | **your AG-UI agent** — retrieval, tools, citations | the Voice Live model |
|
|
36
|
+
| Voice Live's job | speech in, speech out (and the avatar) | the whole conversation |
|
|
37
|
+
| Turns in the thread | ordinary turns — tool cards, citations, **persisted** | live transcript, session-scoped |
|
|
38
|
+
| First word of the reply | as fast as your agent | sub-second |
|
|
39
|
+
|
|
40
|
+
**`relay`** keeps the answer yours. Voice Live transcribes what the user said, the transcript goes
|
|
41
|
+
through the composer exactly as if it had been typed, your agent answers with its full tool and
|
|
42
|
+
retrieval surface, and the reply is spoken back in the Azure voice you configured. Under the hood
|
|
43
|
+
that is `turn_detection.create_response: false` plus `pre_generated_assistant_message` — both
|
|
44
|
+
documented Voice Live features, not tricks.
|
|
45
|
+
|
|
46
|
+
**`realtime`** hands the conversation to the Voice Live model for the lowest possible latency. It
|
|
47
|
+
can still call your client-side tools: the schemas in `clientToolRegistry` are sent with the
|
|
48
|
+
session, and a call is dispatched through the same registry the typed chat uses, so `ui_navigate`
|
|
49
|
+
and your `page_*` tools work by voice with no second declaration.
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
createVoiceModule({
|
|
53
|
+
mode: "realtime",
|
|
54
|
+
connection: { proxyUrl: VOICE_PROXY_URL },
|
|
55
|
+
session: { instructions: "You are a helpful assistant. Keep answers to one or two sentences." },
|
|
56
|
+
});
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### One thing to know about `realtime` and history
|
|
60
|
+
|
|
61
|
+
assistant-ui merges realtime transcripts into the thread while the session is live and clears them
|
|
62
|
+
on disconnect (`disconnectVoice()` empties `_voiceMessages`). They are never handed to a
|
|
63
|
+
`ThreadHistoryAdapter`. That is the framework's design for realtime voice, not a gap here — and it
|
|
64
|
+
is why `relay` is the default. Pass `onTranscript` if you want to persist them yourself.
|
|
65
|
+
|
|
66
|
+
## An avatar
|
|
67
|
+
|
|
68
|
+
Configure one on the session and the panel renders itself once the video track arrives:
|
|
69
|
+
|
|
70
|
+
```ts
|
|
71
|
+
import { sessionConfig } from "@iloveagents/foundry-voice-live-react";
|
|
72
|
+
|
|
73
|
+
createVoiceModule({
|
|
74
|
+
connection: { proxyUrl: VOICE_PROXY_URL },
|
|
75
|
+
session: sessionConfig()
|
|
76
|
+
.hdVoice("en-US-Ava:DragonHDLatestNeural")
|
|
77
|
+
.avatar("lisa", "casual-sitting", { codec: "h264" })
|
|
78
|
+
.build(),
|
|
79
|
+
});
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
In `relay` mode the avatar lip-syncs your own agent's answer, because the words are sent as a
|
|
83
|
+
pre-generated assistant message rather than generated by the Voice Live model.
|
|
84
|
+
|
|
85
|
+
## Options that matter in a room
|
|
86
|
+
|
|
87
|
+
- `halfDuplex: true` — mute the microphone while the assistant speaks, so on laptop speakers it
|
|
88
|
+
cannot hear itself and answer its own voice. Costs barge-in; right on speakerphones, wrong with
|
|
89
|
+
headsets. It never overrides a mute the user set themselves.
|
|
90
|
+
- `chromaKey` — tune the avatar's green-screen removal. The default cuts a _tighter_ edge than the
|
|
91
|
+
SDK's, which left a green fringe on the silhouette in live runs. Tighter, not softer: the subject
|
|
92
|
+
sits closer to the key colour than it looks, so widening the blend band does not soften the edge —
|
|
93
|
+
it turns the whole figure translucent.
|
|
94
|
+
- `stage: false` — keep the thread visible during a call instead of replacing it.
|
|
95
|
+
|
|
96
|
+
## You need a proxy
|
|
97
|
+
|
|
98
|
+
A browser cannot set an `Authorization` header on a WebSocket, so the credential has to live
|
|
99
|
+
somewhere else. Run the SDK's proxy and point `proxyUrl` at it:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
docker run -p 8080:8080 \
|
|
103
|
+
-e FOUNDRY_RESOURCE_NAME=your-foundry-resource \
|
|
104
|
+
-e FOUNDRY_API_KEY="…" \
|
|
105
|
+
-e ALLOWED_ORIGINS="https://your-app.example.com" \
|
|
106
|
+
ghcr.io/iloveagents/foundry-voice-live-proxy:latest
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`connection` accepts everything
|
|
110
|
+
[`@iloveagents/foundry-voice-live-react`](https://www.npmjs.com/package/@iloveagents/foundry-voice-live-react)
|
|
111
|
+
does, including a per-user MSAL token and Foundry Agent Service. An `apiKey` in the browser is for
|
|
112
|
+
local development only.
|
|
113
|
+
|
|
114
|
+
> The default model (`gpt-realtime`) is available in **East US 2** and **Sweden Central**.
|
|
115
|
+
|
|
116
|
+
## Without the shell
|
|
117
|
+
|
|
118
|
+
If you compose the runtime yourself, render `FoundryVoice` anywhere inside `AGUIRuntimeProvider`:
|
|
119
|
+
|
|
120
|
+
```tsx
|
|
121
|
+
<AGUIRuntimeProvider>
|
|
122
|
+
<FoundryVoice connection={{ proxyUrl: VOICE_PROXY_URL }} />
|
|
123
|
+
<ChatContent />
|
|
124
|
+
</AGUIRuntimeProvider>
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Pass `chrome: false` to keep the composer untouched and place `VoiceMicButton` /
|
|
128
|
+
`VoiceStatusStrip` wherever you want them.
|
|
129
|
+
|
|
130
|
+
## How it fits together
|
|
131
|
+
|
|
132
|
+
assistant-ui 0.15 already models realtime voice: `useLocalRuntime` accepts a
|
|
133
|
+
`RealtimeVoiceAdapter`, the thread runtime merges the transcripts it emits into `thread.messages`,
|
|
134
|
+
and `useVoiceControls()` / `useVoiceState()` / `useVoiceVolume()` expose the session. This package
|
|
135
|
+
implements that one interface over Voice Live — which is why the mic button is ordinary
|
|
136
|
+
assistant-ui code and would work for any other voice backend registered the same way.
|
|
137
|
+
|
|
138
|
+
`foundry-web-ui` contributes only the seams: `registerVoiceAdapter()` and a `composerActions` chat
|
|
139
|
+
slot. Neither mentions Voice Live.
|
|
140
|
+
|
|
141
|
+
## API
|
|
142
|
+
|
|
143
|
+
| Export | What it is |
|
|
144
|
+
| -------------------------------------------------------- | ------------------------------------------------------------------------------- |
|
|
145
|
+
| `createVoiceModule(config)` | a `bootstrapShell` module |
|
|
146
|
+
| `FoundryVoice` | the same thing as a component, for shell-less apps |
|
|
147
|
+
| `installVoice(config)` | `{ bridge, Host, dispose }` for full control |
|
|
148
|
+
| `VoiceMicButton`, `VoiceStatusStrip`, `VoiceAvatarPanel` | the chrome, for `chrome: false` |
|
|
149
|
+
| `VoiceBridge` | the protocol layer — no components or hooks; driven by `VoiceTransportControls` |
|
|
150
|
+
| `buildVoiceSession`, `toVoiceLiveTools` | mode → Voice Live session; registry → function tools |
|
|
151
|
+
| `SentenceStream`, `SpeechQueue`, `toSpeakableText` | the markdown-to-speech pipeline |
|
|
152
|
+
| `VoiceStage`, `VoiceVisualizer` | the in-call surface and its theme-aware canvas |
|
|
153
|
+
| `HalfDuplexGate` | when the mic closes/opens around assistant speech (pure, tested) |
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `HalfDuplexGate` — decides when the microphone opens and closes while the
|
|
3
|
+
* assistant is speaking.
|
|
4
|
+
*
|
|
5
|
+
* Half-duplex exists for speakers: the assistant's own voice trips the
|
|
6
|
+
* service's VAD and it starts answering itself. The naive version — "mute
|
|
7
|
+
* while speaking, unmute after" — has two failure modes that only show up
|
|
8
|
+
* with a person in the loop, which is why this is a pure machine with a test
|
|
9
|
+
* for each of them rather than a pair of booleans inside a React effect:
|
|
10
|
+
*
|
|
11
|
+
* - **The mic must never end up hot behind a muted UI.** If the user mutes
|
|
12
|
+
* *while* the auto-mute is engaged, a release that only remembers "I
|
|
13
|
+
* muted" un-mutes on their behalf: the runtime still shows muted, the SDK
|
|
14
|
+
* is capturing, and the user is being recorded while the button says they
|
|
15
|
+
* are not. Releasing therefore also requires that the user has not asked
|
|
16
|
+
* for mute in the meantime.
|
|
17
|
+
*
|
|
18
|
+
* - **An explicit unmute is a decision, not a glitch.** If the user unmutes
|
|
19
|
+
* mid-answer they are choosing barge-in over echo protection; re-muting
|
|
20
|
+
* them on the next tick makes interruption impossible. The gate engages
|
|
21
|
+
* only on the *transition* into speaking, so a mid-answer unmute stands
|
|
22
|
+
* until the next assistant turn.
|
|
23
|
+
*/
|
|
24
|
+
export interface HalfDuplexInputs {
|
|
25
|
+
/** The feature switch (`VoiceConfig.halfDuplex`). */
|
|
26
|
+
enabled: boolean;
|
|
27
|
+
/** The assistant is currently speaking. */
|
|
28
|
+
speaking: boolean;
|
|
29
|
+
/** The SDK's capture-mute state (what the microphone is actually doing). */
|
|
30
|
+
sdkMuted: boolean;
|
|
31
|
+
/** The user's own intent, as expressed through the runtime's mute control. */
|
|
32
|
+
userMuted: boolean;
|
|
33
|
+
}
|
|
34
|
+
export type HalfDuplexAction = "mute" | "unmute" | "none";
|
|
35
|
+
export declare class HalfDuplexGate {
|
|
36
|
+
/** True while a mute this gate applied is outstanding. */
|
|
37
|
+
private autoMuted;
|
|
38
|
+
private wasSpeaking;
|
|
39
|
+
next({ enabled, speaking, sdkMuted, userMuted }: HalfDuplexInputs): HalfDuplexAction;
|
|
40
|
+
/** A session ended; whatever was outstanding no longer is. */
|
|
41
|
+
reset(): void;
|
|
42
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `HalfDuplexGate` — decides when the microphone opens and closes while the
|
|
3
|
+
* assistant is speaking.
|
|
4
|
+
*
|
|
5
|
+
* Half-duplex exists for speakers: the assistant's own voice trips the
|
|
6
|
+
* service's VAD and it starts answering itself. The naive version — "mute
|
|
7
|
+
* while speaking, unmute after" — has two failure modes that only show up
|
|
8
|
+
* with a person in the loop, which is why this is a pure machine with a test
|
|
9
|
+
* for each of them rather than a pair of booleans inside a React effect:
|
|
10
|
+
*
|
|
11
|
+
* - **The mic must never end up hot behind a muted UI.** If the user mutes
|
|
12
|
+
* *while* the auto-mute is engaged, a release that only remembers "I
|
|
13
|
+
* muted" un-mutes on their behalf: the runtime still shows muted, the SDK
|
|
14
|
+
* is capturing, and the user is being recorded while the button says they
|
|
15
|
+
* are not. Releasing therefore also requires that the user has not asked
|
|
16
|
+
* for mute in the meantime.
|
|
17
|
+
*
|
|
18
|
+
* - **An explicit unmute is a decision, not a glitch.** If the user unmutes
|
|
19
|
+
* mid-answer they are choosing barge-in over echo protection; re-muting
|
|
20
|
+
* them on the next tick makes interruption impossible. The gate engages
|
|
21
|
+
* only on the *transition* into speaking, so a mid-answer unmute stands
|
|
22
|
+
* until the next assistant turn.
|
|
23
|
+
*/
|
|
24
|
+
export class HalfDuplexGate {
|
|
25
|
+
constructor() {
|
|
26
|
+
/** True while a mute this gate applied is outstanding. */
|
|
27
|
+
this.autoMuted = false;
|
|
28
|
+
this.wasSpeaking = false;
|
|
29
|
+
}
|
|
30
|
+
next({ enabled, speaking, sdkMuted, userMuted }) {
|
|
31
|
+
if (!enabled) {
|
|
32
|
+
// Turning the feature off mid-session must not strand an auto-mute.
|
|
33
|
+
const release = this.autoMuted && sdkMuted && !userMuted;
|
|
34
|
+
this.autoMuted = false;
|
|
35
|
+
this.wasSpeaking = speaking;
|
|
36
|
+
return release ? "unmute" : "none";
|
|
37
|
+
}
|
|
38
|
+
const startedSpeaking = speaking && !this.wasSpeaking;
|
|
39
|
+
this.wasSpeaking = speaking;
|
|
40
|
+
if (speaking) {
|
|
41
|
+
// Edge-triggered: engage once per assistant turn. A level-triggered
|
|
42
|
+
// gate would instantly undo a user's mid-answer unmute, and with it
|
|
43
|
+
// any possibility of barging in.
|
|
44
|
+
if (startedSpeaking && !sdkMuted && !userMuted) {
|
|
45
|
+
this.autoMuted = true;
|
|
46
|
+
return "mute";
|
|
47
|
+
}
|
|
48
|
+
return "none";
|
|
49
|
+
}
|
|
50
|
+
if (this.autoMuted) {
|
|
51
|
+
this.autoMuted = false;
|
|
52
|
+
// Release only what this gate took. If the user asked for mute while
|
|
53
|
+
// the auto-mute was engaged, unmuting here would leave the microphone
|
|
54
|
+
// hot behind a UI that says muted.
|
|
55
|
+
if (!userMuted && sdkMuted)
|
|
56
|
+
return "unmute";
|
|
57
|
+
}
|
|
58
|
+
return "none";
|
|
59
|
+
}
|
|
60
|
+
/** A session ended; whatever was outstanding no longer is. */
|
|
61
|
+
reset() {
|
|
62
|
+
this.autoMuted = false;
|
|
63
|
+
this.wasSpeaking = false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Derive the Voice Live session from the chosen mode.
|
|
3
|
+
*
|
|
4
|
+
* Two settings carry the whole behavioural difference between the modes, and
|
|
5
|
+
* both are documented Voice Live features rather than anything clever:
|
|
6
|
+
*
|
|
7
|
+
* - `turn_detection.create_response` — "Enable or disable whether a
|
|
8
|
+
* response is generated (default: true)". `relay` sets it false, so the
|
|
9
|
+
* Voice Live model never answers; the app's AG-UI agent does, and the
|
|
10
|
+
* answer is spoken back with `pre_generated_assistant_message`.
|
|
11
|
+
* - `tools` — only `realtime` gets them, because only there is the Voice
|
|
12
|
+
* Live model the one deciding to call something.
|
|
13
|
+
*
|
|
14
|
+
* `input_audio_transcription` is forced on for both modes. The SDK's
|
|
15
|
+
* `DEFAULT_SESSION_CONFIG` omits it, and without it no user transcript ever
|
|
16
|
+
* arrives — which silently reduces `relay` to a microphone that does
|
|
17
|
+
* nothing and `realtime` to an assistant-only transcript.
|
|
18
|
+
*/
|
|
19
|
+
import type { FunctionTool, Tool, VoiceLiveSessionConfig } from "@iloveagents/foundry-voice-live-react";
|
|
20
|
+
import type { ToolRegistry } from "@iloveagents/foundry-agent";
|
|
21
|
+
import type { VoiceMode } from "./types.ts";
|
|
22
|
+
/**
|
|
23
|
+
* AG-UI tool schemas → Voice Live function tools.
|
|
24
|
+
*
|
|
25
|
+
* The two shapes are the same object plus a discriminator, which is the
|
|
26
|
+
* point: a tool the typed chat can call is callable by voice with no second
|
|
27
|
+
* declaration and no drift between them.
|
|
28
|
+
*/
|
|
29
|
+
export declare function toVoiceLiveTools(registry: ToolRegistry): FunctionTool[];
|
|
30
|
+
export interface BuildVoiceSessionOptions {
|
|
31
|
+
mode: VoiceMode;
|
|
32
|
+
/** Host overrides, applied last — an explicit value always wins. */
|
|
33
|
+
overrides?: VoiceLiveSessionConfig;
|
|
34
|
+
/** Present only when the host opted into exposing client tools in `realtime`. */
|
|
35
|
+
registry?: ToolRegistry;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Build the session config for a mode.
|
|
39
|
+
*
|
|
40
|
+
* Precedence is deliberate and shallow-per-key: mode defaults, then host
|
|
41
|
+
* overrides. `turnDetection` merges key-by-key so a host can retune VAD
|
|
42
|
+
* (threshold, eagerness, filler words) without having to re-state
|
|
43
|
+
* `createResponse` and silently break the mode.
|
|
44
|
+
*/
|
|
45
|
+
export declare function buildVoiceSession({ mode, overrides, registry, }: BuildVoiceSessionOptions): VoiceLiveSessionConfig;
|
|
46
|
+
/**
|
|
47
|
+
* The tool catalogue a session carries: the registry's client tools merged
|
|
48
|
+
* with whatever the host declared.
|
|
49
|
+
*
|
|
50
|
+
* Exported because `session.update` REPLACES the catalogue rather than adding
|
|
51
|
+
* to it, so the live re-sync has to rebuild the very same list this file
|
|
52
|
+
* builds at connect time. Two places deriving it independently is precisely
|
|
53
|
+
* how the host's MCP tools went missing the moment a session became ready.
|
|
54
|
+
*/
|
|
55
|
+
export declare function sessionTools(registry: ToolRegistry | undefined, hostTools: Tool[] | undefined): Tool[] | undefined;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Derive the Voice Live session from the chosen mode.
|
|
3
|
+
*
|
|
4
|
+
* Two settings carry the whole behavioural difference between the modes, and
|
|
5
|
+
* both are documented Voice Live features rather than anything clever:
|
|
6
|
+
*
|
|
7
|
+
* - `turn_detection.create_response` — "Enable or disable whether a
|
|
8
|
+
* response is generated (default: true)". `relay` sets it false, so the
|
|
9
|
+
* Voice Live model never answers; the app's AG-UI agent does, and the
|
|
10
|
+
* answer is spoken back with `pre_generated_assistant_message`.
|
|
11
|
+
* - `tools` — only `realtime` gets them, because only there is the Voice
|
|
12
|
+
* Live model the one deciding to call something.
|
|
13
|
+
*
|
|
14
|
+
* `input_audio_transcription` is forced on for both modes. The SDK's
|
|
15
|
+
* `DEFAULT_SESSION_CONFIG` omits it, and without it no user transcript ever
|
|
16
|
+
* arrives — which silently reduces `relay` to a microphone that does
|
|
17
|
+
* nothing and `realtime` to an assistant-only transcript.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Transcription model. `mai-transcribe` is the Voice Live model that works
|
|
21
|
+
* with every model and agent, rather than only the OpenAI-native ones.
|
|
22
|
+
*/
|
|
23
|
+
const DEFAULT_TRANSCRIPTION_MODEL = "mai-transcribe";
|
|
24
|
+
/**
|
|
25
|
+
* Green screen, so the avatar can be keyed out and sit on the app's own
|
|
26
|
+
* background.
|
|
27
|
+
*
|
|
28
|
+
* `VoiceLiveAvatar` chroma-keys by default, but only the *session* can ask
|
|
29
|
+
* Azure to render a keyable background — without this the service composites
|
|
30
|
+
* the avatar onto opaque white, the chroma key finds no green, and the
|
|
31
|
+
* result is a white rectangle pasted over the page.
|
|
32
|
+
*/
|
|
33
|
+
const GREEN_SCREEN = "#00FF00FF";
|
|
34
|
+
/**
|
|
35
|
+
* AG-UI tool schemas → Voice Live function tools.
|
|
36
|
+
*
|
|
37
|
+
* The two shapes are the same object plus a discriminator, which is the
|
|
38
|
+
* point: a tool the typed chat can call is callable by voice with no second
|
|
39
|
+
* declaration and no drift between them.
|
|
40
|
+
*/
|
|
41
|
+
export function toVoiceLiveTools(registry) {
|
|
42
|
+
return registry.getActiveSchemas().map((schema) => ({
|
|
43
|
+
type: "function",
|
|
44
|
+
name: schema.name,
|
|
45
|
+
description: schema.description ?? "",
|
|
46
|
+
parameters: (schema.parameters ?? {}),
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Build the session config for a mode.
|
|
51
|
+
*
|
|
52
|
+
* Precedence is deliberate and shallow-per-key: mode defaults, then host
|
|
53
|
+
* overrides. `turnDetection` merges key-by-key so a host can retune VAD
|
|
54
|
+
* (threshold, eagerness, filler words) without having to re-state
|
|
55
|
+
* `createResponse` and silently break the mode.
|
|
56
|
+
*/
|
|
57
|
+
export function buildVoiceSession({ mode, overrides, registry, }) {
|
|
58
|
+
const isRelay = mode === "relay";
|
|
59
|
+
const tools = registry ? toVoiceLiveTools(registry) : undefined;
|
|
60
|
+
const base = {
|
|
61
|
+
modalities: ["text", "audio"],
|
|
62
|
+
inputAudioTranscription: { model: DEFAULT_TRANSCRIPTION_MODEL },
|
|
63
|
+
turnDetection: {
|
|
64
|
+
type: "azure_semantic_vad",
|
|
65
|
+
// Barge-in, and keep the server's transcript honest about what was
|
|
66
|
+
// actually heard before the interruption.
|
|
67
|
+
interruptResponse: true,
|
|
68
|
+
autoTruncate: true,
|
|
69
|
+
createResponse: !isRelay,
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
// An empty tool list is not the same as "no tools": sending `tools: []`
|
|
73
|
+
// states there are none, which is right for relay and wrong to send when
|
|
74
|
+
// the host simply did not opt in.
|
|
75
|
+
if (tools !== undefined) {
|
|
76
|
+
base.tools = tools;
|
|
77
|
+
base.toolChoice = "auto";
|
|
78
|
+
}
|
|
79
|
+
if (!overrides)
|
|
80
|
+
return withKeyableAvatarBackground(base);
|
|
81
|
+
return withKeyableAvatarBackground({
|
|
82
|
+
...base,
|
|
83
|
+
...overrides,
|
|
84
|
+
// Tools MERGE rather than replace. A host declaring an MCP server or a
|
|
85
|
+
// Foundry agent tool is adding server-side capability, not withdrawing
|
|
86
|
+
// the browser's — and a plain spread would silently drop every client
|
|
87
|
+
// tool the moment one was declared, leaving the model unable to touch
|
|
88
|
+
// the UI with no error to explain why.
|
|
89
|
+
tools: sessionTools(registry, overrides.tools),
|
|
90
|
+
turnDetection: overrides.turnDetection === undefined
|
|
91
|
+
? base.turnDetection
|
|
92
|
+
: overrides.turnDetection === null
|
|
93
|
+
? null
|
|
94
|
+
: { ...base.turnDetection, ...overrides.turnDetection },
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Give a configured avatar a keyable background unless the host chose one.
|
|
99
|
+
*
|
|
100
|
+
* A host that wants the avatar composited on a colour or an image says so and
|
|
101
|
+
* is left alone; the default is the one that composes with any theme.
|
|
102
|
+
*/
|
|
103
|
+
function withKeyableAvatarBackground(session) {
|
|
104
|
+
const avatar = session.avatar;
|
|
105
|
+
if (!avatar || avatar.video?.background !== undefined)
|
|
106
|
+
return session;
|
|
107
|
+
return {
|
|
108
|
+
...session,
|
|
109
|
+
avatar: {
|
|
110
|
+
...avatar,
|
|
111
|
+
video: { ...avatar.video, background: { color: GREEN_SCREEN } },
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* The tool catalogue a session carries: the registry's client tools merged
|
|
117
|
+
* with whatever the host declared.
|
|
118
|
+
*
|
|
119
|
+
* Exported because `session.update` REPLACES the catalogue rather than adding
|
|
120
|
+
* to it, so the live re-sync has to rebuild the very same list this file
|
|
121
|
+
* builds at connect time. Two places deriving it independently is precisely
|
|
122
|
+
* how the host's MCP tools went missing the moment a session became ready.
|
|
123
|
+
*/
|
|
124
|
+
export function sessionTools(registry, hostTools) {
|
|
125
|
+
return mergeTools(registry ? toVoiceLiveTools(registry) : undefined, hostTools);
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Combine the registry's function tools with whatever the host declared.
|
|
129
|
+
*
|
|
130
|
+
* Function tools are keyed by name so a host can deliberately override one
|
|
131
|
+
* of the registry's; everything else (MCP servers, Foundry agent tools) is
|
|
132
|
+
* additive, since those are distinct capabilities rather than replacements.
|
|
133
|
+
*/
|
|
134
|
+
function mergeTools(base, extra) {
|
|
135
|
+
if (!base)
|
|
136
|
+
return extra;
|
|
137
|
+
if (!extra)
|
|
138
|
+
return base;
|
|
139
|
+
const byName = new Map();
|
|
140
|
+
const others = [];
|
|
141
|
+
for (const tool of [...base, ...extra]) {
|
|
142
|
+
if (tool.type === "function")
|
|
143
|
+
byName.set(tool.name, tool);
|
|
144
|
+
else
|
|
145
|
+
others.push(tool);
|
|
146
|
+
}
|
|
147
|
+
return [...byName.values(), ...others];
|
|
148
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning a streaming markdown answer into spoken audio.
|
|
3
|
+
*
|
|
4
|
+
* Three separate problems, kept separate:
|
|
5
|
+
*
|
|
6
|
+
* 1. **What to say.** The agent writes markdown for a reader. Read aloud
|
|
7
|
+
* verbatim it becomes "asterisk asterisk important asterisk asterisk",
|
|
8
|
+
* and foundry's `[1]` citation markers become "bracket one". `toSpeakableText`
|
|
9
|
+
* strips the notation and keeps the prose.
|
|
10
|
+
*
|
|
11
|
+
* 2. **When to say it.** Waiting for the whole answer wastes the seconds the
|
|
12
|
+
* agent spends writing it. `SentenceStream` takes cumulative text and
|
|
13
|
+
* hands back complete sentences as they finish, so speech starts on the
|
|
14
|
+
* first full sentence.
|
|
15
|
+
*
|
|
16
|
+
* 3. **One at a time.** Voice Live rejects overlapping responses. The SDK's
|
|
17
|
+
* own `ResponseGate` serializes them, but it deliberately collapses
|
|
18
|
+
* everything queued during one response into a *single* follow-up — the
|
|
19
|
+
* right behaviour for turns, fatal for a sentence queue, where
|
|
20
|
+
* collapsing three pending sentences into one would drop two of them.
|
|
21
|
+
* `SpeechQueue` therefore keeps its own FIFO and releases the next
|
|
22
|
+
* utterance only once the previous response is done.
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* Markdown → text a voice can read.
|
|
26
|
+
*
|
|
27
|
+
* Deliberately lossy: code blocks and table rules are dropped rather than
|
|
28
|
+
* spelled out, because hearing punctuation read aloud is worse than not
|
|
29
|
+
* hearing it at all. What survives is the prose.
|
|
30
|
+
*/
|
|
31
|
+
export declare function toSpeakableText(markdown: string, options?: {
|
|
32
|
+
final?: boolean;
|
|
33
|
+
}): string;
|
|
34
|
+
/**
|
|
35
|
+
* Cuts cumulative streaming text into speakable utterances.
|
|
36
|
+
*
|
|
37
|
+
* Fed the *whole* answer so far on each call (which is what assistant-ui
|
|
38
|
+
* exposes — parts carry cumulative text, not deltas) and returns only what
|
|
39
|
+
* has not been emitted yet.
|
|
40
|
+
*/
|
|
41
|
+
export declare class SentenceStream {
|
|
42
|
+
/** How much of the answer has been turned into emitted utterances. */
|
|
43
|
+
private consumed;
|
|
44
|
+
/** Exactly that prefix, kept so a rewrite of it can be detected. */
|
|
45
|
+
private consumedText;
|
|
46
|
+
/**
|
|
47
|
+
* Every position an utterance has ended at, 0 first.
|
|
48
|
+
*
|
|
49
|
+
* A rewrite inside already-spoken text can only be repaired at one of these.
|
|
50
|
+
* Rewinding to the raw point of divergence resumes mid-word — "…ly different
|
|
51
|
+
* answer" — because the divergence lands wherever the two strings happen to
|
|
52
|
+
* part, which is usually inside a word.
|
|
53
|
+
*/
|
|
54
|
+
private boundaries;
|
|
55
|
+
/** The last cumulative answer seen, kept so `flush()` can finish it. */
|
|
56
|
+
private text;
|
|
57
|
+
/**
|
|
58
|
+
* @param cumulative the full answer so far, already run through `toSpeakableText`
|
|
59
|
+
* @returns complete utterances that became available with this update
|
|
60
|
+
*/
|
|
61
|
+
push(cumulative: string): string[];
|
|
62
|
+
/** Emit whatever is left, complete sentence or not. Call when the answer ends. */
|
|
63
|
+
flush(): string[];
|
|
64
|
+
reset(): void;
|
|
65
|
+
private take;
|
|
66
|
+
private advanceTo;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* A strict FIFO of utterances, one on the wire at a time.
|
|
70
|
+
*
|
|
71
|
+
* `send` puts one utterance on the wire. `onResponseDone` must be called
|
|
72
|
+
* when the service reports that response finished — that is the only thing
|
|
73
|
+
* that releases the next one.
|
|
74
|
+
*/
|
|
75
|
+
export declare class SpeechQueue {
|
|
76
|
+
private readonly send;
|
|
77
|
+
private readonly pending;
|
|
78
|
+
private speaking;
|
|
79
|
+
constructor(send: (text: string) => void);
|
|
80
|
+
get isSpeaking(): boolean;
|
|
81
|
+
get pendingCount(): number;
|
|
82
|
+
enqueue(text: string): void;
|
|
83
|
+
/** The in-flight response finished; release the next utterance. */
|
|
84
|
+
onResponseDone(): void;
|
|
85
|
+
/**
|
|
86
|
+
* Barge-in or a new turn: forget everything still queued.
|
|
87
|
+
*
|
|
88
|
+
* `speaking` is cleared too. The caller cancels the in-flight response on
|
|
89
|
+
* the wire, and a cancelled response may never produce the `response.done`
|
|
90
|
+
* that would otherwise release the queue — leaving it wedged shut for the
|
|
91
|
+
* rest of the session.
|
|
92
|
+
*/
|
|
93
|
+
cancel(): void;
|
|
94
|
+
private pump;
|
|
95
|
+
}
|