@craftedxp/voice-js 0.4.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONSUMING.md +10 -2
- package/README.md +69 -11
- package/dist/browser.d.mts +178 -1
- package/dist/browser.d.ts +535 -259
- package/dist/browser.js +924 -710
- package/dist/browser.js.map +1 -1
- package/dist/browser.mjs +209 -1
- package/dist/browser.mjs.map +1 -1
- package/dist/embed.iife.js +23373 -42
- package/dist/node.d.mts +177 -1
- package/dist/node.d.ts +516 -256
- package/dist/node.js +471 -432
- package/dist/node.js.map +1 -1
- package/dist/node.mjs +25 -1
- package/dist/node.mjs.map +1 -1
- package/package.json +4 -1
package/dist/browser.js
CHANGED
|
@@ -1,24 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
var __defProp = Object.defineProperty
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty
|
|
1
|
+
'use strict'
|
|
2
|
+
var __defProp = Object.defineProperty
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty
|
|
6
6
|
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
|
|
9
|
-
};
|
|
7
|
+
for (var name in all) __defProp(target, name, { get: all[name], enumerable: true })
|
|
8
|
+
}
|
|
10
9
|
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from ===
|
|
10
|
+
if ((from && typeof from === 'object') || typeof from === 'function') {
|
|
12
11
|
for (let key of __getOwnPropNames(from))
|
|
13
12
|
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, {
|
|
13
|
+
__defProp(to, key, {
|
|
14
|
+
get: () => from[key],
|
|
15
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable,
|
|
16
|
+
})
|
|
15
17
|
}
|
|
16
|
-
return to
|
|
17
|
-
}
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({},
|
|
18
|
+
return to
|
|
19
|
+
}
|
|
20
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, '__esModule', { value: true }), mod)
|
|
19
21
|
|
|
20
22
|
// src/browser.ts
|
|
21
|
-
var browser_exports = {}
|
|
23
|
+
var browser_exports = {}
|
|
22
24
|
__export(browser_exports, {
|
|
23
25
|
buildWsUrl: () => buildWsUrl,
|
|
24
26
|
configureVoiceClient: () => configureVoiceClient,
|
|
@@ -26,57 +28,66 @@ __export(browser_exports, {
|
|
|
26
28
|
createAudioPlayback: () => createAudioPlayback,
|
|
27
29
|
createProtocolState: () => createProtocolState,
|
|
28
30
|
createReconnectingWebSocket: () => createReconnectingWebSocket,
|
|
29
|
-
handleServerMessage: () => handleServerMessage
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
handleServerMessage: () => handleServerMessage,
|
|
32
|
+
joinRoom: () => joinRoom,
|
|
33
|
+
parseIncomingCall: () => parseIncomingCall,
|
|
34
|
+
})
|
|
35
|
+
module.exports = __toCommonJS(browser_exports)
|
|
32
36
|
|
|
33
37
|
// src/config.ts
|
|
34
38
|
function normalizeConfig(config) {
|
|
35
|
-
if (!config) throw new Error(
|
|
36
|
-
if (
|
|
39
|
+
if (!config) throw new Error('configureVoiceClient: config is required')
|
|
40
|
+
if ('apiKey' in config) {
|
|
37
41
|
throw new Error(
|
|
38
|
-
|
|
39
|
-
)
|
|
42
|
+
'configureVoiceClient: `apiKey` is no longer supported. Embedding sk_ in JS code ships server-grade credentials to every client. Pass `fetchToken: async ({ agentId }) => { /* call YOUR backend mint */ }` instead \u2014 see the @craftedxp/voice-js README for the migration recipe.',
|
|
43
|
+
)
|
|
40
44
|
}
|
|
41
45
|
if (!config.apiBase) {
|
|
42
|
-
throw new Error(
|
|
46
|
+
throw new Error('configureVoiceClient: apiBase is required')
|
|
43
47
|
}
|
|
44
|
-
if (typeof config.fetchToken !==
|
|
45
|
-
throw new Error(
|
|
48
|
+
if (typeof config.fetchToken !== 'function') {
|
|
49
|
+
throw new Error('configureVoiceClient: fetchToken must be a function')
|
|
46
50
|
}
|
|
47
51
|
return {
|
|
48
52
|
...config,
|
|
49
|
-
apiBase: config.apiBase.replace(/\/+$/,
|
|
50
|
-
}
|
|
53
|
+
apiBase: config.apiBase.replace(/\/+$/, ''),
|
|
54
|
+
}
|
|
51
55
|
}
|
|
52
56
|
function mergeStartCallContext(factory, call) {
|
|
53
|
-
const context =
|
|
54
|
-
|
|
55
|
-
|
|
57
|
+
const context =
|
|
58
|
+
factory.defaultContext || call.context
|
|
59
|
+
? { ...(factory.defaultContext ?? {}), ...(call.context ?? {}) }
|
|
60
|
+
: void 0
|
|
61
|
+
const metadata =
|
|
62
|
+
factory.defaultMetadata || call.metadata
|
|
63
|
+
? { ...(factory.defaultMetadata ?? {}), ...(call.metadata ?? {}) }
|
|
64
|
+
: void 0
|
|
65
|
+
return { context, metadata }
|
|
56
66
|
}
|
|
57
67
|
|
|
58
68
|
// src/worklets/mic-downsampler.worklet.js
|
|
59
|
-
var mic_downsampler_worklet_default =
|
|
69
|
+
var mic_downsampler_worklet_default =
|
|
70
|
+
"// AudioWorklet \u2014 runs off the main thread in the audio rendering graph.\n//\n// Input: Float32 samples at the AudioContext's native sampleRate (typically\n// 48000 Hz on desktop, 44100 Hz on some iOS devices).\n// Output: 16 kHz mono Int16 PCM, shipped to the main thread via\n// `port.postMessage(ArrayBuffer, [ArrayBuffer])` (transferred, not copied).\n//\n// Why AudioWorklet instead of ScriptProcessorNode: ScriptProcessorNode is\n// deprecated + main-thread-bound, so any JS jank produces audible audio\n// glitches. AudioWorklet's `process()` runs on the audio rendering thread\n// at the graph's block cadence (128 frames by default) and backpressures\n// via returning `true` / `false`.\n//\n// This file is loaded as text (see tsup.config.ts loader) and registered\n// at runtime via `audioWorklet.addModule(blobUrl)`.\n\nclass MicDownsampler extends AudioWorkletProcessor {\n constructor() {\n super()\n // Target sample rate for STT. Matches Deepgram Nova-3 + the platform's\n // server-side SAMPLE_RATE constant in AgentCallHandler.\n this.targetRate = 16000\n // Accumulator for the downsample. We collect incoming samples and emit\n // an Int16 chunk when we've accumulated ~1024 target-rate samples\n // (~64 ms at 16 kHz) \u2014 matches the mobile SDK's chunk size so both\n // platforms have the same server-side framing.\n this.outputFrames = 1024\n this.acc = []\n // Running index used for fractional resampling.\n this.readCursor = 0\n }\n\n // `inputs[0][0]` = first channel of first input. 128 Float32 samples per\n // call at the context's sampleRate. Return true = keep processing.\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n const channel = input[0]\n if (!channel || channel.length === 0) return true\n\n const ctxRate = sampleRate // global inside AudioWorkletProcessor\n const ratio = ctxRate / this.targetRate\n\n // Simple linear-interp downsample. For 48000 \u2192 16000 that's 3:1, which\n // linear handles fine for voice. Anti-alias filtering would be\n // theoretically better but inaudible for speech.\n for (let i = 0; i < channel.length; i++) {\n this.acc.push(channel[i])\n }\n\n while (this.acc.length - this.readCursor >= ratio * this.outputFrames) {\n const out = new Int16Array(this.outputFrames)\n let readIdx = this.readCursor\n for (let i = 0; i < this.outputFrames; i++) {\n // Linear interp between floor(readIdx) and ceil(readIdx)\n const low = Math.floor(readIdx)\n const high = Math.min(low + 1, this.acc.length - 1)\n const frac = readIdx - low\n const sample = this.acc[low] * (1 - frac) + this.acc[high] * frac\n // Clip + convert to int16\n const clipped = Math.max(-1, Math.min(1, sample))\n out[i] = clipped < 0 ? clipped * 0x8000 : clipped * 0x7fff\n readIdx += ratio\n }\n // Transfer the ArrayBuffer (zero-copy) to the main thread.\n this.port.postMessage(out.buffer, [out.buffer])\n this.readCursor = readIdx\n }\n\n // Garbage-collect the consumed portion of `acc` every so often so it\n // doesn't grow without bound. Leave ~one chunk of headroom.\n if (this.readCursor > ratio * this.outputFrames) {\n this.acc = this.acc.slice(Math.floor(this.readCursor))\n this.readCursor -= Math.floor(this.readCursor)\n }\n\n return true\n }\n}\n\nregisterProcessor('mic-downsampler', MicDownsampler)\n"
|
|
60
71
|
|
|
61
72
|
// src/AudioCapture.ts
|
|
62
|
-
var VOLUME_INTERVAL_MS = 100
|
|
73
|
+
var VOLUME_INTERVAL_MS = 100
|
|
63
74
|
var createAudioCapture = (options) => {
|
|
64
|
-
let audioContext = null
|
|
65
|
-
let mediaStream = null
|
|
66
|
-
let sourceNode = null
|
|
67
|
-
let workletNode = null
|
|
68
|
-
let analyser = null
|
|
69
|
-
let volumeTimer = null
|
|
70
|
-
let muted = false
|
|
71
|
-
let capturing = false
|
|
75
|
+
let audioContext = null
|
|
76
|
+
let mediaStream = null
|
|
77
|
+
let sourceNode = null
|
|
78
|
+
let workletNode = null
|
|
79
|
+
let analyser = null
|
|
80
|
+
let volumeTimer = null
|
|
81
|
+
let muted = false
|
|
82
|
+
let capturing = false
|
|
72
83
|
const computeRms = (buf) => {
|
|
73
|
-
let sum = 0
|
|
74
|
-
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]
|
|
75
|
-
const rms = Math.sqrt(sum / buf.length)
|
|
76
|
-
return Math.min(1, rms * 1.8)
|
|
77
|
-
}
|
|
84
|
+
let sum = 0
|
|
85
|
+
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]
|
|
86
|
+
const rms = Math.sqrt(sum / buf.length)
|
|
87
|
+
return Math.min(1, rms * 1.8)
|
|
88
|
+
}
|
|
78
89
|
const start = async () => {
|
|
79
|
-
if (capturing) return
|
|
90
|
+
if (capturing) return
|
|
80
91
|
try {
|
|
81
92
|
mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
82
93
|
audio: {
|
|
@@ -87,751 +98,748 @@ var createAudioCapture = (options) => {
|
|
|
87
98
|
echoCancellation: true,
|
|
88
99
|
noiseSuppression: true,
|
|
89
100
|
autoGainControl: true,
|
|
90
|
-
channelCount: 1
|
|
91
|
-
}
|
|
92
|
-
})
|
|
93
|
-
audioContext = new AudioContext()
|
|
94
|
-
if (audioContext.state ===
|
|
95
|
-
const blob = new Blob([mic_downsampler_worklet_default], { type:
|
|
96
|
-
const url = URL.createObjectURL(blob)
|
|
101
|
+
channelCount: 1,
|
|
102
|
+
},
|
|
103
|
+
})
|
|
104
|
+
audioContext = new AudioContext()
|
|
105
|
+
if (audioContext.state === 'suspended') await audioContext.resume()
|
|
106
|
+
const blob = new Blob([mic_downsampler_worklet_default], { type: 'application/javascript' })
|
|
107
|
+
const url = URL.createObjectURL(blob)
|
|
97
108
|
try {
|
|
98
|
-
await audioContext.audioWorklet.addModule(url)
|
|
109
|
+
await audioContext.audioWorklet.addModule(url)
|
|
99
110
|
} finally {
|
|
100
|
-
URL.revokeObjectURL(url)
|
|
111
|
+
URL.revokeObjectURL(url)
|
|
101
112
|
}
|
|
102
|
-
sourceNode = audioContext.createMediaStreamSource(mediaStream)
|
|
103
|
-
workletNode = new AudioWorkletNode(audioContext,
|
|
113
|
+
sourceNode = audioContext.createMediaStreamSource(mediaStream)
|
|
114
|
+
workletNode = new AudioWorkletNode(audioContext, 'mic-downsampler')
|
|
104
115
|
workletNode.port.onmessage = (event) => {
|
|
105
|
-
if (muted) return
|
|
106
|
-
options.onChunk(event.data)
|
|
107
|
-
}
|
|
116
|
+
if (muted) return
|
|
117
|
+
options.onChunk(event.data)
|
|
118
|
+
}
|
|
108
119
|
if (options.onVolume) {
|
|
109
|
-
analyser = audioContext.createAnalyser()
|
|
110
|
-
analyser.fftSize = 256
|
|
111
|
-
sourceNode.connect(analyser)
|
|
112
|
-
const buf = new Float32Array(analyser.fftSize)
|
|
120
|
+
analyser = audioContext.createAnalyser()
|
|
121
|
+
analyser.fftSize = 256
|
|
122
|
+
sourceNode.connect(analyser)
|
|
123
|
+
const buf = new Float32Array(analyser.fftSize)
|
|
113
124
|
volumeTimer = setInterval(() => {
|
|
114
|
-
if (!analyser) return
|
|
115
|
-
analyser.getFloatTimeDomainData(buf)
|
|
116
|
-
options.onVolume?.(computeRms(buf))
|
|
117
|
-
}, VOLUME_INTERVAL_MS)
|
|
125
|
+
if (!analyser) return
|
|
126
|
+
analyser.getFloatTimeDomainData(buf)
|
|
127
|
+
options.onVolume?.(computeRms(buf))
|
|
128
|
+
}, VOLUME_INTERVAL_MS)
|
|
118
129
|
}
|
|
119
|
-
sourceNode.connect(workletNode)
|
|
120
|
-
const sink = audioContext.createGain()
|
|
121
|
-
sink.gain.value = 0
|
|
122
|
-
workletNode.connect(sink).connect(audioContext.destination)
|
|
123
|
-
capturing = true
|
|
130
|
+
sourceNode.connect(workletNode)
|
|
131
|
+
const sink = audioContext.createGain()
|
|
132
|
+
sink.gain.value = 0
|
|
133
|
+
workletNode.connect(sink).connect(audioContext.destination)
|
|
134
|
+
capturing = true
|
|
124
135
|
} catch (err) {
|
|
125
|
-
const wrapped =
|
|
126
|
-
|
|
127
|
-
|
|
136
|
+
const wrapped =
|
|
137
|
+
err instanceof Error ? err : new Error(typeof err === 'string' ? err : 'capture failed')
|
|
138
|
+
options.onError?.(wrapped)
|
|
139
|
+
throw wrapped
|
|
128
140
|
}
|
|
129
|
-
}
|
|
141
|
+
}
|
|
130
142
|
const stop = () => {
|
|
131
|
-
if (!capturing) return
|
|
132
|
-
capturing = false
|
|
143
|
+
if (!capturing) return
|
|
144
|
+
capturing = false
|
|
133
145
|
if (volumeTimer) {
|
|
134
|
-
clearInterval(volumeTimer)
|
|
135
|
-
volumeTimer = null
|
|
146
|
+
clearInterval(volumeTimer)
|
|
147
|
+
volumeTimer = null
|
|
136
148
|
}
|
|
137
149
|
try {
|
|
138
|
-
workletNode?.disconnect()
|
|
139
|
-
analyser?.disconnect()
|
|
140
|
-
sourceNode?.disconnect()
|
|
141
|
-
} catch {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
sourceNode = null;
|
|
150
|
+
workletNode?.disconnect()
|
|
151
|
+
analyser?.disconnect()
|
|
152
|
+
sourceNode?.disconnect()
|
|
153
|
+
} catch {}
|
|
154
|
+
workletNode = null
|
|
155
|
+
analyser = null
|
|
156
|
+
sourceNode = null
|
|
146
157
|
if (mediaStream) {
|
|
147
|
-
for (const track of mediaStream.getTracks()) track.stop()
|
|
148
|
-
mediaStream = null
|
|
158
|
+
for (const track of mediaStream.getTracks()) track.stop()
|
|
159
|
+
mediaStream = null
|
|
149
160
|
}
|
|
150
|
-
if (audioContext && audioContext.state !==
|
|
151
|
-
void audioContext.close().catch(() => void 0)
|
|
161
|
+
if (audioContext && audioContext.state !== 'closed') {
|
|
162
|
+
void audioContext.close().catch(() => void 0)
|
|
152
163
|
}
|
|
153
|
-
audioContext = null
|
|
154
|
-
}
|
|
164
|
+
audioContext = null
|
|
165
|
+
}
|
|
155
166
|
return {
|
|
156
167
|
start,
|
|
157
168
|
stop,
|
|
158
169
|
mute: (v) => {
|
|
159
|
-
muted = v
|
|
170
|
+
muted = v
|
|
160
171
|
},
|
|
161
|
-
isCapturing: () => capturing
|
|
162
|
-
}
|
|
163
|
-
}
|
|
172
|
+
isCapturing: () => capturing,
|
|
173
|
+
}
|
|
174
|
+
}
|
|
164
175
|
|
|
165
176
|
// src/AudioPlayback.ts
|
|
166
|
-
var DEFAULT_SAMPLE_RATE = 16e3
|
|
167
|
-
var VOLUME_INTERVAL_MS2 = 100
|
|
177
|
+
var DEFAULT_SAMPLE_RATE = 16e3
|
|
178
|
+
var VOLUME_INTERVAL_MS2 = 100
|
|
168
179
|
var createAudioPlayback = (options = {}) => {
|
|
169
|
-
const sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE
|
|
170
|
-
let audioContext = null
|
|
171
|
-
let gainNode = null
|
|
172
|
-
let analyser = null
|
|
173
|
-
let volumeTimer = null
|
|
174
|
-
let nextStartTime = 0
|
|
175
|
-
let scheduledNodes = []
|
|
176
|
-
let speaking = false
|
|
180
|
+
const sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE
|
|
181
|
+
let audioContext = null
|
|
182
|
+
let gainNode = null
|
|
183
|
+
let analyser = null
|
|
184
|
+
let volumeTimer = null
|
|
185
|
+
let nextStartTime = 0
|
|
186
|
+
let scheduledNodes = []
|
|
187
|
+
let speaking = false
|
|
177
188
|
const ensureContext = async () => {
|
|
178
189
|
if (audioContext) {
|
|
179
|
-
if (audioContext.state ===
|
|
180
|
-
return
|
|
190
|
+
if (audioContext.state === 'suspended') await audioContext.resume()
|
|
191
|
+
return
|
|
181
192
|
}
|
|
182
|
-
audioContext = new AudioContext({ sampleRate })
|
|
183
|
-
gainNode = audioContext.createGain()
|
|
193
|
+
audioContext = new AudioContext({ sampleRate })
|
|
194
|
+
gainNode = audioContext.createGain()
|
|
184
195
|
if (options.onVolume) {
|
|
185
|
-
analyser = audioContext.createAnalyser()
|
|
186
|
-
analyser.fftSize = 256
|
|
187
|
-
gainNode.connect(analyser)
|
|
188
|
-
const buf = new Float32Array(analyser.fftSize)
|
|
196
|
+
analyser = audioContext.createAnalyser()
|
|
197
|
+
analyser.fftSize = 256
|
|
198
|
+
gainNode.connect(analyser)
|
|
199
|
+
const buf = new Float32Array(analyser.fftSize)
|
|
189
200
|
volumeTimer = setInterval(() => {
|
|
190
|
-
if (!analyser) return
|
|
191
|
-
analyser.getFloatTimeDomainData(buf)
|
|
192
|
-
let sum = 0
|
|
193
|
-
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]
|
|
194
|
-
const rms = Math.sqrt(sum / buf.length)
|
|
195
|
-
options.onVolume?.(Math.min(1, rms * 1.8))
|
|
196
|
-
}, VOLUME_INTERVAL_MS2)
|
|
201
|
+
if (!analyser) return
|
|
202
|
+
analyser.getFloatTimeDomainData(buf)
|
|
203
|
+
let sum = 0
|
|
204
|
+
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]
|
|
205
|
+
const rms = Math.sqrt(sum / buf.length)
|
|
206
|
+
options.onVolume?.(Math.min(1, rms * 1.8))
|
|
207
|
+
}, VOLUME_INTERVAL_MS2)
|
|
197
208
|
}
|
|
198
|
-
gainNode.connect(audioContext.destination)
|
|
199
|
-
nextStartTime = audioContext.currentTime
|
|
200
|
-
}
|
|
209
|
+
gainNode.connect(audioContext.destination)
|
|
210
|
+
nextStartTime = audioContext.currentTime
|
|
211
|
+
}
|
|
201
212
|
const setSpeaking = (v) => {
|
|
202
|
-
if (v === speaking) return
|
|
203
|
-
speaking = v
|
|
204
|
-
options.onSpeakingChange?.(v)
|
|
205
|
-
}
|
|
213
|
+
if (v === speaking) return
|
|
214
|
+
speaking = v
|
|
215
|
+
options.onSpeakingChange?.(v)
|
|
216
|
+
}
|
|
206
217
|
const pruneFinished = () => {
|
|
207
|
-
const now = audioContext?.currentTime ?? 0
|
|
218
|
+
const now = audioContext?.currentTime ?? 0
|
|
208
219
|
scheduledNodes = scheduledNodes.filter((n) => {
|
|
209
|
-
const node = n
|
|
210
|
-
return (node._endsAt ?? 0) > now
|
|
211
|
-
})
|
|
212
|
-
if (scheduledNodes.length === 0) setSpeaking(false)
|
|
213
|
-
}
|
|
220
|
+
const node = n
|
|
221
|
+
return (node._endsAt ?? 0) > now
|
|
222
|
+
})
|
|
223
|
+
if (scheduledNodes.length === 0) setSpeaking(false)
|
|
224
|
+
}
|
|
214
225
|
const enqueue = (pcm) => {
|
|
215
226
|
if (!audioContext) {
|
|
216
|
-
void ensureContext().then(() => enqueue(pcm))
|
|
217
|
-
return
|
|
227
|
+
void ensureContext().then(() => enqueue(pcm))
|
|
228
|
+
return
|
|
218
229
|
}
|
|
219
|
-
if (!audioContext || !gainNode) return
|
|
220
|
-
const int16 = new Int16Array(pcm)
|
|
221
|
-
if (int16.length === 0) return
|
|
222
|
-
const audioBuffer = audioContext.createBuffer(1, int16.length, sampleRate)
|
|
223
|
-
const float32 = audioBuffer.getChannelData(0)
|
|
230
|
+
if (!audioContext || !gainNode) return
|
|
231
|
+
const int16 = new Int16Array(pcm)
|
|
232
|
+
if (int16.length === 0) return
|
|
233
|
+
const audioBuffer = audioContext.createBuffer(1, int16.length, sampleRate)
|
|
234
|
+
const float32 = audioBuffer.getChannelData(0)
|
|
224
235
|
for (let i = 0; i < int16.length; i++) {
|
|
225
|
-
float32[i] = int16[i] / 32768
|
|
236
|
+
float32[i] = int16[i] / 32768
|
|
226
237
|
}
|
|
227
|
-
const node = audioContext.createBufferSource()
|
|
228
|
-
node.buffer = audioBuffer
|
|
229
|
-
node.connect(gainNode)
|
|
230
|
-
const now = audioContext.currentTime
|
|
231
|
-
const startAt = Math.max(now, nextStartTime)
|
|
232
|
-
node.start(startAt)
|
|
233
|
-
const duration = int16.length / sampleRate
|
|
234
|
-
node._endsAt = startAt + duration
|
|
235
|
-
nextStartTime = startAt + duration
|
|
236
|
-
scheduledNodes.push(node)
|
|
237
|
-
setSpeaking(true)
|
|
238
|
-
node.onended = () => pruneFinished()
|
|
239
|
-
}
|
|
238
|
+
const node = audioContext.createBufferSource()
|
|
239
|
+
node.buffer = audioBuffer
|
|
240
|
+
node.connect(gainNode)
|
|
241
|
+
const now = audioContext.currentTime
|
|
242
|
+
const startAt = Math.max(now, nextStartTime)
|
|
243
|
+
node.start(startAt)
|
|
244
|
+
const duration = int16.length / sampleRate
|
|
245
|
+
node._endsAt = startAt + duration
|
|
246
|
+
nextStartTime = startAt + duration
|
|
247
|
+
scheduledNodes.push(node)
|
|
248
|
+
setSpeaking(true)
|
|
249
|
+
node.onended = () => pruneFinished()
|
|
250
|
+
}
|
|
240
251
|
const flush = () => {
|
|
241
|
-
if (!audioContext || !gainNode) return
|
|
252
|
+
if (!audioContext || !gainNode) return
|
|
242
253
|
for (const node of scheduledNodes) {
|
|
243
254
|
try {
|
|
244
|
-
node.stop()
|
|
245
|
-
} catch {
|
|
246
|
-
}
|
|
255
|
+
node.stop()
|
|
256
|
+
} catch {}
|
|
247
257
|
}
|
|
248
|
-
scheduledNodes = []
|
|
249
|
-
gainNode.disconnect()
|
|
250
|
-
gainNode = audioContext.createGain()
|
|
258
|
+
scheduledNodes = []
|
|
259
|
+
gainNode.disconnect()
|
|
260
|
+
gainNode = audioContext.createGain()
|
|
251
261
|
if (analyser) {
|
|
252
|
-
analyser.disconnect()
|
|
253
|
-
gainNode.connect(analyser)
|
|
262
|
+
analyser.disconnect()
|
|
263
|
+
gainNode.connect(analyser)
|
|
254
264
|
}
|
|
255
|
-
gainNode.connect(audioContext.destination)
|
|
256
|
-
nextStartTime = audioContext.currentTime
|
|
257
|
-
setSpeaking(false)
|
|
258
|
-
}
|
|
265
|
+
gainNode.connect(audioContext.destination)
|
|
266
|
+
nextStartTime = audioContext.currentTime
|
|
267
|
+
setSpeaking(false)
|
|
268
|
+
}
|
|
259
269
|
const close = () => {
|
|
260
|
-
flush()
|
|
270
|
+
flush()
|
|
261
271
|
if (volumeTimer) {
|
|
262
|
-
clearInterval(volumeTimer)
|
|
263
|
-
volumeTimer = null
|
|
272
|
+
clearInterval(volumeTimer)
|
|
273
|
+
volumeTimer = null
|
|
264
274
|
}
|
|
265
|
-
if (audioContext && audioContext.state !==
|
|
266
|
-
void audioContext.close().catch(() => void 0)
|
|
275
|
+
if (audioContext && audioContext.state !== 'closed') {
|
|
276
|
+
void audioContext.close().catch(() => void 0)
|
|
267
277
|
}
|
|
268
|
-
audioContext = null
|
|
269
|
-
gainNode = null
|
|
270
|
-
analyser = null
|
|
271
|
-
}
|
|
278
|
+
audioContext = null
|
|
279
|
+
gainNode = null
|
|
280
|
+
analyser = null
|
|
281
|
+
}
|
|
272
282
|
const resume = async () => {
|
|
273
|
-
await ensureContext()
|
|
274
|
-
}
|
|
275
|
-
return { enqueue, flush, close, resume }
|
|
276
|
-
}
|
|
283
|
+
await ensureContext()
|
|
284
|
+
}
|
|
285
|
+
return { enqueue, flush, close, resume }
|
|
286
|
+
}
|
|
277
287
|
|
|
278
288
|
// src/ReconnectingWebSocket.ts
|
|
279
|
-
var READYSTATE_OPEN = 1
|
|
280
|
-
var READYSTATE_CLOSED = 3
|
|
289
|
+
var READYSTATE_OPEN = 1
|
|
290
|
+
var READYSTATE_CLOSED = 3
|
|
281
291
|
var createReconnectingWebSocket = (options, onEvent) => {
|
|
282
|
-
const maxRetries = options.maxRetries ?? 3
|
|
283
|
-
const initialBackoff = options.initialBackoffMs ?? 500
|
|
284
|
-
const maxBackoff = options.maxBackoffMs ?? 8e3
|
|
285
|
-
let ws = null
|
|
286
|
-
let intentionalClose = false
|
|
287
|
-
let retries = 0
|
|
288
|
-
let backoff = initialBackoff
|
|
289
|
-
let reconnectTimer = null
|
|
292
|
+
const maxRetries = options.maxRetries ?? 3
|
|
293
|
+
const initialBackoff = options.initialBackoffMs ?? 500
|
|
294
|
+
const maxBackoff = options.maxBackoffMs ?? 8e3
|
|
295
|
+
let ws = null
|
|
296
|
+
let intentionalClose = false
|
|
297
|
+
let retries = 0
|
|
298
|
+
let backoff = initialBackoff
|
|
299
|
+
let reconnectTimer = null
|
|
290
300
|
const openOnce = () => {
|
|
291
|
-
ws = options.wsFactory(options.url)
|
|
292
|
-
ws.binaryType =
|
|
301
|
+
ws = options.wsFactory(options.url)
|
|
302
|
+
ws.binaryType = 'arraybuffer'
|
|
293
303
|
ws.onopen = () => {
|
|
294
|
-
if (retries === 0) onEvent({ type:
|
|
295
|
-
else onEvent({ type:
|
|
296
|
-
retries = 0
|
|
297
|
-
backoff = initialBackoff
|
|
298
|
-
}
|
|
304
|
+
if (retries === 0) onEvent({ type: 'open' })
|
|
305
|
+
else onEvent({ type: 'reconnected' })
|
|
306
|
+
retries = 0
|
|
307
|
+
backoff = initialBackoff
|
|
308
|
+
}
|
|
299
309
|
ws.onmessage = (ev) => {
|
|
300
|
-
onEvent({ type:
|
|
301
|
-
}
|
|
310
|
+
onEvent({ type: 'message', data: ev.data })
|
|
311
|
+
}
|
|
302
312
|
ws.onerror = () => {
|
|
303
|
-
onEvent({ type:
|
|
304
|
-
}
|
|
313
|
+
onEvent({ type: 'error', error: new Error('WebSocket error') })
|
|
314
|
+
}
|
|
305
315
|
ws.onclose = (ev) => {
|
|
306
|
-
ws = null
|
|
307
|
-
const shouldRetry = !intentionalClose && retries < maxRetries
|
|
316
|
+
ws = null
|
|
317
|
+
const shouldRetry = !intentionalClose && retries < maxRetries
|
|
308
318
|
if (!shouldRetry) {
|
|
309
319
|
onEvent({
|
|
310
|
-
type:
|
|
320
|
+
type: 'close',
|
|
311
321
|
code: ev.code,
|
|
312
322
|
reason: ev.reason,
|
|
313
|
-
permanent: true
|
|
314
|
-
})
|
|
315
|
-
return
|
|
323
|
+
permanent: true,
|
|
324
|
+
})
|
|
325
|
+
return
|
|
316
326
|
}
|
|
317
327
|
onEvent({
|
|
318
|
-
type:
|
|
328
|
+
type: 'close',
|
|
319
329
|
code: ev.code,
|
|
320
330
|
reason: ev.reason,
|
|
321
|
-
permanent: false
|
|
322
|
-
})
|
|
323
|
-
retries
|
|
324
|
-
const delay = Math.min(backoff, maxBackoff)
|
|
325
|
-
backoff = Math.min(backoff * 2, maxBackoff)
|
|
326
|
-
reconnectTimer = setTimeout(openOnce, delay)
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
openOnce()
|
|
331
|
+
permanent: false,
|
|
332
|
+
})
|
|
333
|
+
retries++
|
|
334
|
+
const delay = Math.min(backoff, maxBackoff)
|
|
335
|
+
backoff = Math.min(backoff * 2, maxBackoff)
|
|
336
|
+
reconnectTimer = setTimeout(openOnce, delay)
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
openOnce()
|
|
330
340
|
return {
|
|
331
341
|
send: (data) => {
|
|
332
|
-
if (ws && ws.readyState === READYSTATE_OPEN) ws.send(data)
|
|
342
|
+
if (ws && ws.readyState === READYSTATE_OPEN) ws.send(data)
|
|
333
343
|
},
|
|
334
|
-
close: (code = 1e3, reason =
|
|
335
|
-
intentionalClose = true
|
|
344
|
+
close: (code = 1e3, reason = 'client-requested') => {
|
|
345
|
+
intentionalClose = true
|
|
336
346
|
if (reconnectTimer) {
|
|
337
|
-
clearTimeout(reconnectTimer)
|
|
338
|
-
reconnectTimer = null
|
|
347
|
+
clearTimeout(reconnectTimer)
|
|
348
|
+
reconnectTimer = null
|
|
339
349
|
}
|
|
340
350
|
try {
|
|
341
|
-
ws?.close(code, reason)
|
|
342
|
-
} catch {
|
|
343
|
-
}
|
|
351
|
+
ws?.close(code, reason)
|
|
352
|
+
} catch {}
|
|
344
353
|
},
|
|
345
|
-
readyState: () => ws?.readyState ?? READYSTATE_CLOSED
|
|
346
|
-
}
|
|
347
|
-
}
|
|
354
|
+
readyState: () => ws?.readyState ?? READYSTATE_CLOSED,
|
|
355
|
+
}
|
|
356
|
+
}
|
|
348
357
|
|
|
349
358
|
// src/protocol.ts
|
|
350
359
|
var createProtocolState = () => ({
|
|
351
|
-
state:
|
|
360
|
+
state: 'idle',
|
|
352
361
|
transcript: [],
|
|
353
362
|
agentBubbleId: null,
|
|
354
363
|
idCounter: 0,
|
|
355
|
-
endReason: null
|
|
356
|
-
})
|
|
364
|
+
endReason: null,
|
|
365
|
+
})
|
|
357
366
|
var mapEndReason = (raw) => {
|
|
358
|
-
if (raw ===
|
|
359
|
-
if (raw ===
|
|
360
|
-
if (raw ===
|
|
361
|
-
return
|
|
362
|
-
}
|
|
367
|
+
if (raw === 'agent_ended') return 'agent_ended'
|
|
368
|
+
if (raw === 'caller_hung_up') return 'user_hangup'
|
|
369
|
+
if (raw === 'silence_timeout' || raw === 'max_duration') return 'timeout'
|
|
370
|
+
return 'error'
|
|
371
|
+
}
|
|
363
372
|
function handleServerMessage(raw, state, cb) {
|
|
364
|
-
let msg
|
|
373
|
+
let msg
|
|
365
374
|
try {
|
|
366
|
-
msg = JSON.parse(raw)
|
|
375
|
+
msg = JSON.parse(raw)
|
|
367
376
|
} catch {
|
|
368
|
-
return
|
|
377
|
+
return
|
|
369
378
|
}
|
|
370
379
|
switch (msg.type) {
|
|
371
|
-
case
|
|
372
|
-
cb.onConnected()
|
|
373
|
-
setState(state,
|
|
374
|
-
return
|
|
375
|
-
case
|
|
376
|
-
const text = msg.text ??
|
|
377
|
-
if (!text) return
|
|
378
|
-
const isFinal = !!msg.isFinal
|
|
379
|
-
if (!isFinal) setState(state,
|
|
380
|
-
upsertUserPartial(state, text, isFinal)
|
|
381
|
-
cb.onTranscript(state.transcript)
|
|
382
|
-
return
|
|
380
|
+
case 'connected':
|
|
381
|
+
cb.onConnected()
|
|
382
|
+
setState(state, 'listening', cb)
|
|
383
|
+
return
|
|
384
|
+
case 'transcript': {
|
|
385
|
+
const text = msg.text ?? ''
|
|
386
|
+
if (!text) return
|
|
387
|
+
const isFinal = !!msg.isFinal
|
|
388
|
+
if (!isFinal) setState(state, 'user_speaking', cb)
|
|
389
|
+
upsertUserPartial(state, text, isFinal)
|
|
390
|
+
cb.onTranscript(state.transcript)
|
|
391
|
+
return
|
|
383
392
|
}
|
|
384
|
-
case
|
|
385
|
-
const id = `m${state.idCounter++}
|
|
386
|
-
state.agentBubbleId = id
|
|
387
|
-
state.transcript = [...state.transcript, { id, role:
|
|
388
|
-
cb.onTranscript(state.transcript)
|
|
389
|
-
const seq = typeof msg.seq ===
|
|
390
|
-
cb.onAgentTurnStart(seq)
|
|
391
|
-
setState(state,
|
|
392
|
-
return
|
|
393
|
+
case 'agent_turn_start': {
|
|
394
|
+
const id = `m${state.idCounter++}`
|
|
395
|
+
state.agentBubbleId = id
|
|
396
|
+
state.transcript = [...state.transcript, { id, role: 'agent', text: '' }]
|
|
397
|
+
cb.onTranscript(state.transcript)
|
|
398
|
+
const seq = typeof msg.seq === 'number' ? msg.seq : void 0
|
|
399
|
+
cb.onAgentTurnStart(seq)
|
|
400
|
+
setState(state, 'agent_speaking', cb)
|
|
401
|
+
return
|
|
393
402
|
}
|
|
394
|
-
case
|
|
395
|
-
const delta = msg.text ??
|
|
396
|
-
if (!delta || !state.agentBubbleId) return
|
|
397
|
-
const id = state.agentBubbleId
|
|
398
|
-
state.transcript = state.transcript.map(
|
|
399
|
-
|
|
400
|
-
)
|
|
401
|
-
cb.onTranscript(state.transcript)
|
|
402
|
-
return
|
|
403
|
+
case 'agent_text': {
|
|
404
|
+
const delta = msg.text ?? ''
|
|
405
|
+
if (!delta || !state.agentBubbleId) return
|
|
406
|
+
const id = state.agentBubbleId
|
|
407
|
+
state.transcript = state.transcript.map((e) =>
|
|
408
|
+
e.id === id && e.role === 'agent' ? { ...e, text: e.text + delta } : e,
|
|
409
|
+
)
|
|
410
|
+
cb.onTranscript(state.transcript)
|
|
411
|
+
return
|
|
403
412
|
}
|
|
404
|
-
case
|
|
405
|
-
state.agentBubbleId = null
|
|
406
|
-
const seq = typeof msg.seq ===
|
|
407
|
-
cb.onAgentTurnEnd(seq)
|
|
408
|
-
setState(state,
|
|
409
|
-
return
|
|
413
|
+
case 'agent_turn_end': {
|
|
414
|
+
state.agentBubbleId = null
|
|
415
|
+
const seq = typeof msg.seq === 'number' ? msg.seq : void 0
|
|
416
|
+
cb.onAgentTurnEnd(seq)
|
|
417
|
+
setState(state, 'listening', cb)
|
|
418
|
+
return
|
|
410
419
|
}
|
|
411
|
-
case
|
|
412
|
-
cb.onInterrupt()
|
|
413
|
-
return
|
|
414
|
-
case
|
|
415
|
-
const committed = (msg.committedText ??
|
|
420
|
+
case 'interrupt':
|
|
421
|
+
cb.onInterrupt()
|
|
422
|
+
return
|
|
423
|
+
case 'agent_turn_abort': {
|
|
424
|
+
const committed = (msg.committedText ?? '').trim()
|
|
416
425
|
if (state.agentBubbleId) {
|
|
417
|
-
const id = state.agentBubbleId
|
|
426
|
+
const id = state.agentBubbleId
|
|
418
427
|
if (committed) {
|
|
419
|
-
state.transcript = state.transcript.map(
|
|
420
|
-
|
|
421
|
-
)
|
|
428
|
+
state.transcript = state.transcript.map((e) =>
|
|
429
|
+
e.id === id && e.role === 'agent' ? { ...e, text: committed, interrupted: true } : e,
|
|
430
|
+
)
|
|
422
431
|
} else {
|
|
423
|
-
state.transcript = state.transcript.filter((e) => e.id !== id)
|
|
432
|
+
state.transcript = state.transcript.filter((e) => e.id !== id)
|
|
424
433
|
}
|
|
425
|
-
cb.onTranscript(state.transcript)
|
|
434
|
+
cb.onTranscript(state.transcript)
|
|
426
435
|
}
|
|
427
|
-
state.agentBubbleId = null
|
|
428
|
-
return
|
|
436
|
+
state.agentBubbleId = null
|
|
437
|
+
return
|
|
429
438
|
}
|
|
430
|
-
case
|
|
439
|
+
case 'tool_call':
|
|
431
440
|
state.transcript = [
|
|
432
441
|
...state.transcript,
|
|
433
442
|
{
|
|
434
443
|
id: `m${state.idCounter++}`,
|
|
435
|
-
role:
|
|
436
|
-
text: `\u2192 ${String(msg.tool ??
|
|
437
|
-
}
|
|
438
|
-
]
|
|
439
|
-
cb.onTranscript(state.transcript)
|
|
440
|
-
return
|
|
441
|
-
case
|
|
444
|
+
role: 'tool',
|
|
445
|
+
text: `\u2192 ${String(msg.tool ?? '?')}(${msg.args ? JSON.stringify(msg.args) : ''})`,
|
|
446
|
+
},
|
|
447
|
+
]
|
|
448
|
+
cb.onTranscript(state.transcript)
|
|
449
|
+
return
|
|
450
|
+
case 'tool_result':
|
|
442
451
|
state.transcript = [
|
|
443
452
|
...state.transcript,
|
|
444
453
|
{
|
|
445
454
|
id: `m${state.idCounter++}`,
|
|
446
|
-
role:
|
|
447
|
-
text: `${msg.ok ?
|
|
448
|
-
}
|
|
449
|
-
]
|
|
450
|
-
cb.onTranscript(state.transcript)
|
|
451
|
-
return
|
|
452
|
-
case
|
|
453
|
-
const toolCallId = String(msg.toolCallId ??
|
|
454
|
-
const name = String(msg.name ??
|
|
455
|
-
const args = msg.args ?? {}
|
|
456
|
-
if (!toolCallId || !name) return
|
|
457
|
-
cb.onClientToolCall({ toolCallId, name, args })
|
|
458
|
-
return
|
|
455
|
+
role: 'tool',
|
|
456
|
+
text: `${msg.ok ? '\u2713' : '\u2717'} ${String(msg.tool ?? '?')}`,
|
|
457
|
+
},
|
|
458
|
+
]
|
|
459
|
+
cb.onTranscript(state.transcript)
|
|
460
|
+
return
|
|
461
|
+
case 'client_tool_call': {
|
|
462
|
+
const toolCallId = String(msg.toolCallId ?? '')
|
|
463
|
+
const name = String(msg.name ?? '')
|
|
464
|
+
const args = msg.args ?? {}
|
|
465
|
+
if (!toolCallId || !name) return
|
|
466
|
+
cb.onClientToolCall({ toolCallId, name, args })
|
|
467
|
+
return
|
|
459
468
|
}
|
|
460
|
-
case
|
|
461
|
-
const reasonRaw = String(msg.reason ??
|
|
462
|
-
const reason = mapEndReason(reasonRaw)
|
|
463
|
-
state.endReason = reason
|
|
469
|
+
case 'call_end': {
|
|
470
|
+
const reasonRaw = String(msg.reason ?? '')
|
|
471
|
+
const reason = mapEndReason(reasonRaw)
|
|
472
|
+
state.endReason = reason
|
|
464
473
|
state.transcript = [
|
|
465
474
|
...state.transcript,
|
|
466
475
|
{
|
|
467
476
|
id: `m${state.idCounter++}`,
|
|
468
|
-
role:
|
|
469
|
-
text: `call ended${reasonRaw ? ` (${reasonRaw})` :
|
|
470
|
-
}
|
|
471
|
-
]
|
|
472
|
-
cb.onTranscript(state.transcript)
|
|
473
|
-
cb.onCallEnd(reason)
|
|
474
|
-
return
|
|
477
|
+
role: 'system',
|
|
478
|
+
text: `call ended${reasonRaw ? ` (${reasonRaw})` : ''}`,
|
|
479
|
+
},
|
|
480
|
+
]
|
|
481
|
+
cb.onTranscript(state.transcript)
|
|
482
|
+
cb.onCallEnd(reason)
|
|
483
|
+
return
|
|
475
484
|
}
|
|
476
|
-
case
|
|
477
|
-
const code = msg.code ??
|
|
478
|
-
const message = msg.message ??
|
|
479
|
-
cb.onError({ code, message })
|
|
480
|
-
return
|
|
485
|
+
case 'error': {
|
|
486
|
+
const code = msg.code ?? 'server_error'
|
|
487
|
+
const message = msg.message ?? 'server error'
|
|
488
|
+
cb.onError({ code, message })
|
|
489
|
+
return
|
|
481
490
|
}
|
|
482
491
|
}
|
|
483
492
|
}
|
|
484
493
|
var setState = (state, next, cb) => {
|
|
485
|
-
if (state.state === next) return
|
|
486
|
-
cb.onState(next)
|
|
487
|
-
}
|
|
494
|
+
if (state.state === next) return
|
|
495
|
+
cb.onState(next)
|
|
496
|
+
}
|
|
488
497
|
var upsertUserPartial = (state, text, isFinal) => {
|
|
489
|
-
let idx = -1
|
|
498
|
+
let idx = -1
|
|
490
499
|
for (let i = state.transcript.length - 1; i >= 0; i--) {
|
|
491
|
-
const e = state.transcript[i]
|
|
492
|
-
if (e.role ===
|
|
493
|
-
idx = i
|
|
494
|
-
break
|
|
500
|
+
const e = state.transcript[i]
|
|
501
|
+
if (e.role === 'user' && e.committed === false) {
|
|
502
|
+
idx = i
|
|
503
|
+
break
|
|
495
504
|
}
|
|
496
505
|
}
|
|
497
506
|
if (idx === -1) {
|
|
498
507
|
state.transcript = [
|
|
499
508
|
...state.transcript,
|
|
500
|
-
{ id: `m${state.idCounter++}`, role:
|
|
501
|
-
]
|
|
502
|
-
return
|
|
503
|
-
}
|
|
504
|
-
const target = state.transcript[idx]
|
|
505
|
-
const next = [...state.transcript]
|
|
506
|
-
next[idx] = { ...target, text, committed: isFinal }
|
|
507
|
-
state.transcript = next
|
|
508
|
-
}
|
|
509
|
+
{ id: `m${state.idCounter++}`, role: 'user', text, committed: isFinal },
|
|
510
|
+
]
|
|
511
|
+
return
|
|
512
|
+
}
|
|
513
|
+
const target = state.transcript[idx]
|
|
514
|
+
const next = [...state.transcript]
|
|
515
|
+
next[idx] = { ...target, text, committed: isFinal }
|
|
516
|
+
state.transcript = next
|
|
517
|
+
}
|
|
509
518
|
function buildWsUrl(args) {
|
|
510
|
-
const base = new URL(args.apiBase)
|
|
511
|
-
const proto = base.protocol ===
|
|
512
|
-
const bargeQS = args.bargeIn === false ?
|
|
513
|
-
return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}
|
|
519
|
+
const base = new URL(args.apiBase)
|
|
520
|
+
const proto = base.protocol === 'https:' ? 'wss:' : 'ws:'
|
|
521
|
+
const bargeQS = args.bargeIn === false ? '&barge=off' : ''
|
|
522
|
+
return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}`
|
|
514
523
|
}
|
|
515
524
|
|
|
516
525
|
// src/clientTools.ts
|
|
517
|
-
var NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]
|
|
518
|
-
var MAX_TOOLS = 64
|
|
519
|
-
var MAX_USAGE = 500
|
|
520
|
-
var MAX_TIMEOUT_MS = 3e4
|
|
526
|
+
var NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/
|
|
527
|
+
var MAX_TOOLS = 64
|
|
528
|
+
var MAX_USAGE = 500
|
|
529
|
+
var MAX_TIMEOUT_MS = 3e4
|
|
521
530
|
var validateClientToolMap = (tools) => {
|
|
522
|
-
if (tools === void 0) return
|
|
523
|
-
if (typeof tools !==
|
|
524
|
-
throw new Error(
|
|
531
|
+
if (tools === void 0) return
|
|
532
|
+
if (typeof tools !== 'object' || tools === null || Array.isArray(tools)) {
|
|
533
|
+
throw new Error('clientTools must be an object keyed by tool name')
|
|
525
534
|
}
|
|
526
|
-
const entries = Object.entries(tools)
|
|
535
|
+
const entries = Object.entries(tools)
|
|
527
536
|
if (entries.length > MAX_TOOLS) {
|
|
528
|
-
throw new Error(`clientTools may declare at most 64 tools (got ${entries.length})`)
|
|
537
|
+
throw new Error(`clientTools may declare at most 64 tools (got ${entries.length})`)
|
|
529
538
|
}
|
|
530
539
|
for (const [name, def] of entries) {
|
|
531
540
|
if (!NAME_RE.test(name)) {
|
|
532
541
|
throw new Error(
|
|
533
|
-
`clientTools["${name}"]: name must be a valid identifier (^[a-zA-Z_][a-zA-Z0-9_]*$)
|
|
534
|
-
)
|
|
542
|
+
`clientTools["${name}"]: name must be a valid identifier (^[a-zA-Z_][a-zA-Z0-9_]*$)`,
|
|
543
|
+
)
|
|
535
544
|
}
|
|
536
|
-
if (!def || typeof def !==
|
|
537
|
-
throw new Error(`clientTools["${name}"]: must be an object`)
|
|
545
|
+
if (!def || typeof def !== 'object') {
|
|
546
|
+
throw new Error(`clientTools["${name}"]: must be an object`)
|
|
538
547
|
}
|
|
539
|
-
if (typeof def.description !==
|
|
540
|
-
throw new Error(`clientTools["${name}"]: must have a description`)
|
|
548
|
+
if (typeof def.description !== 'string' || def.description.length === 0) {
|
|
549
|
+
throw new Error(`clientTools["${name}"]: must have a description`)
|
|
541
550
|
}
|
|
542
|
-
if (typeof def.handler !==
|
|
543
|
-
throw new Error(`clientTools["${name}"]: must have a handler function`)
|
|
551
|
+
if (typeof def.handler !== 'function') {
|
|
552
|
+
throw new Error(`clientTools["${name}"]: must have a handler function`)
|
|
544
553
|
}
|
|
545
554
|
if (def.usage !== void 0 && def.usage.length > MAX_USAGE) {
|
|
546
|
-
throw new Error(`clientTools["${name}"]: usage must be \u2264500 chars`)
|
|
555
|
+
throw new Error(`clientTools["${name}"]: usage must be \u2264500 chars`)
|
|
547
556
|
}
|
|
548
|
-
if (
|
|
549
|
-
|
|
557
|
+
if (
|
|
558
|
+
def.timeoutMs !== void 0 &&
|
|
559
|
+
(!Number.isFinite(def.timeoutMs) || def.timeoutMs <= 0 || def.timeoutMs > MAX_TIMEOUT_MS)
|
|
560
|
+
) {
|
|
561
|
+
throw new Error(`clientTools["${name}"]: timeoutMs must be in (0, 30000]`)
|
|
550
562
|
}
|
|
551
563
|
}
|
|
552
|
-
}
|
|
564
|
+
}
|
|
553
565
|
var buildRegisterFrame = (tools) => ({
|
|
554
|
-
type:
|
|
566
|
+
type: 'client_tools_register',
|
|
555
567
|
tools: Object.entries(tools).map(([name, def]) => ({
|
|
556
568
|
name,
|
|
557
569
|
description: def.description,
|
|
558
570
|
parameters: def.parameters,
|
|
559
|
-
...def.usage !== void 0 ? { usage: def.usage } : {},
|
|
560
|
-
...def.timeoutMs !== void 0 ? { timeoutMs: def.timeoutMs } : {}
|
|
561
|
-
}))
|
|
562
|
-
})
|
|
571
|
+
...(def.usage !== void 0 ? { usage: def.usage } : {}),
|
|
572
|
+
...(def.timeoutMs !== void 0 ? { timeoutMs: def.timeoutMs } : {}),
|
|
573
|
+
})),
|
|
574
|
+
})
|
|
563
575
|
var dispatchClientToolCall = (send, tools, frame) => {
|
|
564
576
|
const safeSend = (payload) => {
|
|
565
577
|
try {
|
|
566
|
-
send(payload)
|
|
567
|
-
} catch {
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
const tool = tools[frame.name];
|
|
578
|
+
send(payload)
|
|
579
|
+
} catch {}
|
|
580
|
+
}
|
|
581
|
+
const tool = tools[frame.name]
|
|
571
582
|
if (!tool) {
|
|
572
583
|
safeSend({
|
|
573
|
-
type:
|
|
584
|
+
type: 'client_tool_result',
|
|
574
585
|
toolCallId: frame.toolCallId,
|
|
575
|
-
error: `No handler for ${frame.name}
|
|
576
|
-
})
|
|
577
|
-
return
|
|
586
|
+
error: `No handler for ${frame.name}`,
|
|
587
|
+
})
|
|
588
|
+
return
|
|
578
589
|
}
|
|
579
590
|
void (async () => {
|
|
580
591
|
try {
|
|
581
|
-
const out = await tool.handler(frame.args)
|
|
592
|
+
const out = await tool.handler(frame.args)
|
|
582
593
|
safeSend({
|
|
583
|
-
type:
|
|
594
|
+
type: 'client_tool_result',
|
|
584
595
|
toolCallId: frame.toolCallId,
|
|
585
|
-
result: typeof out ===
|
|
586
|
-
})
|
|
596
|
+
result: typeof out === 'string' ? out : JSON.stringify(out),
|
|
597
|
+
})
|
|
587
598
|
} catch (err) {
|
|
588
599
|
safeSend({
|
|
589
|
-
type:
|
|
600
|
+
type: 'client_tool_result',
|
|
590
601
|
toolCallId: frame.toolCallId,
|
|
591
|
-
error: err instanceof Error ? err.message : String(err)
|
|
592
|
-
})
|
|
602
|
+
error: err instanceof Error ? err.message : String(err),
|
|
603
|
+
})
|
|
593
604
|
}
|
|
594
|
-
})()
|
|
595
|
-
}
|
|
605
|
+
})()
|
|
606
|
+
}
|
|
596
607
|
|
|
597
608
|
// src/ClientMarksBuffer.ts
|
|
598
609
|
var createClientMarksBuffer = (args) => {
|
|
599
|
-
const now = args.now ?? (() => performance.now())
|
|
600
|
-
let pendingFirstOutboundAt = null
|
|
601
|
-
const inFlight = /* @__PURE__ */ new Map()
|
|
610
|
+
const now = args.now ?? (() => performance.now())
|
|
611
|
+
let pendingFirstOutboundAt = null
|
|
612
|
+
const inFlight = /* @__PURE__ */ new Map()
|
|
602
613
|
const tryEmit = (seq) => {
|
|
603
|
-
const slot = inFlight.get(seq)
|
|
604
|
-
if (!slot) return
|
|
605
|
-
if (!slot.ended) return
|
|
606
|
-
const marks = {}
|
|
614
|
+
const slot = inFlight.get(seq)
|
|
615
|
+
if (!slot) return
|
|
616
|
+
if (!slot.ended) return
|
|
617
|
+
const marks = {}
|
|
607
618
|
if (slot.firstOutboundAt !== null && slot.firstAudibleAt !== null) {
|
|
608
|
-
marks.client_mic_to_first_audible_ms = slot.firstAudibleAt - slot.firstOutboundAt
|
|
619
|
+
marks.client_mic_to_first_audible_ms = slot.firstAudibleAt - slot.firstOutboundAt
|
|
609
620
|
}
|
|
610
621
|
args.send({
|
|
611
|
-
type:
|
|
622
|
+
type: 'client_marks',
|
|
612
623
|
seq,
|
|
613
624
|
marks,
|
|
614
|
-
clientNow: Date.now()
|
|
615
|
-
})
|
|
616
|
-
inFlight.delete(seq)
|
|
617
|
-
}
|
|
625
|
+
clientNow: Date.now(),
|
|
626
|
+
})
|
|
627
|
+
inFlight.delete(seq)
|
|
628
|
+
}
|
|
618
629
|
const markFirstOutboundAudio = () => {
|
|
619
|
-
if (pendingFirstOutboundAt !== null) return
|
|
620
|
-
pendingFirstOutboundAt = now()
|
|
621
|
-
}
|
|
630
|
+
if (pendingFirstOutboundAt !== null) return
|
|
631
|
+
pendingFirstOutboundAt = now()
|
|
632
|
+
}
|
|
622
633
|
const markFirstAudibleOutput = () => {
|
|
623
|
-
let target
|
|
634
|
+
let target
|
|
624
635
|
for (const slot of inFlight.values()) {
|
|
625
636
|
if (!slot.ended) {
|
|
626
|
-
target = slot
|
|
637
|
+
target = slot
|
|
627
638
|
}
|
|
628
639
|
}
|
|
629
|
-
if (!target) return
|
|
630
|
-
if (target.firstAudibleAt !== null) return
|
|
631
|
-
target.firstAudibleAt = now()
|
|
632
|
-
}
|
|
640
|
+
if (!target) return
|
|
641
|
+
if (target.firstAudibleAt !== null) return
|
|
642
|
+
target.firstAudibleAt = now()
|
|
643
|
+
}
|
|
633
644
|
const onAgentTurnStart = (seq) => {
|
|
634
645
|
inFlight.set(seq, {
|
|
635
646
|
firstOutboundAt: pendingFirstOutboundAt,
|
|
636
647
|
firstAudibleAt: null,
|
|
637
|
-
ended: false
|
|
638
|
-
})
|
|
639
|
-
pendingFirstOutboundAt = null
|
|
640
|
-
}
|
|
648
|
+
ended: false,
|
|
649
|
+
})
|
|
650
|
+
pendingFirstOutboundAt = null
|
|
651
|
+
}
|
|
641
652
|
const onAgentTurnEnd = (seq) => {
|
|
642
|
-
const slot = inFlight.get(seq)
|
|
653
|
+
const slot = inFlight.get(seq)
|
|
643
654
|
if (!slot) {
|
|
644
|
-
args.send({ type:
|
|
645
|
-
return
|
|
655
|
+
args.send({ type: 'client_marks', seq, marks: {}, clientNow: Date.now() })
|
|
656
|
+
return
|
|
646
657
|
}
|
|
647
|
-
slot.ended = true
|
|
648
|
-
tryEmit(seq)
|
|
649
|
-
}
|
|
658
|
+
slot.ended = true
|
|
659
|
+
tryEmit(seq)
|
|
660
|
+
}
|
|
650
661
|
const flush = () => {
|
|
651
662
|
for (const seq of [...inFlight.keys()]) {
|
|
652
|
-
const slot = inFlight.get(seq)
|
|
653
|
-
slot.ended = true
|
|
654
|
-
tryEmit(seq)
|
|
663
|
+
const slot = inFlight.get(seq)
|
|
664
|
+
slot.ended = true
|
|
665
|
+
tryEmit(seq)
|
|
655
666
|
}
|
|
656
|
-
pendingFirstOutboundAt = null
|
|
657
|
-
}
|
|
667
|
+
pendingFirstOutboundAt = null
|
|
668
|
+
}
|
|
658
669
|
return {
|
|
659
670
|
markFirstOutboundAudio,
|
|
660
671
|
markFirstAudibleOutput,
|
|
661
672
|
onAgentTurnStart,
|
|
662
673
|
onAgentTurnEnd,
|
|
663
|
-
flush
|
|
664
|
-
}
|
|
665
|
-
}
|
|
674
|
+
flush,
|
|
675
|
+
}
|
|
676
|
+
}
|
|
666
677
|
|
|
667
678
|
// src/VoiceClient.ts
|
|
668
679
|
var BrowserVoiceClient = class {
|
|
669
680
|
constructor(args) {
|
|
670
|
-
this.rws = null
|
|
671
|
-
this.capture = null
|
|
672
|
-
this.playback = null
|
|
673
|
-
this.muted = false
|
|
674
|
-
this.inputVolume = 0
|
|
675
|
-
this.outputVolume = 0
|
|
676
|
-
this.startedAt = null
|
|
677
|
-
this.endedFired = false
|
|
678
|
-
this.lastError = null
|
|
681
|
+
this.rws = null
|
|
682
|
+
this.capture = null
|
|
683
|
+
this.playback = null
|
|
684
|
+
this.muted = false
|
|
685
|
+
this.inputVolume = 0
|
|
686
|
+
this.outputVolume = 0
|
|
687
|
+
this.startedAt = null
|
|
688
|
+
this.endedFired = false
|
|
689
|
+
this.lastError = null
|
|
679
690
|
this.end = () => {
|
|
680
|
-
this.teardown(
|
|
681
|
-
}
|
|
691
|
+
this.teardown('user_hangup')
|
|
692
|
+
}
|
|
682
693
|
this.mute = () => {
|
|
683
|
-
if (this.muted) return
|
|
684
|
-
this.muted = true
|
|
685
|
-
this.capture?.mute(true)
|
|
686
|
-
}
|
|
694
|
+
if (this.muted) return
|
|
695
|
+
this.muted = true
|
|
696
|
+
this.capture?.mute(true)
|
|
697
|
+
}
|
|
687
698
|
this.unmute = () => {
|
|
688
|
-
if (!this.muted) return
|
|
689
|
-
this.muted = false
|
|
690
|
-
this.capture?.mute(false)
|
|
691
|
-
}
|
|
699
|
+
if (!this.muted) return
|
|
700
|
+
this.muted = false
|
|
701
|
+
this.capture?.mute(false)
|
|
702
|
+
}
|
|
692
703
|
// ---------------------------------------------------------------
|
|
693
704
|
// Internal
|
|
694
705
|
// ---------------------------------------------------------------
|
|
695
706
|
this.sendClientToolsRegister = () => {
|
|
696
|
-
const frame = buildRegisterFrame(this.args.options.clientTools ?? {})
|
|
697
|
-
this.rws?.send(JSON.stringify(frame))
|
|
698
|
-
}
|
|
707
|
+
const frame = buildRegisterFrame(this.args.options.clientTools ?? {})
|
|
708
|
+
this.rws?.send(JSON.stringify(frame))
|
|
709
|
+
}
|
|
699
710
|
this.setState = (next) => {
|
|
700
|
-
if (this.proto.state === next) return
|
|
701
|
-
this.proto.state = next
|
|
702
|
-
this.args.options.onStateChange?.(next)
|
|
703
|
-
}
|
|
711
|
+
if (this.proto.state === next) return
|
|
712
|
+
this.proto.state = next
|
|
713
|
+
this.args.options.onStateChange?.(next)
|
|
714
|
+
}
|
|
704
715
|
this.emitError = (err) => {
|
|
705
|
-
this.lastError = err
|
|
706
|
-
this.args.options.onError?.(err)
|
|
707
|
-
}
|
|
716
|
+
this.lastError = err
|
|
717
|
+
this.args.options.onError?.(err)
|
|
718
|
+
}
|
|
708
719
|
this.handleSocketEvent = (ev) => {
|
|
709
720
|
switch (ev.type) {
|
|
710
|
-
case
|
|
711
|
-
void this.startCapture()
|
|
712
|
-
break
|
|
713
|
-
case
|
|
714
|
-
this.proto.transcript = []
|
|
715
|
-
this.proto.agentBubbleId = null
|
|
716
|
-
this.args.options.onTranscript?.(this.proto.transcript)
|
|
717
|
-
void this.startCapture()
|
|
718
|
-
this.setState(
|
|
719
|
-
break
|
|
720
|
-
case
|
|
721
|
-
if (typeof ev.data ===
|
|
721
|
+
case 'open':
|
|
722
|
+
void this.startCapture()
|
|
723
|
+
break
|
|
724
|
+
case 'reconnected':
|
|
725
|
+
this.proto.transcript = []
|
|
726
|
+
this.proto.agentBubbleId = null
|
|
727
|
+
this.args.options.onTranscript?.(this.proto.transcript)
|
|
728
|
+
void this.startCapture()
|
|
729
|
+
this.setState('listening')
|
|
730
|
+
break
|
|
731
|
+
case 'message':
|
|
732
|
+
if (typeof ev.data === 'string') {
|
|
722
733
|
handleServerMessage(ev.data, this.proto, {
|
|
723
734
|
onState: this.setState,
|
|
724
735
|
onTranscript: (entries) => this.args.options.onTranscript?.(entries),
|
|
725
736
|
onError: this.emitError,
|
|
726
737
|
onInterrupt: () => {
|
|
727
|
-
this.playback?.flush()
|
|
728
|
-
this.args.options.onInterrupt?.()
|
|
738
|
+
this.playback?.flush()
|
|
739
|
+
this.args.options.onInterrupt?.()
|
|
729
740
|
},
|
|
730
741
|
onAgentTurnStart: (seq) => {
|
|
731
|
-
if (typeof seq ===
|
|
732
|
-
this.args.options.onAgentTurnStart?.()
|
|
742
|
+
if (typeof seq === 'number') this.marks.onAgentTurnStart(seq)
|
|
743
|
+
this.args.options.onAgentTurnStart?.()
|
|
733
744
|
},
|
|
734
745
|
onAgentTurnEnd: (seq) => {
|
|
735
|
-
if (typeof seq ===
|
|
746
|
+
if (typeof seq === 'number') this.marks.onAgentTurnEnd(seq)
|
|
736
747
|
},
|
|
737
748
|
onCallEnd: (reason) => this.teardown(reason),
|
|
738
749
|
onConnected: () => this.sendClientToolsRegister(),
|
|
739
|
-
onClientToolCall: (frame) =>
|
|
740
|
-
(
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
750
|
+
onClientToolCall: (frame) =>
|
|
751
|
+
dispatchClientToolCall(
|
|
752
|
+
(f) => this.rws?.send(JSON.stringify(f)),
|
|
753
|
+
this.args.options.clientTools ?? {},
|
|
754
|
+
frame,
|
|
755
|
+
),
|
|
756
|
+
})
|
|
745
757
|
} else {
|
|
746
|
-
this.marks.markFirstAudibleOutput()
|
|
747
|
-
this.playback?.enqueue(ev.data)
|
|
758
|
+
this.marks.markFirstAudibleOutput()
|
|
759
|
+
this.playback?.enqueue(ev.data)
|
|
748
760
|
}
|
|
749
|
-
break
|
|
750
|
-
case
|
|
761
|
+
break
|
|
762
|
+
case 'close':
|
|
751
763
|
if (ev.permanent) {
|
|
752
|
-
const reason = this.proto.endReason ?? (this.lastError ?
|
|
753
|
-
this.teardown(reason)
|
|
764
|
+
const reason = this.proto.endReason ?? (this.lastError ? 'error' : 'user_hangup')
|
|
765
|
+
this.teardown(reason)
|
|
754
766
|
}
|
|
755
|
-
break
|
|
756
|
-
case
|
|
757
|
-
this.emitError({ code:
|
|
758
|
-
break
|
|
767
|
+
break
|
|
768
|
+
case 'error':
|
|
769
|
+
this.emitError({ code: 'socket_error', message: ev.error.message })
|
|
770
|
+
break
|
|
759
771
|
}
|
|
760
|
-
}
|
|
772
|
+
}
|
|
761
773
|
this.startCapture = async () => {
|
|
762
|
-
if (this.capture?.isCapturing()) return
|
|
774
|
+
if (this.capture?.isCapturing()) return
|
|
763
775
|
this.capture = createAudioCapture({
|
|
764
776
|
onChunk: (pcm) => {
|
|
765
|
-
this.marks.markFirstOutboundAudio()
|
|
766
|
-
this.rws?.send(pcm)
|
|
777
|
+
this.marks.markFirstOutboundAudio()
|
|
778
|
+
this.rws?.send(pcm)
|
|
767
779
|
},
|
|
768
780
|
onVolume: (v) => {
|
|
769
|
-
this.inputVolume = v
|
|
770
|
-
this.args.options.onVolume?.({ input: v, output: this.outputVolume })
|
|
781
|
+
this.inputVolume = v
|
|
782
|
+
this.args.options.onVolume?.({ input: v, output: this.outputVolume })
|
|
771
783
|
},
|
|
772
784
|
onError: (err) => {
|
|
773
785
|
this.emitError({
|
|
774
|
-
code: err.name ===
|
|
775
|
-
message: err.message
|
|
776
|
-
})
|
|
777
|
-
}
|
|
778
|
-
})
|
|
779
|
-
if (this.muted) this.capture.mute(true)
|
|
786
|
+
code: err.name === 'NotAllowedError' ? 'mic_denied' : 'mic_start_failed',
|
|
787
|
+
message: err.message,
|
|
788
|
+
})
|
|
789
|
+
},
|
|
790
|
+
})
|
|
791
|
+
if (this.muted) this.capture.mute(true)
|
|
780
792
|
try {
|
|
781
|
-
await this.capture.start()
|
|
782
|
-
} catch {
|
|
783
|
-
|
|
784
|
-
};
|
|
793
|
+
await this.capture.start()
|
|
794
|
+
} catch {}
|
|
795
|
+
}
|
|
785
796
|
this.teardown = (reason) => {
|
|
786
797
|
try {
|
|
787
|
-
this.marks.flush()
|
|
788
|
-
} catch {
|
|
789
|
-
|
|
790
|
-
this.capture
|
|
791
|
-
this.
|
|
792
|
-
this.playback
|
|
793
|
-
this.playback = null;
|
|
798
|
+
this.marks.flush()
|
|
799
|
+
} catch {}
|
|
800
|
+
this.capture?.stop()
|
|
801
|
+
this.capture = null
|
|
802
|
+
this.playback?.close()
|
|
803
|
+
this.playback = null
|
|
794
804
|
try {
|
|
795
|
-
this.rws?.close(1e3, reason)
|
|
796
|
-
} catch {
|
|
797
|
-
|
|
798
|
-
this.
|
|
799
|
-
this.
|
|
800
|
-
|
|
801
|
-
};
|
|
805
|
+
this.rws?.close(1e3, reason)
|
|
806
|
+
} catch {}
|
|
807
|
+
this.rws = null
|
|
808
|
+
this.setState('ended')
|
|
809
|
+
this.fireEndOnce(reason)
|
|
810
|
+
}
|
|
802
811
|
this.fireEndOnce = (reason) => {
|
|
803
|
-
if (this.endedFired) return
|
|
804
|
-
this.endedFired = true
|
|
805
|
-
const startedAt = this.startedAt ?? Date.now()
|
|
812
|
+
if (this.endedFired) return
|
|
813
|
+
this.endedFired = true
|
|
814
|
+
const startedAt = this.startedAt ?? Date.now()
|
|
806
815
|
this.args.options.onEnd?.({
|
|
807
816
|
reason,
|
|
808
|
-
errorCode: reason ===
|
|
809
|
-
durationMs: Date.now() - startedAt
|
|
810
|
-
})
|
|
811
|
-
}
|
|
812
|
-
this.args = args
|
|
813
|
-
this.proto = createProtocolState()
|
|
814
|
-
validateClientToolMap(args.options.clientTools)
|
|
817
|
+
errorCode: reason === 'error' ? this.lastError?.code : void 0,
|
|
818
|
+
durationMs: Date.now() - startedAt,
|
|
819
|
+
})
|
|
820
|
+
}
|
|
821
|
+
this.args = args
|
|
822
|
+
this.proto = createProtocolState()
|
|
823
|
+
validateClientToolMap(args.options.clientTools)
|
|
815
824
|
this.marks = createClientMarksBuffer({
|
|
816
825
|
send: (frame) => {
|
|
817
826
|
try {
|
|
818
|
-
this.rws?.send(JSON.stringify(frame))
|
|
819
|
-
} catch {
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
});
|
|
827
|
+
this.rws?.send(JSON.stringify(frame))
|
|
828
|
+
} catch {}
|
|
829
|
+
},
|
|
830
|
+
})
|
|
823
831
|
}
|
|
824
832
|
// ---------------------------------------------------------------
|
|
825
833
|
// Call interface
|
|
826
834
|
// ---------------------------------------------------------------
|
|
827
835
|
get state() {
|
|
828
|
-
return this.proto.state
|
|
836
|
+
return this.proto.state
|
|
829
837
|
}
|
|
830
838
|
get transcript() {
|
|
831
|
-
return this.proto.transcript.slice()
|
|
839
|
+
return this.proto.transcript.slice()
|
|
832
840
|
}
|
|
833
841
|
get isMuted() {
|
|
834
|
-
return this.muted
|
|
842
|
+
return this.muted
|
|
835
843
|
}
|
|
836
844
|
// ---------------------------------------------------------------
|
|
837
845
|
// Lifecycle — called by the factory immediately after construction.
|
|
@@ -839,54 +847,52 @@ var BrowserVoiceClient = class {
|
|
|
839
847
|
// failures arrive via `onError`.
|
|
840
848
|
// ---------------------------------------------------------------
|
|
841
849
|
async start() {
|
|
842
|
-
this.setState(
|
|
843
|
-
this.startedAt = Date.now()
|
|
850
|
+
this.setState('connecting')
|
|
851
|
+
this.startedAt = Date.now()
|
|
844
852
|
const url = buildWsUrl({
|
|
845
853
|
apiBase: this.args.config.apiBase,
|
|
846
854
|
agentId: this.args.options.agentId,
|
|
847
855
|
token: this.args.token,
|
|
848
|
-
bargeIn: this.args.options.bargeIn
|
|
849
|
-
})
|
|
856
|
+
bargeIn: this.args.options.bargeIn,
|
|
857
|
+
})
|
|
850
858
|
this.playback = createAudioPlayback({
|
|
851
859
|
onVolume: (v) => {
|
|
852
|
-
this.outputVolume = v
|
|
853
|
-
this.args.options.onVolume?.({ input: this.inputVolume, output: v })
|
|
854
|
-
}
|
|
855
|
-
})
|
|
860
|
+
this.outputVolume = v
|
|
861
|
+
this.args.options.onVolume?.({ input: this.inputVolume, output: v })
|
|
862
|
+
},
|
|
863
|
+
})
|
|
856
864
|
try {
|
|
857
|
-
await this.playback.resume()
|
|
858
|
-
} catch {
|
|
859
|
-
}
|
|
865
|
+
await this.playback.resume()
|
|
866
|
+
} catch {}
|
|
860
867
|
this.rws = createReconnectingWebSocket(
|
|
861
868
|
{
|
|
862
869
|
url,
|
|
863
870
|
wsFactory: this.args.wsFactory,
|
|
864
|
-
maxRetries: 3
|
|
871
|
+
maxRetries: 3,
|
|
865
872
|
},
|
|
866
|
-
(ev) => this.handleSocketEvent(ev)
|
|
867
|
-
)
|
|
873
|
+
(ev) => this.handleSocketEvent(ev),
|
|
874
|
+
)
|
|
868
875
|
}
|
|
869
|
-
}
|
|
876
|
+
}
|
|
870
877
|
|
|
871
878
|
// src/webrtc/createWebRtcCall.ts
|
|
872
879
|
async function createWebRtcCall(opts) {
|
|
873
|
-
validateClientToolMap(opts.clientTools)
|
|
874
|
-
const proto = createProtocolState()
|
|
875
|
-
let muted = false
|
|
876
|
-
let ended = false
|
|
877
|
-
const tools = opts.clientTools ?? {}
|
|
880
|
+
validateClientToolMap(opts.clientTools)
|
|
881
|
+
const proto = createProtocolState()
|
|
882
|
+
let muted = false
|
|
883
|
+
let ended = false
|
|
884
|
+
const tools = opts.clientTools ?? {}
|
|
878
885
|
const sendControl = (frame) => {
|
|
879
|
-
if (dc?.readyState !==
|
|
886
|
+
if (dc?.readyState !== 'open') return
|
|
880
887
|
try {
|
|
881
|
-
dc.send(JSON.stringify(frame))
|
|
882
|
-
} catch {
|
|
883
|
-
|
|
884
|
-
};
|
|
888
|
+
dc.send(JSON.stringify(frame))
|
|
889
|
+
} catch {}
|
|
890
|
+
}
|
|
885
891
|
const fireState = (next) => {
|
|
886
|
-
if (proto.state === next) return
|
|
887
|
-
proto.state = next
|
|
888
|
-
opts.onStateChange?.(next)
|
|
889
|
-
}
|
|
892
|
+
if (proto.state === next) return
|
|
893
|
+
proto.state = next
|
|
894
|
+
opts.onStateChange?.(next)
|
|
895
|
+
}
|
|
890
896
|
const dispatch = (raw) => {
|
|
891
897
|
handleServerMessage(raw, proto, {
|
|
892
898
|
onState: fireState,
|
|
@@ -894,188 +900,383 @@ async function createWebRtcCall(opts) {
|
|
|
894
900
|
onError: (err) => opts.onError?.(err),
|
|
895
901
|
onInterrupt: () => opts.onInterrupt?.(),
|
|
896
902
|
onAgentTurnStart: () => opts.onAgentTurnStart?.(),
|
|
897
|
-
onAgentTurnEnd: () => {
|
|
898
|
-
},
|
|
903
|
+
onAgentTurnEnd: () => {},
|
|
899
904
|
onCallEnd: () => teardown(),
|
|
900
905
|
onConnected: () => {
|
|
901
906
|
if (Object.keys(tools).length > 0) {
|
|
902
|
-
sendControl(buildRegisterFrame(tools))
|
|
907
|
+
sendControl(buildRegisterFrame(tools))
|
|
903
908
|
}
|
|
904
909
|
},
|
|
905
910
|
onClientToolCall: (frame) => {
|
|
906
|
-
dispatchClientToolCall(sendControl, tools, frame)
|
|
907
|
-
}
|
|
908
|
-
})
|
|
909
|
-
}
|
|
910
|
-
fireState(
|
|
911
|
+
dispatchClientToolCall(sendControl, tools, frame)
|
|
912
|
+
},
|
|
913
|
+
})
|
|
914
|
+
}
|
|
915
|
+
fireState('connecting')
|
|
911
916
|
const pc = new RTCPeerConnection({
|
|
912
|
-
iceServers: [{ urls:
|
|
913
|
-
})
|
|
914
|
-
const audioEl = document.createElement(
|
|
915
|
-
audioEl.autoplay = true
|
|
916
|
-
audioEl.style.display =
|
|
917
|
-
document.body.appendChild(audioEl)
|
|
917
|
+
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
|
|
918
|
+
})
|
|
919
|
+
const audioEl = document.createElement('audio')
|
|
920
|
+
audioEl.autoplay = true
|
|
921
|
+
audioEl.style.display = 'none'
|
|
922
|
+
document.body.appendChild(audioEl)
|
|
918
923
|
pc.ontrack = (event) => {
|
|
919
|
-
audioEl.srcObject = event.streams[0] ?? new MediaStream([event.track])
|
|
920
|
-
}
|
|
921
|
-
let mic
|
|
924
|
+
audioEl.srcObject = event.streams[0] ?? new MediaStream([event.track])
|
|
925
|
+
}
|
|
926
|
+
let mic
|
|
922
927
|
try {
|
|
923
|
-
mic = await navigator.mediaDevices.getUserMedia({ audio: true })
|
|
928
|
+
mic = await navigator.mediaDevices.getUserMedia({ audio: true })
|
|
924
929
|
} catch (err) {
|
|
925
|
-
const code =
|
|
930
|
+
const code =
|
|
931
|
+
err instanceof DOMException && err.name === 'NotAllowedError'
|
|
932
|
+
? 'mic_denied'
|
|
933
|
+
: 'mic_start_failed'
|
|
926
934
|
opts.onError?.({
|
|
927
935
|
code,
|
|
928
|
-
message: err instanceof Error ? err.message :
|
|
929
|
-
})
|
|
930
|
-
fireState(
|
|
931
|
-
pc.close()
|
|
932
|
-
audioEl.remove()
|
|
933
|
-
throw err
|
|
934
|
-
}
|
|
935
|
-
for (const track of mic.getAudioTracks()) pc.addTrack(track, mic)
|
|
936
|
-
const dc = pc.createDataChannel(
|
|
936
|
+
message: err instanceof Error ? err.message : 'getUserMedia failed',
|
|
937
|
+
})
|
|
938
|
+
fireState('error')
|
|
939
|
+
pc.close()
|
|
940
|
+
audioEl.remove()
|
|
941
|
+
throw err
|
|
942
|
+
}
|
|
943
|
+
for (const track of mic.getAudioTracks()) pc.addTrack(track, mic)
|
|
944
|
+
const dc = pc.createDataChannel('control', { ordered: true })
|
|
937
945
|
dc.onmessage = (e) => {
|
|
938
|
-
if (typeof e.data ===
|
|
939
|
-
}
|
|
946
|
+
if (typeof e.data === 'string') dispatch(e.data)
|
|
947
|
+
}
|
|
940
948
|
dc.onerror = () => {
|
|
941
|
-
opts.onError?.({ code:
|
|
942
|
-
}
|
|
949
|
+
opts.onError?.({ code: 'socket_error', message: 'control channel error' })
|
|
950
|
+
}
|
|
943
951
|
dc.onopen = () => {
|
|
944
952
|
if (Object.keys(tools).length > 0) {
|
|
945
|
-
sendControl(buildRegisterFrame(tools))
|
|
953
|
+
sendControl(buildRegisterFrame(tools))
|
|
946
954
|
}
|
|
947
|
-
}
|
|
948
|
-
const gateway = opts.webrtcGatewayBase ||
|
|
949
|
-
const offerUrl = gateway
|
|
950
|
-
|
|
955
|
+
}
|
|
956
|
+
const gateway = opts.webrtcGatewayBase || ''
|
|
957
|
+
const offerUrl = gateway
|
|
958
|
+
? `${gateway}/webrtc/offer?token=${encodeURIComponent(opts.token)}`
|
|
959
|
+
: `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/offer?token=${encodeURIComponent(opts.token)}`
|
|
960
|
+
const iceUrl = gateway
|
|
961
|
+
? `${gateway}/webrtc/ice?token=${encodeURIComponent(opts.token)}`
|
|
962
|
+
: `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/ice?token=${encodeURIComponent(opts.token)}`
|
|
951
963
|
const teardown = () => {
|
|
952
|
-
if (ended) return
|
|
953
|
-
ended = true
|
|
964
|
+
if (ended) return
|
|
965
|
+
ended = true
|
|
954
966
|
try {
|
|
955
|
-
mic.getTracks().forEach((t) => t.stop())
|
|
956
|
-
} catch {
|
|
957
|
-
}
|
|
967
|
+
mic.getTracks().forEach((t) => t.stop())
|
|
968
|
+
} catch {}
|
|
958
969
|
try {
|
|
959
|
-
pc.close()
|
|
960
|
-
} catch {
|
|
961
|
-
}
|
|
970
|
+
pc.close()
|
|
971
|
+
} catch {}
|
|
962
972
|
try {
|
|
963
|
-
audioEl.remove()
|
|
964
|
-
} catch {
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
const pendingCandidates = [];
|
|
973
|
+
audioEl.remove()
|
|
974
|
+
} catch {}
|
|
975
|
+
fireState('ended')
|
|
976
|
+
opts.onEnd?.()
|
|
977
|
+
}
|
|
978
|
+
let callId = null
|
|
979
|
+
const pendingCandidates = []
|
|
971
980
|
const postCandidate = (candidate) => {
|
|
972
981
|
void fetch(iceUrl, {
|
|
973
|
-
method:
|
|
974
|
-
headers: {
|
|
975
|
-
body: JSON.stringify({ callId, candidate })
|
|
976
|
-
}).catch(() => {
|
|
977
|
-
|
|
978
|
-
};
|
|
982
|
+
method: 'POST',
|
|
983
|
+
headers: { 'content-type': 'application/json' },
|
|
984
|
+
body: JSON.stringify({ callId, candidate }),
|
|
985
|
+
}).catch(() => {})
|
|
986
|
+
}
|
|
979
987
|
pc.onicecandidate = (e) => {
|
|
980
|
-
if (!e.candidate) return
|
|
981
|
-
if (callId) postCandidate(e.candidate)
|
|
982
|
-
else pendingCandidates.push(e.candidate)
|
|
983
|
-
}
|
|
988
|
+
if (!e.candidate) return
|
|
989
|
+
if (callId) postCandidate(e.candidate)
|
|
990
|
+
else pendingCandidates.push(e.candidate)
|
|
991
|
+
}
|
|
984
992
|
pc.onconnectionstatechange = () => {
|
|
985
|
-
const s = pc.connectionState
|
|
986
|
-
if (s ===
|
|
987
|
-
if (s ===
|
|
988
|
-
opts.onError?.({ code:
|
|
989
|
-
teardown()
|
|
993
|
+
const s = pc.connectionState
|
|
994
|
+
if (s === 'connected') fireState('listening')
|
|
995
|
+
if (s === 'failed' || s === 'disconnected') {
|
|
996
|
+
opts.onError?.({ code: 'socket_error', message: `webrtc connection ${s}` })
|
|
997
|
+
teardown()
|
|
990
998
|
}
|
|
991
|
-
if (s ===
|
|
992
|
-
}
|
|
993
|
-
await pc.setLocalDescription(await pc.createOffer())
|
|
999
|
+
if (s === 'closed' && !ended) teardown()
|
|
1000
|
+
}
|
|
1001
|
+
await pc.setLocalDescription(await pc.createOffer())
|
|
994
1002
|
try {
|
|
995
1003
|
const offerRes = await fetch(offerUrl, {
|
|
996
|
-
method:
|
|
997
|
-
headers: {
|
|
998
|
-
body: JSON.stringify({ sdp: pc.localDescription.sdp, type:
|
|
999
|
-
})
|
|
1004
|
+
method: 'POST',
|
|
1005
|
+
headers: { 'content-type': 'application/json' },
|
|
1006
|
+
body: JSON.stringify({ sdp: pc.localDescription.sdp, type: 'offer', agentId: opts.agentId }),
|
|
1007
|
+
})
|
|
1000
1008
|
if (!offerRes.ok) {
|
|
1001
|
-
const code = offerRes.status === 401 ?
|
|
1002
|
-
opts.onError?.({ code, message: `signaling failed: HTTP ${offerRes.status}` })
|
|
1003
|
-
fireState(
|
|
1004
|
-
mic.getTracks().forEach((t) => t.stop())
|
|
1005
|
-
pc.close()
|
|
1006
|
-
audioEl.remove()
|
|
1007
|
-
throw new Error(`webrtc offer failed: ${offerRes.status}`)
|
|
1009
|
+
const code = offerRes.status === 401 ? 'unauthorized' : 'server_error'
|
|
1010
|
+
opts.onError?.({ code, message: `signaling failed: HTTP ${offerRes.status}` })
|
|
1011
|
+
fireState('error')
|
|
1012
|
+
mic.getTracks().forEach((t) => t.stop())
|
|
1013
|
+
pc.close()
|
|
1014
|
+
audioEl.remove()
|
|
1015
|
+
throw new Error(`webrtc offer failed: ${offerRes.status}`)
|
|
1008
1016
|
}
|
|
1009
|
-
const body = await offerRes.json()
|
|
1010
|
-
callId = body.callId
|
|
1011
|
-
await pc.setRemoteDescription({ type:
|
|
1012
|
-
while (pendingCandidates.length > 0) postCandidate(pendingCandidates.shift())
|
|
1017
|
+
const body = await offerRes.json()
|
|
1018
|
+
callId = body.callId
|
|
1019
|
+
await pc.setRemoteDescription({ type: 'answer', sdp: body.sdp })
|
|
1020
|
+
while (pendingCandidates.length > 0) postCandidate(pendingCandidates.shift())
|
|
1013
1021
|
} catch (err) {
|
|
1014
1022
|
if (!ended) {
|
|
1015
1023
|
opts.onError?.({
|
|
1016
|
-
code:
|
|
1017
|
-
message: err instanceof Error ? err.message :
|
|
1018
|
-
})
|
|
1019
|
-
fireState(
|
|
1020
|
-
mic.getTracks().forEach((t) => t.stop())
|
|
1021
|
-
pc.close()
|
|
1022
|
-
audioEl.remove()
|
|
1024
|
+
code: 'network_unreachable',
|
|
1025
|
+
message: err instanceof Error ? err.message : 'signaling failed',
|
|
1026
|
+
})
|
|
1027
|
+
fireState('error')
|
|
1028
|
+
mic.getTracks().forEach((t) => t.stop())
|
|
1029
|
+
pc.close()
|
|
1030
|
+
audioEl.remove()
|
|
1023
1031
|
}
|
|
1024
|
-
throw err
|
|
1032
|
+
throw err
|
|
1025
1033
|
}
|
|
1026
1034
|
return {
|
|
1027
1035
|
get state() {
|
|
1028
|
-
return proto.state
|
|
1036
|
+
return proto.state
|
|
1029
1037
|
},
|
|
1030
1038
|
get transcript() {
|
|
1031
|
-
return proto.transcript.slice()
|
|
1039
|
+
return proto.transcript.slice()
|
|
1032
1040
|
},
|
|
1033
1041
|
get isMuted() {
|
|
1034
|
-
return muted
|
|
1042
|
+
return muted
|
|
1035
1043
|
},
|
|
1036
1044
|
end: () => teardown(),
|
|
1037
1045
|
mute: () => {
|
|
1038
|
-
if (muted) return
|
|
1039
|
-
muted = true
|
|
1040
|
-
mic.getAudioTracks().forEach((t) => t.enabled = false)
|
|
1046
|
+
if (muted) return
|
|
1047
|
+
muted = true
|
|
1048
|
+
mic.getAudioTracks().forEach((t) => (t.enabled = false))
|
|
1041
1049
|
},
|
|
1042
1050
|
unmute: () => {
|
|
1043
|
-
if (!muted) return
|
|
1044
|
-
muted = false
|
|
1045
|
-
mic.getAudioTracks().forEach((t) => t.enabled = true)
|
|
1051
|
+
if (!muted) return
|
|
1052
|
+
muted = false
|
|
1053
|
+
mic.getAudioTracks().forEach((t) => (t.enabled = true))
|
|
1054
|
+
},
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// src/room.ts
|
|
1059
|
+
var import_livekit_client = require('livekit-client')
|
|
1060
|
+
|
|
1061
|
+
// src/roomProtocol.ts
|
|
1062
|
+
var SYSTEM_TOPIC = 'system'
|
|
1063
|
+
var TRANSCRIPT_TOPIC = 'transcript'
|
|
1064
|
+
var decodeSystem = (bytes) => {
|
|
1065
|
+
try {
|
|
1066
|
+
const v = JSON.parse(new TextDecoder().decode(bytes))
|
|
1067
|
+
if (v && typeof v.kind === 'string') return v
|
|
1068
|
+
return null
|
|
1069
|
+
} catch {
|
|
1070
|
+
return null
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
var decodeTranscript = (bytes) => {
|
|
1074
|
+
try {
|
|
1075
|
+
const v = JSON.parse(new TextDecoder().decode(bytes))
|
|
1076
|
+
if (v && v.kind === 'partial') return v
|
|
1077
|
+
return null
|
|
1078
|
+
} catch {
|
|
1079
|
+
return null
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// src/room.ts
|
|
1084
|
+
var identityToPid = (identity) =>
|
|
1085
|
+
identity.startsWith('guest:') ? identity.slice('guest:'.length) : identity
|
|
1086
|
+
var joinRoom = async (opts) => {
|
|
1087
|
+
const exchangeUrl = `${opts.apiBase.replace(/\/+$/, '')}/v1/rooms/${encodeURIComponent(
|
|
1088
|
+
opts.roomId,
|
|
1089
|
+
)}/join`
|
|
1090
|
+
const exchangeRes = await fetch(exchangeUrl, {
|
|
1091
|
+
method: 'POST',
|
|
1092
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1093
|
+
body: JSON.stringify({ code: opts.joinCode, name: opts.name }),
|
|
1094
|
+
})
|
|
1095
|
+
if (!exchangeRes.ok) {
|
|
1096
|
+
const err = await exchangeRes.json().catch(() => ({}))
|
|
1097
|
+
throw new Error(err.error?.code ?? `join_failed_${exchangeRes.status}`)
|
|
1098
|
+
}
|
|
1099
|
+
const exchange = await exchangeRes.json()
|
|
1100
|
+
const handlers = /* @__PURE__ */ new Map()
|
|
1101
|
+
const emit = (e, payload) => {
|
|
1102
|
+
handlers.get(e)?.forEach((h) => {
|
|
1103
|
+
try {
|
|
1104
|
+
h(payload)
|
|
1105
|
+
} catch {}
|
|
1106
|
+
})
|
|
1107
|
+
}
|
|
1108
|
+
const room = new import_livekit_client.Room({ adaptiveStream: true, dynacast: true })
|
|
1109
|
+
room.on(import_livekit_client.RoomEvent.ParticipantConnected, (p) =>
|
|
1110
|
+
emit('participant.joined', {
|
|
1111
|
+
participantId: identityToPid(p.identity),
|
|
1112
|
+
name: p.name ?? '',
|
|
1113
|
+
}),
|
|
1114
|
+
)
|
|
1115
|
+
room.on(import_livekit_client.RoomEvent.ParticipantDisconnected, (p) =>
|
|
1116
|
+
emit('participant.left', {
|
|
1117
|
+
participantId: identityToPid(p.identity),
|
|
1118
|
+
name: p.name ?? '',
|
|
1119
|
+
}),
|
|
1120
|
+
)
|
|
1121
|
+
room.on(import_livekit_client.RoomEvent.Disconnected, () => emit('room.ended', void 0))
|
|
1122
|
+
room.on(import_livekit_client.RoomEvent.DataReceived, (data, _participant, _kind, topic) => {
|
|
1123
|
+
if (topic === SYSTEM_TOPIC) {
|
|
1124
|
+
const m = decodeSystem(data)
|
|
1125
|
+
if (m) emit('system.message', m)
|
|
1126
|
+
} else if (topic === TRANSCRIPT_TOPIC) {
|
|
1127
|
+
const m = decodeTranscript(data)
|
|
1128
|
+
if (m) emit('transcript.partial', m)
|
|
1129
|
+
}
|
|
1130
|
+
})
|
|
1131
|
+
const trackKind = (t) => (t.kind === import_livekit_client.Track.Kind.Video ? 'video' : 'audio')
|
|
1132
|
+
const trackSource = (s) => {
|
|
1133
|
+
switch (s) {
|
|
1134
|
+
case import_livekit_client.Track.Source.Camera:
|
|
1135
|
+
return 'camera'
|
|
1136
|
+
case import_livekit_client.Track.Source.Microphone:
|
|
1137
|
+
return 'microphone'
|
|
1138
|
+
case import_livekit_client.Track.Source.ScreenShare:
|
|
1139
|
+
return 'screen_share'
|
|
1140
|
+
case import_livekit_client.Track.Source.ScreenShareAudio:
|
|
1141
|
+
return 'screen_share_audio'
|
|
1142
|
+
default:
|
|
1143
|
+
return 'unknown'
|
|
1046
1144
|
}
|
|
1047
|
-
}
|
|
1145
|
+
}
|
|
1146
|
+
room.on(import_livekit_client.RoomEvent.TrackSubscribed, (track, pub, participant) =>
|
|
1147
|
+
emit('track.subscribed', {
|
|
1148
|
+
participantId: identityToPid(participant.identity),
|
|
1149
|
+
kind: trackKind(track),
|
|
1150
|
+
source: trackSource(pub.source),
|
|
1151
|
+
track,
|
|
1152
|
+
}),
|
|
1153
|
+
)
|
|
1154
|
+
room.on(import_livekit_client.RoomEvent.TrackUnsubscribed, (track, pub, participant) =>
|
|
1155
|
+
emit('track.unsubscribed', {
|
|
1156
|
+
participantId: identityToPid(participant.identity),
|
|
1157
|
+
kind: trackKind(track),
|
|
1158
|
+
source: trackSource(pub.source),
|
|
1159
|
+
track,
|
|
1160
|
+
}),
|
|
1161
|
+
)
|
|
1162
|
+
room.on(import_livekit_client.RoomEvent.ActiveSpeakersChanged, (speakers) =>
|
|
1163
|
+
emit(
|
|
1164
|
+
'active.speakers',
|
|
1165
|
+
speakers.map((p) => identityToPid(p.identity)),
|
|
1166
|
+
),
|
|
1167
|
+
)
|
|
1168
|
+
await room.connect(exchange.livekit.url, exchange.livekit.token)
|
|
1169
|
+
return {
|
|
1170
|
+
participantId: exchange.participantId,
|
|
1171
|
+
get participants() {
|
|
1172
|
+
return [...room.remoteParticipants.values()].map((p) => ({
|
|
1173
|
+
participantId: identityToPid(p.identity),
|
|
1174
|
+
name: p.name ?? '',
|
|
1175
|
+
}))
|
|
1176
|
+
},
|
|
1177
|
+
on(event, handler) {
|
|
1178
|
+
const set = handlers.get(event) ?? /* @__PURE__ */ new Set()
|
|
1179
|
+
set.add(handler)
|
|
1180
|
+
handlers.set(event, set)
|
|
1181
|
+
},
|
|
1182
|
+
publishMic: async () => {
|
|
1183
|
+
await room.localParticipant.setMicrophoneEnabled(true)
|
|
1184
|
+
},
|
|
1185
|
+
publishCamera: async () => {
|
|
1186
|
+
await room.localParticipant.setCameraEnabled(true)
|
|
1187
|
+
},
|
|
1188
|
+
setMicEnabled: async (on) => {
|
|
1189
|
+
await room.localParticipant.setMicrophoneEnabled(on)
|
|
1190
|
+
},
|
|
1191
|
+
setCameraEnabled: async (on) => {
|
|
1192
|
+
await room.localParticipant.setCameraEnabled(on)
|
|
1193
|
+
},
|
|
1194
|
+
isMicEnabled: () => room.localParticipant.isMicrophoneEnabled,
|
|
1195
|
+
isCameraEnabled: () => room.localParticipant.isCameraEnabled,
|
|
1196
|
+
getLocalCameraTrack: () =>
|
|
1197
|
+
room.localParticipant.getTrackPublication(import_livekit_client.Track.Source.Camera)
|
|
1198
|
+
?.videoTrack ?? null,
|
|
1199
|
+
getRemoteTracks: () => {
|
|
1200
|
+
const out = []
|
|
1201
|
+
for (const p of room.remoteParticipants.values()) {
|
|
1202
|
+
for (const pub of p.trackPublications.values()) {
|
|
1203
|
+
const track = pub.track
|
|
1204
|
+
if (!track) continue
|
|
1205
|
+
out.push({
|
|
1206
|
+
participantId: identityToPid(p.identity),
|
|
1207
|
+
kind: trackKind(track),
|
|
1208
|
+
source: trackSource(pub.source),
|
|
1209
|
+
track,
|
|
1210
|
+
})
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
return out
|
|
1214
|
+
},
|
|
1215
|
+
setScreenShareEnabled: async (on, opts2) => {
|
|
1216
|
+
await room.localParticipant.setScreenShareEnabled(on, { audio: opts2?.audio ?? false })
|
|
1217
|
+
},
|
|
1218
|
+
isScreenShareEnabled: () => room.localParticipant.isScreenShareEnabled,
|
|
1219
|
+
getLocalScreenTrack: () =>
|
|
1220
|
+
room.localParticipant.getTrackPublication(import_livekit_client.Track.Source.ScreenShare)
|
|
1221
|
+
?.videoTrack ?? null,
|
|
1222
|
+
leave: async () => {
|
|
1223
|
+
await room.disconnect()
|
|
1224
|
+
},
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
// src/incomingCall.ts
|
|
1229
|
+
var parseIncomingCall = (raw) => {
|
|
1230
|
+
if (typeof raw !== 'object' || raw === null) {
|
|
1231
|
+
throw new Error('parseIncomingCall: payload must be an object')
|
|
1232
|
+
}
|
|
1233
|
+
const p = raw
|
|
1234
|
+
if (typeof p.token !== 'string' || !p.token.startsWith('ct_')) {
|
|
1235
|
+
throw new Error('parseIncomingCall: missing or invalid `token` (expected a ct_ string)')
|
|
1236
|
+
}
|
|
1237
|
+
if (typeof p.agentId !== 'string' || p.agentId.length === 0) {
|
|
1238
|
+
throw new Error('parseIncomingCall: missing `agentId`')
|
|
1239
|
+
}
|
|
1240
|
+
const transport = p.transport === 'webrtc' ? 'webrtc' : 'ws'
|
|
1241
|
+
const out = { token: p.token, agentId: p.agentId, transport }
|
|
1242
|
+
if (transport === 'webrtc' && typeof p.webrtcGatewayBase === 'string') {
|
|
1243
|
+
out.webrtcGatewayBase = p.webrtcGatewayBase
|
|
1244
|
+
}
|
|
1245
|
+
if (typeof p.expiresAt === 'number') out.expiresAt = p.expiresAt
|
|
1246
|
+
if (typeof p.agentName === 'string') out.agentName = p.agentName
|
|
1247
|
+
if (typeof p.agentAvatarUrl === 'string') out.agentAvatarUrl = p.agentAvatarUrl
|
|
1248
|
+
return out
|
|
1048
1249
|
}
|
|
1049
1250
|
|
|
1050
1251
|
// src/browser.ts
|
|
1051
|
-
var browserWsFactory = (url) => new globalThis.WebSocket(url)
|
|
1252
|
+
var browserWsFactory = (url) => new globalThis.WebSocket(url)
|
|
1052
1253
|
var BrowserVoiceFactory = class {
|
|
1053
1254
|
constructor(config) {
|
|
1054
1255
|
this.startCall = async (options) => {
|
|
1055
1256
|
if (!options.agentId) {
|
|
1056
|
-
throw new Error(
|
|
1257
|
+
throw new Error('startCall: agentId is required')
|
|
1057
1258
|
}
|
|
1058
|
-
const { context, metadata } = mergeStartCallContext(this.config, options)
|
|
1259
|
+
const { context, metadata } = mergeStartCallContext(this.config, options)
|
|
1059
1260
|
const fetchArgs = {
|
|
1060
1261
|
agentId: options.agentId,
|
|
1061
1262
|
userId: options.userId,
|
|
1062
1263
|
context,
|
|
1063
|
-
metadata
|
|
1064
|
-
}
|
|
1065
|
-
let resolved
|
|
1264
|
+
metadata,
|
|
1265
|
+
}
|
|
1266
|
+
let resolved
|
|
1066
1267
|
if (options.token) {
|
|
1067
|
-
resolved = { token: options.token, transport:
|
|
1268
|
+
resolved = { token: options.token, transport: 'ws' }
|
|
1068
1269
|
} else {
|
|
1069
|
-
const r = await this.config.fetchToken(fetchArgs)
|
|
1270
|
+
const r = await this.config.fetchToken(fetchArgs)
|
|
1070
1271
|
if (!r) {
|
|
1071
|
-
throw new Error(
|
|
1272
|
+
throw new Error('configureVoiceClient.fetchToken returned empty token')
|
|
1072
1273
|
}
|
|
1073
|
-
resolved = typeof r ===
|
|
1274
|
+
resolved = typeof r === 'string' ? { token: r, transport: 'ws' } : r
|
|
1074
1275
|
if (!resolved.token) {
|
|
1075
|
-
throw new Error(
|
|
1276
|
+
throw new Error('configureVoiceClient.fetchToken returned an object without `token`')
|
|
1076
1277
|
}
|
|
1077
1278
|
}
|
|
1078
|
-
if (resolved.transport ===
|
|
1279
|
+
if (resolved.transport === 'webrtc') {
|
|
1079
1280
|
return createWebRtcCall({
|
|
1080
1281
|
agentId: options.agentId,
|
|
1081
1282
|
apiBase: this.config.apiBase,
|
|
@@ -1087,11 +1288,13 @@ var BrowserVoiceFactory = class {
|
|
|
1087
1288
|
// Synthesise a minimal CallEndEvent. WebRTC doesn't carry an end reason
|
|
1088
1289
|
// from the server yet — use 'agent_ended' as placeholder. durationMs is
|
|
1089
1290
|
// tracked at 0 until the followup lands (see spec Followups section).
|
|
1090
|
-
onEnd: options.onEnd
|
|
1291
|
+
onEnd: options.onEnd
|
|
1292
|
+
? () => options.onEnd({ reason: 'agent_ended', durationMs: 0 })
|
|
1293
|
+
: void 0,
|
|
1091
1294
|
onInterrupt: options.onInterrupt,
|
|
1092
1295
|
onAgentTurnStart: options.onAgentTurnStart,
|
|
1093
|
-
clientTools: options.clientTools
|
|
1094
|
-
})
|
|
1296
|
+
clientTools: options.clientTools,
|
|
1297
|
+
})
|
|
1095
1298
|
}
|
|
1096
1299
|
const client = new BrowserVoiceClient({
|
|
1097
1300
|
config: this.config,
|
|
@@ -1099,25 +1302,36 @@ var BrowserVoiceFactory = class {
|
|
|
1099
1302
|
// see what the SDK saw.
|
|
1100
1303
|
options: { ...options, context, metadata },
|
|
1101
1304
|
token: resolved.token,
|
|
1102
|
-
wsFactory: browserWsFactory
|
|
1103
|
-
})
|
|
1104
|
-
await client.start()
|
|
1105
|
-
return client
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1305
|
+
wsFactory: browserWsFactory,
|
|
1306
|
+
})
|
|
1307
|
+
await client.start()
|
|
1308
|
+
return client
|
|
1309
|
+
}
|
|
1310
|
+
// Multi-party rooms (Phase 7 video).
|
|
1311
|
+
//
|
|
1312
|
+
// The guest's browser calls this with the roomId + joinCode it parsed
|
|
1313
|
+
// out of the invite link. The SDK exchanges the code for a LiveKit
|
|
1314
|
+
// JWT against `${apiBase}/v1/rooms/:roomId/join` (an AUTH-EXEMPT
|
|
1315
|
+
// endpoint — the opaque code is the only credential), then connects
|
|
1316
|
+
// to LiveKit and returns a typed event surface.
|
|
1317
|
+
this.joinRoom = (opts) => joinRoom({ apiBase: this.config.apiBase, ...opts })
|
|
1318
|
+
this.config = config
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1110
1321
|
function configureVoiceClient(config) {
|
|
1111
|
-
return new BrowserVoiceFactory(normalizeConfig(config))
|
|
1322
|
+
return new BrowserVoiceFactory(normalizeConfig(config))
|
|
1112
1323
|
}
|
|
1113
1324
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1114
|
-
0 &&
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1325
|
+
0 &&
|
|
1326
|
+
(module.exports = {
|
|
1327
|
+
buildWsUrl,
|
|
1328
|
+
configureVoiceClient,
|
|
1329
|
+
createAudioCapture,
|
|
1330
|
+
createAudioPlayback,
|
|
1331
|
+
createProtocolState,
|
|
1332
|
+
createReconnectingWebSocket,
|
|
1333
|
+
handleServerMessage,
|
|
1334
|
+
joinRoom,
|
|
1335
|
+
parseIncomingCall,
|
|
1336
|
+
})
|
|
1337
|
+
//# sourceMappingURL=browser.js.map
|