@craftedxp/voice-js 0.5.4 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONSUMING.md +6 -2
- package/README.md +31 -4
- package/dist/assistant.d.mts +32 -0
- package/dist/assistant.d.ts +32 -0
- package/dist/assistant.js +1241 -0
- package/dist/assistant.js.map +1 -0
- package/dist/assistant.mjs +23 -0
- package/dist/assistant.mjs.map +1 -0
- package/dist/browser.d.mts +12 -509
- package/dist/browser.d.ts +12 -608
- package/dist/browser.js +1020 -896
- package/dist/browser.js.map +1 -1
- package/dist/browser.mjs +25 -1283
- package/dist/browser.mjs.map +1 -1
- package/dist/chunk-LV7JGPYW.mjs +200 -0
- package/dist/chunk-LV7JGPYW.mjs.map +1 -0
- package/dist/chunk-ZW22Y67M.mjs +1208 -0
- package/dist/chunk-ZW22Y67M.mjs.map +1 -0
- package/dist/config-D2TbvIqT.d.mts +297 -0
- package/dist/config-D2TbvIqT.d.ts +297 -0
- package/dist/embed.iife.js +30 -23358
- package/dist/incomingCall-CfRRzj2P.d.mts +103 -0
- package/dist/incomingCall-CfRRzj2P.d.ts +103 -0
- package/dist/node.d.mts +69 -139
- package/dist/node.d.ts +342 -496
- package/dist/node.js +472 -467
- package/dist/node.js.map +1 -1
- package/dist/node.mjs +19 -0
- package/dist/node.mjs.map +1 -1
- package/dist/room.d.mts +156 -0
- package/dist/room.d.ts +156 -0
- package/dist/room.js +236 -0
- package/dist/room.js.map +1 -0
- package/dist/room.mjs +7 -0
- package/dist/room.mjs.map +1 -0
- package/dist/transcribe.d.mts +14 -0
- package/dist/transcribe.d.ts +14 -0
- package/dist/transcribe.js +1213 -0
- package/dist/transcribe.js.map +1 -0
- package/dist/transcribe.mjs +18 -0
- package/dist/transcribe.mjs.map +1 -0
- package/package.json +22 -4
package/dist/browser.js
CHANGED
|
@@ -1,29 +1,37 @@
|
|
|
1
|
-
|
|
2
|
-
var
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
var
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
}
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
9
12
|
var __copyProps = (to, from, except, desc) => {
|
|
10
|
-
if (
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
14
|
for (let key of __getOwnPropNames(from))
|
|
12
15
|
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
-
__defProp(to, key, {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
21
29
|
|
|
22
30
|
// src/browser.ts
|
|
23
|
-
var browser_exports = {}
|
|
31
|
+
var browser_exports = {};
|
|
24
32
|
__export(browser_exports, {
|
|
25
33
|
buildWsUrl: () => buildWsUrl,
|
|
26
|
-
configureVoiceClient: () =>
|
|
34
|
+
configureVoiceClient: () => configureVoiceClient2,
|
|
27
35
|
createAudioCapture: () => createAudioCapture,
|
|
28
36
|
createAudioPlayback: () => createAudioPlayback,
|
|
29
37
|
createProtocolState: () => createProtocolState,
|
|
@@ -31,63 +39,57 @@ __export(browser_exports, {
|
|
|
31
39
|
handleServerMessage: () => handleServerMessage,
|
|
32
40
|
joinRoom: () => joinRoom,
|
|
33
41
|
parseIncomingCall: () => parseIncomingCall,
|
|
34
|
-
|
|
35
|
-
|
|
42
|
+
startTextSession: () => startTextSession
|
|
43
|
+
});
|
|
44
|
+
module.exports = __toCommonJS(browser_exports);
|
|
36
45
|
|
|
37
46
|
// src/config.ts
|
|
38
47
|
function normalizeConfig(config) {
|
|
39
|
-
if (!config) throw new Error(
|
|
40
|
-
if (
|
|
48
|
+
if (!config) throw new Error("configureVoiceClient: config is required");
|
|
49
|
+
if ("apiKey" in config) {
|
|
41
50
|
throw new Error(
|
|
42
|
-
|
|
43
|
-
)
|
|
51
|
+
"configureVoiceClient: `apiKey` is no longer supported. Embedding sk_ in JS code ships server-grade credentials to every client. Pass `fetchToken: async ({ agentId }) => { /* call YOUR backend mint */ }` instead \u2014 see the @craftedxp/voice-js README for the migration recipe."
|
|
52
|
+
);
|
|
44
53
|
}
|
|
45
54
|
if (!config.apiBase) {
|
|
46
|
-
throw new Error(
|
|
55
|
+
throw new Error("configureVoiceClient: apiBase is required");
|
|
47
56
|
}
|
|
48
|
-
if (typeof config.fetchToken !==
|
|
49
|
-
throw new Error(
|
|
57
|
+
if (typeof config.fetchToken !== "function") {
|
|
58
|
+
throw new Error("configureVoiceClient: fetchToken must be a function");
|
|
50
59
|
}
|
|
51
60
|
return {
|
|
52
61
|
...config,
|
|
53
|
-
apiBase: config.apiBase.replace(/\/+$/,
|
|
54
|
-
}
|
|
62
|
+
apiBase: config.apiBase.replace(/\/+$/, "")
|
|
63
|
+
};
|
|
55
64
|
}
|
|
56
65
|
function mergeStartCallContext(factory, call) {
|
|
57
|
-
const context =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
: void 0
|
|
61
|
-
const metadata =
|
|
62
|
-
factory.defaultMetadata || call.metadata
|
|
63
|
-
? { ...(factory.defaultMetadata ?? {}), ...(call.metadata ?? {}) }
|
|
64
|
-
: void 0
|
|
65
|
-
return { context, metadata }
|
|
66
|
+
const context = factory.defaultContext || call.context ? { ...factory.defaultContext ?? {}, ...call.context ?? {} } : void 0;
|
|
67
|
+
const metadata = factory.defaultMetadata || call.metadata ? { ...factory.defaultMetadata ?? {}, ...call.metadata ?? {} } : void 0;
|
|
68
|
+
return { context, metadata };
|
|
66
69
|
}
|
|
67
70
|
|
|
68
71
|
// src/worklets/mic-downsampler.worklet.js
|
|
69
|
-
var mic_downsampler_worklet_default =
|
|
70
|
-
"// AudioWorklet \u2014 runs off the main thread in the audio rendering graph.\n//\n// Input: Float32 samples at the AudioContext's native sampleRate (typically\n// 48000 Hz on desktop, 44100 Hz on some iOS devices).\n// Output: 16 kHz mono Int16 PCM, shipped to the main thread via\n// `port.postMessage(ArrayBuffer, [ArrayBuffer])` (transferred, not copied).\n//\n// Why AudioWorklet instead of ScriptProcessorNode: ScriptProcessorNode is\n// deprecated + main-thread-bound, so any JS jank produces audible audio\n// glitches. AudioWorklet's `process()` runs on the audio rendering thread\n// at the graph's block cadence (128 frames by default) and backpressures\n// via returning `true` / `false`.\n//\n// This file is loaded as text (see tsup.config.ts loader) and registered\n// at runtime via `audioWorklet.addModule(blobUrl)`.\n\nclass MicDownsampler extends AudioWorkletProcessor {\n constructor() {\n super()\n // Target sample rate for STT. Matches Deepgram Nova-3 + the platform's\n // server-side SAMPLE_RATE constant in AgentCallHandler.\n this.targetRate = 16000\n // Accumulator for the downsample. We collect incoming samples and emit\n // an Int16 chunk when we've accumulated ~1024 target-rate samples\n // (~64 ms at 16 kHz) \u2014 matches the mobile SDK's chunk size so both\n // platforms have the same server-side framing.\n this.outputFrames = 1024\n this.acc = []\n // Running index used for fractional resampling.\n this.readCursor = 0\n }\n\n // `inputs[0][0]` = first channel of first input. 128 Float32 samples per\n // call at the context's sampleRate. Return true = keep processing.\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n const channel = input[0]\n if (!channel || channel.length === 0) return true\n\n const ctxRate = sampleRate // global inside AudioWorkletProcessor\n const ratio = ctxRate / this.targetRate\n\n // Simple linear-interp downsample. For 48000 \u2192 16000 that's 3:1, which\n // linear handles fine for voice. Anti-alias filtering would be\n // theoretically better but inaudible for speech.\n for (let i = 0; i < channel.length; i++) {\n this.acc.push(channel[i])\n }\n\n while (this.acc.length - this.readCursor >= ratio * this.outputFrames) {\n const out = new Int16Array(this.outputFrames)\n let readIdx = this.readCursor\n for (let i = 0; i < this.outputFrames; i++) {\n // Linear interp between floor(readIdx) and ceil(readIdx)\n const low = Math.floor(readIdx)\n const high = Math.min(low + 1, this.acc.length - 1)\n const frac = readIdx - low\n const sample = this.acc[low] * (1 - frac) + this.acc[high] * frac\n // Clip + convert to int16\n const clipped = Math.max(-1, Math.min(1, sample))\n out[i] = clipped < 0 ? clipped * 0x8000 : clipped * 0x7fff\n readIdx += ratio\n }\n // Transfer the ArrayBuffer (zero-copy) to the main thread.\n this.port.postMessage(out.buffer, [out.buffer])\n this.readCursor = readIdx\n }\n\n // Garbage-collect the consumed portion of `acc` every so often so it\n // doesn't grow without bound. Leave ~one chunk of headroom.\n if (this.readCursor > ratio * this.outputFrames) {\n this.acc = this.acc.slice(Math.floor(this.readCursor))\n this.readCursor -= Math.floor(this.readCursor)\n }\n\n return true\n }\n}\n\nregisterProcessor('mic-downsampler', MicDownsampler)\n"
|
|
72
|
+
var mic_downsampler_worklet_default = "// AudioWorklet \u2014 runs off the main thread in the audio rendering graph.\n//\n// Input: Float32 samples at the AudioContext's native sampleRate (typically\n// 48000 Hz on desktop, 44100 Hz on some iOS devices).\n// Output: 16 kHz mono Int16 PCM, shipped to the main thread via\n// `port.postMessage(ArrayBuffer, [ArrayBuffer])` (transferred, not copied).\n//\n// Why AudioWorklet instead of ScriptProcessorNode: ScriptProcessorNode is\n// deprecated + main-thread-bound, so any JS jank produces audible audio\n// glitches. AudioWorklet's `process()` runs on the audio rendering thread\n// at the graph's block cadence (128 frames by default) and backpressures\n// via returning `true` / `false`.\n//\n// This file is loaded as text (see tsup.config.ts loader) and registered\n// at runtime via `audioWorklet.addModule(blobUrl)`.\n\nclass MicDownsampler extends AudioWorkletProcessor {\n constructor() {\n super()\n // Target sample rate for STT. Matches Deepgram Nova-3 + the platform's\n // server-side SAMPLE_RATE constant in AgentCallHandler.\n this.targetRate = 16000\n // Accumulator for the downsample. We collect incoming samples and emit\n // an Int16 chunk when we've accumulated ~1024 target-rate samples\n // (~64 ms at 16 kHz) \u2014 matches the mobile SDK's chunk size so both\n // platforms have the same server-side framing.\n this.outputFrames = 1024\n this.acc = []\n // Running index used for fractional resampling.\n this.readCursor = 0\n }\n\n // `inputs[0][0]` = first channel of first input. 128 Float32 samples per\n // call at the context's sampleRate. Return true = keep processing.\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n const channel = input[0]\n if (!channel || channel.length === 0) return true\n\n const ctxRate = sampleRate // global inside AudioWorkletProcessor\n const ratio = ctxRate / this.targetRate\n\n // Simple linear-interp downsample. For 48000 \u2192 16000 that's 3:1, which\n // linear handles fine for voice. Anti-alias filtering would be\n // theoretically better but inaudible for speech.\n for (let i = 0; i < channel.length; i++) {\n this.acc.push(channel[i])\n }\n\n while (this.acc.length - this.readCursor >= ratio * this.outputFrames) {\n const out = new Int16Array(this.outputFrames)\n let readIdx = this.readCursor\n for (let i = 0; i < this.outputFrames; i++) {\n // Linear interp between floor(readIdx) and ceil(readIdx)\n const low = Math.floor(readIdx)\n const high = Math.min(low + 1, this.acc.length - 1)\n const frac = readIdx - low\n const sample = this.acc[low] * (1 - frac) + this.acc[high] * frac\n // Clip + convert to int16\n const clipped = Math.max(-1, Math.min(1, sample))\n out[i] = clipped < 0 ? clipped * 0x8000 : clipped * 0x7fff\n readIdx += ratio\n }\n // Transfer the ArrayBuffer (zero-copy) to the main thread.\n this.port.postMessage(out.buffer, [out.buffer])\n this.readCursor = readIdx\n }\n\n // Garbage-collect the consumed portion of `acc` every so often so it\n // doesn't grow without bound. Leave ~one chunk of headroom.\n if (this.readCursor > ratio * this.outputFrames) {\n this.acc = this.acc.slice(Math.floor(this.readCursor))\n this.readCursor -= Math.floor(this.readCursor)\n }\n\n return true\n }\n}\n\nregisterProcessor('mic-downsampler', MicDownsampler)\n";
|
|
71
73
|
|
|
72
74
|
// src/AudioCapture.ts
|
|
73
|
-
var VOLUME_INTERVAL_MS = 100
|
|
75
|
+
var VOLUME_INTERVAL_MS = 100;
|
|
74
76
|
var createAudioCapture = (options) => {
|
|
75
|
-
let audioContext = null
|
|
76
|
-
let mediaStream = null
|
|
77
|
-
let sourceNode = null
|
|
78
|
-
let workletNode = null
|
|
79
|
-
let analyser = null
|
|
80
|
-
let volumeTimer = null
|
|
81
|
-
let muted = false
|
|
82
|
-
let capturing = false
|
|
77
|
+
let audioContext = null;
|
|
78
|
+
let mediaStream = null;
|
|
79
|
+
let sourceNode = null;
|
|
80
|
+
let workletNode = null;
|
|
81
|
+
let analyser = null;
|
|
82
|
+
let volumeTimer = null;
|
|
83
|
+
let muted = false;
|
|
84
|
+
let capturing = false;
|
|
83
85
|
const computeRms = (buf) => {
|
|
84
|
-
let sum = 0
|
|
85
|
-
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]
|
|
86
|
-
const rms = Math.sqrt(sum / buf.length)
|
|
87
|
-
return Math.min(1, rms * 1.8)
|
|
88
|
-
}
|
|
86
|
+
let sum = 0;
|
|
87
|
+
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i];
|
|
88
|
+
const rms = Math.sqrt(sum / buf.length);
|
|
89
|
+
return Math.min(1, rms * 1.8);
|
|
90
|
+
};
|
|
89
91
|
const start = async () => {
|
|
90
|
-
if (capturing) return
|
|
92
|
+
if (capturing) return;
|
|
91
93
|
try {
|
|
92
94
|
mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
93
95
|
audio: {
|
|
@@ -98,748 +100,751 @@ var createAudioCapture = (options) => {
|
|
|
98
100
|
echoCancellation: true,
|
|
99
101
|
noiseSuppression: true,
|
|
100
102
|
autoGainControl: true,
|
|
101
|
-
channelCount: 1
|
|
102
|
-
}
|
|
103
|
-
})
|
|
104
|
-
audioContext = new AudioContext()
|
|
105
|
-
if (audioContext.state ===
|
|
106
|
-
const blob = new Blob([mic_downsampler_worklet_default], { type:
|
|
107
|
-
const url = URL.createObjectURL(blob)
|
|
103
|
+
channelCount: 1
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
audioContext = new AudioContext();
|
|
107
|
+
if (audioContext.state === "suspended") await audioContext.resume();
|
|
108
|
+
const blob = new Blob([mic_downsampler_worklet_default], { type: "application/javascript" });
|
|
109
|
+
const url = URL.createObjectURL(blob);
|
|
108
110
|
try {
|
|
109
|
-
await audioContext.audioWorklet.addModule(url)
|
|
111
|
+
await audioContext.audioWorklet.addModule(url);
|
|
110
112
|
} finally {
|
|
111
|
-
URL.revokeObjectURL(url)
|
|
113
|
+
URL.revokeObjectURL(url);
|
|
112
114
|
}
|
|
113
|
-
sourceNode = audioContext.createMediaStreamSource(mediaStream)
|
|
114
|
-
workletNode = new AudioWorkletNode(audioContext,
|
|
115
|
+
sourceNode = audioContext.createMediaStreamSource(mediaStream);
|
|
116
|
+
workletNode = new AudioWorkletNode(audioContext, "mic-downsampler");
|
|
115
117
|
workletNode.port.onmessage = (event) => {
|
|
116
|
-
if (muted) return
|
|
117
|
-
options.onChunk(event.data)
|
|
118
|
-
}
|
|
118
|
+
if (muted) return;
|
|
119
|
+
options.onChunk(event.data);
|
|
120
|
+
};
|
|
119
121
|
if (options.onVolume) {
|
|
120
|
-
analyser = audioContext.createAnalyser()
|
|
121
|
-
analyser.fftSize = 256
|
|
122
|
-
sourceNode.connect(analyser)
|
|
123
|
-
const buf = new Float32Array(analyser.fftSize)
|
|
122
|
+
analyser = audioContext.createAnalyser();
|
|
123
|
+
analyser.fftSize = 256;
|
|
124
|
+
sourceNode.connect(analyser);
|
|
125
|
+
const buf = new Float32Array(analyser.fftSize);
|
|
124
126
|
volumeTimer = setInterval(() => {
|
|
125
|
-
if (!analyser) return
|
|
126
|
-
analyser.getFloatTimeDomainData(buf)
|
|
127
|
-
options.onVolume?.(computeRms(buf))
|
|
128
|
-
}, VOLUME_INTERVAL_MS)
|
|
127
|
+
if (!analyser) return;
|
|
128
|
+
analyser.getFloatTimeDomainData(buf);
|
|
129
|
+
options.onVolume?.(computeRms(buf));
|
|
130
|
+
}, VOLUME_INTERVAL_MS);
|
|
129
131
|
}
|
|
130
|
-
sourceNode.connect(workletNode)
|
|
131
|
-
const sink = audioContext.createGain()
|
|
132
|
-
sink.gain.value = 0
|
|
133
|
-
workletNode.connect(sink).connect(audioContext.destination)
|
|
134
|
-
capturing = true
|
|
132
|
+
sourceNode.connect(workletNode);
|
|
133
|
+
const sink = audioContext.createGain();
|
|
134
|
+
sink.gain.value = 0;
|
|
135
|
+
workletNode.connect(sink).connect(audioContext.destination);
|
|
136
|
+
capturing = true;
|
|
135
137
|
} catch (err) {
|
|
136
|
-
const wrapped =
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
throw wrapped
|
|
138
|
+
const wrapped = err instanceof Error ? err : new Error(typeof err === "string" ? err : "capture failed");
|
|
139
|
+
options.onError?.(wrapped);
|
|
140
|
+
throw wrapped;
|
|
140
141
|
}
|
|
141
|
-
}
|
|
142
|
+
};
|
|
142
143
|
const stop = () => {
|
|
143
|
-
if (!capturing) return
|
|
144
|
-
capturing = false
|
|
144
|
+
if (!capturing) return;
|
|
145
|
+
capturing = false;
|
|
145
146
|
if (volumeTimer) {
|
|
146
|
-
clearInterval(volumeTimer)
|
|
147
|
-
volumeTimer = null
|
|
147
|
+
clearInterval(volumeTimer);
|
|
148
|
+
volumeTimer = null;
|
|
148
149
|
}
|
|
149
150
|
try {
|
|
150
|
-
workletNode?.disconnect()
|
|
151
|
-
analyser?.disconnect()
|
|
152
|
-
sourceNode?.disconnect()
|
|
153
|
-
} catch {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
151
|
+
workletNode?.disconnect();
|
|
152
|
+
analyser?.disconnect();
|
|
153
|
+
sourceNode?.disconnect();
|
|
154
|
+
} catch {
|
|
155
|
+
}
|
|
156
|
+
workletNode = null;
|
|
157
|
+
analyser = null;
|
|
158
|
+
sourceNode = null;
|
|
157
159
|
if (mediaStream) {
|
|
158
|
-
for (const track of mediaStream.getTracks()) track.stop()
|
|
159
|
-
mediaStream = null
|
|
160
|
+
for (const track of mediaStream.getTracks()) track.stop();
|
|
161
|
+
mediaStream = null;
|
|
160
162
|
}
|
|
161
|
-
if (audioContext && audioContext.state !==
|
|
162
|
-
void audioContext.close().catch(() => void 0)
|
|
163
|
+
if (audioContext && audioContext.state !== "closed") {
|
|
164
|
+
void audioContext.close().catch(() => void 0);
|
|
163
165
|
}
|
|
164
|
-
audioContext = null
|
|
165
|
-
}
|
|
166
|
+
audioContext = null;
|
|
167
|
+
};
|
|
166
168
|
return {
|
|
167
169
|
start,
|
|
168
170
|
stop,
|
|
169
171
|
mute: (v) => {
|
|
170
|
-
muted = v
|
|
172
|
+
muted = v;
|
|
171
173
|
},
|
|
172
|
-
isCapturing: () => capturing
|
|
173
|
-
}
|
|
174
|
-
}
|
|
174
|
+
isCapturing: () => capturing
|
|
175
|
+
};
|
|
176
|
+
};
|
|
175
177
|
|
|
176
178
|
// src/AudioPlayback.ts
|
|
177
|
-
var DEFAULT_SAMPLE_RATE = 16e3
|
|
178
|
-
var VOLUME_INTERVAL_MS2 = 100
|
|
179
|
+
var DEFAULT_SAMPLE_RATE = 16e3;
|
|
180
|
+
var VOLUME_INTERVAL_MS2 = 100;
|
|
179
181
|
var createAudioPlayback = (options = {}) => {
|
|
180
|
-
const sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE
|
|
181
|
-
let audioContext = null
|
|
182
|
-
let gainNode = null
|
|
183
|
-
let analyser = null
|
|
184
|
-
let volumeTimer = null
|
|
185
|
-
let nextStartTime = 0
|
|
186
|
-
let scheduledNodes = []
|
|
187
|
-
let speaking = false
|
|
182
|
+
const sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE;
|
|
183
|
+
let audioContext = null;
|
|
184
|
+
let gainNode = null;
|
|
185
|
+
let analyser = null;
|
|
186
|
+
let volumeTimer = null;
|
|
187
|
+
let nextStartTime = 0;
|
|
188
|
+
let scheduledNodes = [];
|
|
189
|
+
let speaking = false;
|
|
188
190
|
const ensureContext = async () => {
|
|
189
191
|
if (audioContext) {
|
|
190
|
-
if (audioContext.state ===
|
|
191
|
-
return
|
|
192
|
+
if (audioContext.state === "suspended") await audioContext.resume();
|
|
193
|
+
return;
|
|
192
194
|
}
|
|
193
|
-
audioContext = new AudioContext({ sampleRate })
|
|
194
|
-
gainNode = audioContext.createGain()
|
|
195
|
+
audioContext = new AudioContext({ sampleRate });
|
|
196
|
+
gainNode = audioContext.createGain();
|
|
195
197
|
if (options.onVolume) {
|
|
196
|
-
analyser = audioContext.createAnalyser()
|
|
197
|
-
analyser.fftSize = 256
|
|
198
|
-
gainNode.connect(analyser)
|
|
199
|
-
const buf = new Float32Array(analyser.fftSize)
|
|
198
|
+
analyser = audioContext.createAnalyser();
|
|
199
|
+
analyser.fftSize = 256;
|
|
200
|
+
gainNode.connect(analyser);
|
|
201
|
+
const buf = new Float32Array(analyser.fftSize);
|
|
200
202
|
volumeTimer = setInterval(() => {
|
|
201
|
-
if (!analyser) return
|
|
202
|
-
analyser.getFloatTimeDomainData(buf)
|
|
203
|
-
let sum = 0
|
|
204
|
-
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]
|
|
205
|
-
const rms = Math.sqrt(sum / buf.length)
|
|
206
|
-
options.onVolume?.(Math.min(1, rms * 1.8))
|
|
207
|
-
}, VOLUME_INTERVAL_MS2)
|
|
203
|
+
if (!analyser) return;
|
|
204
|
+
analyser.getFloatTimeDomainData(buf);
|
|
205
|
+
let sum = 0;
|
|
206
|
+
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i];
|
|
207
|
+
const rms = Math.sqrt(sum / buf.length);
|
|
208
|
+
options.onVolume?.(Math.min(1, rms * 1.8));
|
|
209
|
+
}, VOLUME_INTERVAL_MS2);
|
|
208
210
|
}
|
|
209
|
-
gainNode.connect(audioContext.destination)
|
|
210
|
-
nextStartTime = audioContext.currentTime
|
|
211
|
-
}
|
|
211
|
+
gainNode.connect(audioContext.destination);
|
|
212
|
+
nextStartTime = audioContext.currentTime;
|
|
213
|
+
};
|
|
212
214
|
const setSpeaking = (v) => {
|
|
213
|
-
if (v === speaking) return
|
|
214
|
-
speaking = v
|
|
215
|
-
options.onSpeakingChange?.(v)
|
|
216
|
-
}
|
|
215
|
+
if (v === speaking) return;
|
|
216
|
+
speaking = v;
|
|
217
|
+
options.onSpeakingChange?.(v);
|
|
218
|
+
};
|
|
217
219
|
const pruneFinished = () => {
|
|
218
|
-
const now = audioContext?.currentTime ?? 0
|
|
220
|
+
const now = audioContext?.currentTime ?? 0;
|
|
219
221
|
scheduledNodes = scheduledNodes.filter((n) => {
|
|
220
|
-
const node = n
|
|
221
|
-
return (node._endsAt ?? 0) > now
|
|
222
|
-
})
|
|
223
|
-
if (scheduledNodes.length === 0) setSpeaking(false)
|
|
224
|
-
}
|
|
222
|
+
const node = n;
|
|
223
|
+
return (node._endsAt ?? 0) > now;
|
|
224
|
+
});
|
|
225
|
+
if (scheduledNodes.length === 0) setSpeaking(false);
|
|
226
|
+
};
|
|
225
227
|
const enqueue = (pcm) => {
|
|
226
228
|
if (!audioContext) {
|
|
227
|
-
void ensureContext().then(() => enqueue(pcm))
|
|
228
|
-
return
|
|
229
|
+
void ensureContext().then(() => enqueue(pcm));
|
|
230
|
+
return;
|
|
229
231
|
}
|
|
230
|
-
if (!audioContext || !gainNode) return
|
|
231
|
-
const int16 = new Int16Array(pcm)
|
|
232
|
-
if (int16.length === 0) return
|
|
233
|
-
const audioBuffer = audioContext.createBuffer(1, int16.length, sampleRate)
|
|
234
|
-
const float32 = audioBuffer.getChannelData(0)
|
|
232
|
+
if (!audioContext || !gainNode) return;
|
|
233
|
+
const int16 = new Int16Array(pcm);
|
|
234
|
+
if (int16.length === 0) return;
|
|
235
|
+
const audioBuffer = audioContext.createBuffer(1, int16.length, sampleRate);
|
|
236
|
+
const float32 = audioBuffer.getChannelData(0);
|
|
235
237
|
for (let i = 0; i < int16.length; i++) {
|
|
236
|
-
float32[i] = int16[i] / 32768
|
|
238
|
+
float32[i] = int16[i] / 32768;
|
|
237
239
|
}
|
|
238
|
-
const node = audioContext.createBufferSource()
|
|
239
|
-
node.buffer = audioBuffer
|
|
240
|
-
node.connect(gainNode)
|
|
241
|
-
const now = audioContext.currentTime
|
|
242
|
-
const startAt = Math.max(now, nextStartTime)
|
|
243
|
-
node.start(startAt)
|
|
244
|
-
const duration = int16.length / sampleRate
|
|
245
|
-
node._endsAt = startAt + duration
|
|
246
|
-
nextStartTime = startAt + duration
|
|
247
|
-
scheduledNodes.push(node)
|
|
248
|
-
setSpeaking(true)
|
|
249
|
-
node.onended = () => pruneFinished()
|
|
250
|
-
}
|
|
240
|
+
const node = audioContext.createBufferSource();
|
|
241
|
+
node.buffer = audioBuffer;
|
|
242
|
+
node.connect(gainNode);
|
|
243
|
+
const now = audioContext.currentTime;
|
|
244
|
+
const startAt = Math.max(now, nextStartTime);
|
|
245
|
+
node.start(startAt);
|
|
246
|
+
const duration = int16.length / sampleRate;
|
|
247
|
+
node._endsAt = startAt + duration;
|
|
248
|
+
nextStartTime = startAt + duration;
|
|
249
|
+
scheduledNodes.push(node);
|
|
250
|
+
setSpeaking(true);
|
|
251
|
+
node.onended = () => pruneFinished();
|
|
252
|
+
};
|
|
251
253
|
const flush = () => {
|
|
252
|
-
if (!audioContext || !gainNode) return
|
|
254
|
+
if (!audioContext || !gainNode) return;
|
|
253
255
|
for (const node of scheduledNodes) {
|
|
254
256
|
try {
|
|
255
|
-
node.stop()
|
|
256
|
-
} catch {
|
|
257
|
+
node.stop();
|
|
258
|
+
} catch {
|
|
259
|
+
}
|
|
257
260
|
}
|
|
258
|
-
scheduledNodes = []
|
|
259
|
-
gainNode.disconnect()
|
|
260
|
-
gainNode = audioContext.createGain()
|
|
261
|
+
scheduledNodes = [];
|
|
262
|
+
gainNode.disconnect();
|
|
263
|
+
gainNode = audioContext.createGain();
|
|
261
264
|
if (analyser) {
|
|
262
|
-
analyser.disconnect()
|
|
263
|
-
gainNode.connect(analyser)
|
|
265
|
+
analyser.disconnect();
|
|
266
|
+
gainNode.connect(analyser);
|
|
264
267
|
}
|
|
265
|
-
gainNode.connect(audioContext.destination)
|
|
266
|
-
nextStartTime = audioContext.currentTime
|
|
267
|
-
setSpeaking(false)
|
|
268
|
-
}
|
|
268
|
+
gainNode.connect(audioContext.destination);
|
|
269
|
+
nextStartTime = audioContext.currentTime;
|
|
270
|
+
setSpeaking(false);
|
|
271
|
+
};
|
|
269
272
|
const close = () => {
|
|
270
|
-
flush()
|
|
273
|
+
flush();
|
|
271
274
|
if (volumeTimer) {
|
|
272
|
-
clearInterval(volumeTimer)
|
|
273
|
-
volumeTimer = null
|
|
275
|
+
clearInterval(volumeTimer);
|
|
276
|
+
volumeTimer = null;
|
|
274
277
|
}
|
|
275
|
-
if (audioContext && audioContext.state !==
|
|
276
|
-
void audioContext.close().catch(() => void 0)
|
|
278
|
+
if (audioContext && audioContext.state !== "closed") {
|
|
279
|
+
void audioContext.close().catch(() => void 0);
|
|
277
280
|
}
|
|
278
|
-
audioContext = null
|
|
279
|
-
gainNode = null
|
|
280
|
-
analyser = null
|
|
281
|
-
}
|
|
281
|
+
audioContext = null;
|
|
282
|
+
gainNode = null;
|
|
283
|
+
analyser = null;
|
|
284
|
+
};
|
|
282
285
|
const resume = async () => {
|
|
283
|
-
await ensureContext()
|
|
284
|
-
}
|
|
285
|
-
return { enqueue, flush, close, resume }
|
|
286
|
-
}
|
|
286
|
+
await ensureContext();
|
|
287
|
+
};
|
|
288
|
+
return { enqueue, flush, close, resume };
|
|
289
|
+
};
|
|
287
290
|
|
|
288
291
|
// src/ReconnectingWebSocket.ts
|
|
289
|
-
var READYSTATE_OPEN = 1
|
|
290
|
-
var READYSTATE_CLOSED = 3
|
|
292
|
+
var READYSTATE_OPEN = 1;
|
|
293
|
+
var READYSTATE_CLOSED = 3;
|
|
291
294
|
var createReconnectingWebSocket = (options, onEvent) => {
|
|
292
|
-
const maxRetries = options.maxRetries ?? 3
|
|
293
|
-
const initialBackoff = options.initialBackoffMs ?? 500
|
|
294
|
-
const maxBackoff = options.maxBackoffMs ?? 8e3
|
|
295
|
-
let ws = null
|
|
296
|
-
let intentionalClose = false
|
|
297
|
-
let retries = 0
|
|
298
|
-
let backoff = initialBackoff
|
|
299
|
-
let reconnectTimer = null
|
|
295
|
+
const maxRetries = options.maxRetries ?? 3;
|
|
296
|
+
const initialBackoff = options.initialBackoffMs ?? 500;
|
|
297
|
+
const maxBackoff = options.maxBackoffMs ?? 8e3;
|
|
298
|
+
let ws = null;
|
|
299
|
+
let intentionalClose = false;
|
|
300
|
+
let retries = 0;
|
|
301
|
+
let backoff = initialBackoff;
|
|
302
|
+
let reconnectTimer = null;
|
|
300
303
|
const openOnce = () => {
|
|
301
|
-
ws = options.wsFactory(options.url)
|
|
302
|
-
ws.binaryType =
|
|
304
|
+
ws = options.wsFactory(options.url);
|
|
305
|
+
ws.binaryType = "arraybuffer";
|
|
303
306
|
ws.onopen = () => {
|
|
304
|
-
if (retries === 0) onEvent({ type:
|
|
305
|
-
else onEvent({ type:
|
|
306
|
-
retries = 0
|
|
307
|
-
backoff = initialBackoff
|
|
308
|
-
}
|
|
307
|
+
if (retries === 0) onEvent({ type: "open" });
|
|
308
|
+
else onEvent({ type: "reconnected" });
|
|
309
|
+
retries = 0;
|
|
310
|
+
backoff = initialBackoff;
|
|
311
|
+
};
|
|
309
312
|
ws.onmessage = (ev) => {
|
|
310
|
-
onEvent({ type:
|
|
311
|
-
}
|
|
313
|
+
onEvent({ type: "message", data: ev.data });
|
|
314
|
+
};
|
|
312
315
|
ws.onerror = () => {
|
|
313
|
-
onEvent({ type:
|
|
314
|
-
}
|
|
316
|
+
onEvent({ type: "error", error: new Error("WebSocket error") });
|
|
317
|
+
};
|
|
315
318
|
ws.onclose = (ev) => {
|
|
316
|
-
ws = null
|
|
317
|
-
const shouldRetry = !intentionalClose && retries < maxRetries
|
|
319
|
+
ws = null;
|
|
320
|
+
const shouldRetry = !intentionalClose && retries < maxRetries;
|
|
318
321
|
if (!shouldRetry) {
|
|
319
322
|
onEvent({
|
|
320
|
-
type:
|
|
323
|
+
type: "close",
|
|
321
324
|
code: ev.code,
|
|
322
325
|
reason: ev.reason,
|
|
323
|
-
permanent: true
|
|
324
|
-
})
|
|
325
|
-
return
|
|
326
|
+
permanent: true
|
|
327
|
+
});
|
|
328
|
+
return;
|
|
326
329
|
}
|
|
327
330
|
onEvent({
|
|
328
|
-
type:
|
|
331
|
+
type: "close",
|
|
329
332
|
code: ev.code,
|
|
330
333
|
reason: ev.reason,
|
|
331
|
-
permanent: false
|
|
332
|
-
})
|
|
333
|
-
retries
|
|
334
|
-
const delay = Math.min(backoff, maxBackoff)
|
|
335
|
-
backoff = Math.min(backoff * 2, maxBackoff)
|
|
336
|
-
reconnectTimer = setTimeout(openOnce, delay)
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
openOnce()
|
|
334
|
+
permanent: false
|
|
335
|
+
});
|
|
336
|
+
retries++;
|
|
337
|
+
const delay = Math.min(backoff, maxBackoff);
|
|
338
|
+
backoff = Math.min(backoff * 2, maxBackoff);
|
|
339
|
+
reconnectTimer = setTimeout(openOnce, delay);
|
|
340
|
+
};
|
|
341
|
+
};
|
|
342
|
+
openOnce();
|
|
340
343
|
return {
|
|
341
344
|
send: (data) => {
|
|
342
|
-
if (ws && ws.readyState === READYSTATE_OPEN) ws.send(data)
|
|
345
|
+
if (ws && ws.readyState === READYSTATE_OPEN) ws.send(data);
|
|
343
346
|
},
|
|
344
|
-
close: (code = 1e3, reason =
|
|
345
|
-
intentionalClose = true
|
|
347
|
+
close: (code = 1e3, reason = "client-requested") => {
|
|
348
|
+
intentionalClose = true;
|
|
346
349
|
if (reconnectTimer) {
|
|
347
|
-
clearTimeout(reconnectTimer)
|
|
348
|
-
reconnectTimer = null
|
|
350
|
+
clearTimeout(reconnectTimer);
|
|
351
|
+
reconnectTimer = null;
|
|
349
352
|
}
|
|
350
353
|
try {
|
|
351
|
-
ws?.close(code, reason)
|
|
352
|
-
} catch {
|
|
354
|
+
ws?.close(code, reason);
|
|
355
|
+
} catch {
|
|
356
|
+
}
|
|
353
357
|
},
|
|
354
|
-
readyState: () => ws?.readyState ?? READYSTATE_CLOSED
|
|
355
|
-
}
|
|
356
|
-
}
|
|
358
|
+
readyState: () => ws?.readyState ?? READYSTATE_CLOSED
|
|
359
|
+
};
|
|
360
|
+
};
|
|
357
361
|
|
|
358
362
|
// src/protocol.ts
|
|
359
363
|
var createProtocolState = () => ({
|
|
360
|
-
state:
|
|
364
|
+
state: "idle",
|
|
361
365
|
transcript: [],
|
|
362
366
|
agentBubbleId: null,
|
|
363
367
|
idCounter: 0,
|
|
364
|
-
endReason: null
|
|
365
|
-
})
|
|
368
|
+
endReason: null
|
|
369
|
+
});
|
|
366
370
|
var mapEndReason = (raw) => {
|
|
367
|
-
if (raw ===
|
|
368
|
-
if (raw ===
|
|
369
|
-
if (raw ===
|
|
370
|
-
return
|
|
371
|
-
}
|
|
371
|
+
if (raw === "agent_ended") return "agent_ended";
|
|
372
|
+
if (raw === "caller_hung_up") return "user_hangup";
|
|
373
|
+
if (raw === "silence_timeout" || raw === "max_duration") return "timeout";
|
|
374
|
+
return "error";
|
|
375
|
+
};
|
|
372
376
|
function handleServerMessage(raw, state, cb) {
|
|
373
|
-
let msg
|
|
377
|
+
let msg;
|
|
374
378
|
try {
|
|
375
|
-
msg = JSON.parse(raw)
|
|
379
|
+
msg = JSON.parse(raw);
|
|
376
380
|
} catch {
|
|
377
|
-
return
|
|
381
|
+
return;
|
|
378
382
|
}
|
|
379
383
|
switch (msg.type) {
|
|
380
|
-
case
|
|
381
|
-
cb.onConnected()
|
|
382
|
-
setState(state,
|
|
383
|
-
return
|
|
384
|
-
case
|
|
385
|
-
const text = msg.text ??
|
|
386
|
-
if (!text) return
|
|
387
|
-
const isFinal = !!msg.isFinal
|
|
388
|
-
if (!isFinal) setState(state,
|
|
389
|
-
upsertUserPartial(state, text, isFinal)
|
|
390
|
-
cb.onTranscript(state.transcript)
|
|
391
|
-
return
|
|
384
|
+
case "connected":
|
|
385
|
+
cb.onConnected();
|
|
386
|
+
setState(state, "listening", cb);
|
|
387
|
+
return;
|
|
388
|
+
case "transcript": {
|
|
389
|
+
const text = msg.text ?? "";
|
|
390
|
+
if (!text) return;
|
|
391
|
+
const isFinal = !!msg.isFinal;
|
|
392
|
+
if (!isFinal) setState(state, "user_speaking", cb);
|
|
393
|
+
upsertUserPartial(state, text, isFinal);
|
|
394
|
+
cb.onTranscript(state.transcript);
|
|
395
|
+
return;
|
|
392
396
|
}
|
|
393
|
-
case
|
|
394
|
-
const id = `m${state.idCounter++}
|
|
395
|
-
state.agentBubbleId = id
|
|
396
|
-
state.transcript = [...state.transcript, { id, role:
|
|
397
|
-
cb.onTranscript(state.transcript)
|
|
398
|
-
const seq = typeof msg.seq ===
|
|
399
|
-
cb.onAgentTurnStart(seq)
|
|
400
|
-
setState(state,
|
|
401
|
-
return
|
|
397
|
+
case "agent_turn_start": {
|
|
398
|
+
const id = `m${state.idCounter++}`;
|
|
399
|
+
state.agentBubbleId = id;
|
|
400
|
+
state.transcript = [...state.transcript, { id, role: "agent", text: "" }];
|
|
401
|
+
cb.onTranscript(state.transcript);
|
|
402
|
+
const seq = typeof msg.seq === "number" ? msg.seq : void 0;
|
|
403
|
+
cb.onAgentTurnStart(seq);
|
|
404
|
+
setState(state, "agent_speaking", cb);
|
|
405
|
+
return;
|
|
402
406
|
}
|
|
403
|
-
case
|
|
404
|
-
const delta = msg.text ??
|
|
405
|
-
if (!delta || !state.agentBubbleId) return
|
|
406
|
-
const id = state.agentBubbleId
|
|
407
|
-
state.transcript = state.transcript.map(
|
|
408
|
-
e.id === id && e.role ===
|
|
409
|
-
)
|
|
410
|
-
cb.onTranscript(state.transcript)
|
|
411
|
-
return
|
|
407
|
+
case "agent_text": {
|
|
408
|
+
const delta = msg.text ?? "";
|
|
409
|
+
if (!delta || !state.agentBubbleId) return;
|
|
410
|
+
const id = state.agentBubbleId;
|
|
411
|
+
state.transcript = state.transcript.map(
|
|
412
|
+
(e) => e.id === id && e.role === "agent" ? { ...e, text: e.text + delta } : e
|
|
413
|
+
);
|
|
414
|
+
cb.onTranscript(state.transcript);
|
|
415
|
+
return;
|
|
412
416
|
}
|
|
413
|
-
case
|
|
414
|
-
state.agentBubbleId = null
|
|
415
|
-
const seq = typeof msg.seq ===
|
|
416
|
-
cb.onAgentTurnEnd(seq)
|
|
417
|
-
setState(state,
|
|
418
|
-
return
|
|
417
|
+
case "agent_turn_end": {
|
|
418
|
+
state.agentBubbleId = null;
|
|
419
|
+
const seq = typeof msg.seq === "number" ? msg.seq : void 0;
|
|
420
|
+
cb.onAgentTurnEnd(seq);
|
|
421
|
+
setState(state, "listening", cb);
|
|
422
|
+
return;
|
|
419
423
|
}
|
|
420
|
-
case
|
|
421
|
-
cb.onInterrupt()
|
|
422
|
-
return
|
|
423
|
-
case
|
|
424
|
-
const committed = (msg.committedText ??
|
|
424
|
+
case "interrupt":
|
|
425
|
+
cb.onInterrupt();
|
|
426
|
+
return;
|
|
427
|
+
case "agent_turn_abort": {
|
|
428
|
+
const committed = (msg.committedText ?? "").trim();
|
|
425
429
|
if (state.agentBubbleId) {
|
|
426
|
-
const id = state.agentBubbleId
|
|
430
|
+
const id = state.agentBubbleId;
|
|
427
431
|
if (committed) {
|
|
428
|
-
state.transcript = state.transcript.map(
|
|
429
|
-
e.id === id && e.role ===
|
|
430
|
-
)
|
|
432
|
+
state.transcript = state.transcript.map(
|
|
433
|
+
(e) => e.id === id && e.role === "agent" ? { ...e, text: committed, interrupted: true } : e
|
|
434
|
+
);
|
|
431
435
|
} else {
|
|
432
|
-
state.transcript = state.transcript.filter((e) => e.id !== id)
|
|
436
|
+
state.transcript = state.transcript.filter((e) => e.id !== id);
|
|
433
437
|
}
|
|
434
|
-
cb.onTranscript(state.transcript)
|
|
438
|
+
cb.onTranscript(state.transcript);
|
|
435
439
|
}
|
|
436
|
-
state.agentBubbleId = null
|
|
437
|
-
return
|
|
440
|
+
state.agentBubbleId = null;
|
|
441
|
+
return;
|
|
438
442
|
}
|
|
439
|
-
case
|
|
443
|
+
case "tool_call":
|
|
440
444
|
state.transcript = [
|
|
441
445
|
...state.transcript,
|
|
442
446
|
{
|
|
443
447
|
id: `m${state.idCounter++}`,
|
|
444
|
-
role:
|
|
445
|
-
text: `\u2192 ${String(msg.tool ??
|
|
446
|
-
}
|
|
447
|
-
]
|
|
448
|
-
cb.onTranscript(state.transcript)
|
|
449
|
-
return
|
|
450
|
-
case
|
|
448
|
+
role: "tool",
|
|
449
|
+
text: `\u2192 ${String(msg.tool ?? "?")}(${msg.args ? JSON.stringify(msg.args) : ""})`
|
|
450
|
+
}
|
|
451
|
+
];
|
|
452
|
+
cb.onTranscript(state.transcript);
|
|
453
|
+
return;
|
|
454
|
+
case "tool_result":
|
|
451
455
|
state.transcript = [
|
|
452
456
|
...state.transcript,
|
|
453
457
|
{
|
|
454
458
|
id: `m${state.idCounter++}`,
|
|
455
|
-
role:
|
|
456
|
-
text: `${msg.ok ?
|
|
457
|
-
}
|
|
458
|
-
]
|
|
459
|
-
cb.onTranscript(state.transcript)
|
|
460
|
-
return
|
|
461
|
-
case
|
|
462
|
-
const toolCallId = String(msg.toolCallId ??
|
|
463
|
-
const name = String(msg.name ??
|
|
464
|
-
const args = msg.args ?? {}
|
|
465
|
-
if (!toolCallId || !name) return
|
|
466
|
-
cb.onClientToolCall({ toolCallId, name, args })
|
|
467
|
-
return
|
|
459
|
+
role: "tool",
|
|
460
|
+
text: `${msg.ok ? "\u2713" : "\u2717"} ${String(msg.tool ?? "?")}`
|
|
461
|
+
}
|
|
462
|
+
];
|
|
463
|
+
cb.onTranscript(state.transcript);
|
|
464
|
+
return;
|
|
465
|
+
case "client_tool_call": {
|
|
466
|
+
const toolCallId = String(msg.toolCallId ?? "");
|
|
467
|
+
const name = String(msg.name ?? "");
|
|
468
|
+
const args = msg.args ?? {};
|
|
469
|
+
if (!toolCallId || !name) return;
|
|
470
|
+
cb.onClientToolCall({ toolCallId, name, args });
|
|
471
|
+
return;
|
|
468
472
|
}
|
|
469
|
-
case
|
|
470
|
-
const reasonRaw = String(msg.reason ??
|
|
471
|
-
const reason = mapEndReason(reasonRaw)
|
|
472
|
-
state.endReason = reason
|
|
473
|
+
case "call_end": {
|
|
474
|
+
const reasonRaw = String(msg.reason ?? "");
|
|
475
|
+
const reason = mapEndReason(reasonRaw);
|
|
476
|
+
state.endReason = reason;
|
|
473
477
|
state.transcript = [
|
|
474
478
|
...state.transcript,
|
|
475
479
|
{
|
|
476
480
|
id: `m${state.idCounter++}`,
|
|
477
|
-
role:
|
|
478
|
-
text: `call ended${reasonRaw ? ` (${reasonRaw})` :
|
|
479
|
-
}
|
|
480
|
-
]
|
|
481
|
-
cb.onTranscript(state.transcript)
|
|
482
|
-
cb.onCallEnd(reason)
|
|
483
|
-
return
|
|
481
|
+
role: "system",
|
|
482
|
+
text: `call ended${reasonRaw ? ` (${reasonRaw})` : ""}`
|
|
483
|
+
}
|
|
484
|
+
];
|
|
485
|
+
cb.onTranscript(state.transcript);
|
|
486
|
+
cb.onCallEnd(reason);
|
|
487
|
+
return;
|
|
484
488
|
}
|
|
485
|
-
case
|
|
486
|
-
const code = msg.code ??
|
|
487
|
-
const message = msg.message ??
|
|
488
|
-
cb.onError({ code, message })
|
|
489
|
-
return
|
|
489
|
+
case "error": {
|
|
490
|
+
const code = msg.code ?? "server_error";
|
|
491
|
+
const message = msg.message ?? "server error";
|
|
492
|
+
cb.onError({ code, message });
|
|
493
|
+
return;
|
|
490
494
|
}
|
|
491
495
|
}
|
|
492
496
|
}
|
|
493
497
|
var setState = (state, next, cb) => {
|
|
494
|
-
if (state.state === next) return
|
|
495
|
-
cb.onState(next)
|
|
496
|
-
}
|
|
498
|
+
if (state.state === next) return;
|
|
499
|
+
cb.onState(next);
|
|
500
|
+
};
|
|
497
501
|
var upsertUserPartial = (state, text, isFinal) => {
|
|
498
|
-
let idx = -1
|
|
502
|
+
let idx = -1;
|
|
499
503
|
for (let i = state.transcript.length - 1; i >= 0; i--) {
|
|
500
|
-
const e = state.transcript[i]
|
|
501
|
-
if (e.role ===
|
|
502
|
-
idx = i
|
|
503
|
-
break
|
|
504
|
+
const e = state.transcript[i];
|
|
505
|
+
if (e.role === "user" && e.committed === false) {
|
|
506
|
+
idx = i;
|
|
507
|
+
break;
|
|
504
508
|
}
|
|
505
509
|
}
|
|
506
510
|
if (idx === -1) {
|
|
507
511
|
state.transcript = [
|
|
508
512
|
...state.transcript,
|
|
509
|
-
{ id: `m${state.idCounter++}`, role:
|
|
510
|
-
]
|
|
511
|
-
return
|
|
512
|
-
}
|
|
513
|
-
const target = state.transcript[idx]
|
|
514
|
-
const next = [...state.transcript]
|
|
515
|
-
next[idx] = { ...target, text, committed: isFinal }
|
|
516
|
-
state.transcript = next
|
|
517
|
-
}
|
|
513
|
+
{ id: `m${state.idCounter++}`, role: "user", text, committed: isFinal }
|
|
514
|
+
];
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
const target = state.transcript[idx];
|
|
518
|
+
const next = [...state.transcript];
|
|
519
|
+
next[idx] = { ...target, text, committed: isFinal };
|
|
520
|
+
state.transcript = next;
|
|
521
|
+
};
|
|
518
522
|
function buildWsUrl(args) {
|
|
519
|
-
const base = new URL(args.apiBase)
|
|
520
|
-
const proto = base.protocol ===
|
|
521
|
-
const bargeQS = args.bargeIn === false ?
|
|
522
|
-
return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}
|
|
523
|
+
const base = new URL(args.apiBase);
|
|
524
|
+
const proto = base.protocol === "https:" ? "wss:" : "ws:";
|
|
525
|
+
const bargeQS = args.bargeIn === false ? "&barge=off" : "";
|
|
526
|
+
return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}`;
|
|
523
527
|
}
|
|
524
528
|
|
|
525
529
|
// src/clientTools.ts
|
|
526
|
-
var NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]
|
|
527
|
-
var MAX_TOOLS = 64
|
|
528
|
-
var MAX_USAGE = 500
|
|
529
|
-
var MAX_TIMEOUT_MS = 3e4
|
|
530
|
+
var NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
531
|
+
var MAX_TOOLS = 64;
|
|
532
|
+
var MAX_USAGE = 500;
|
|
533
|
+
var MAX_TIMEOUT_MS = 3e4;
|
|
530
534
|
var validateClientToolMap = (tools) => {
|
|
531
|
-
if (tools === void 0) return
|
|
532
|
-
if (typeof tools !==
|
|
533
|
-
throw new Error(
|
|
535
|
+
if (tools === void 0) return;
|
|
536
|
+
if (typeof tools !== "object" || tools === null || Array.isArray(tools)) {
|
|
537
|
+
throw new Error("clientTools must be an object keyed by tool name");
|
|
534
538
|
}
|
|
535
|
-
const entries = Object.entries(tools)
|
|
539
|
+
const entries = Object.entries(tools);
|
|
536
540
|
if (entries.length > MAX_TOOLS) {
|
|
537
|
-
throw new Error(`clientTools may declare at most 64 tools (got ${entries.length})`)
|
|
541
|
+
throw new Error(`clientTools may declare at most 64 tools (got ${entries.length})`);
|
|
538
542
|
}
|
|
539
543
|
for (const [name, def] of entries) {
|
|
540
544
|
if (!NAME_RE.test(name)) {
|
|
541
545
|
throw new Error(
|
|
542
|
-
`clientTools["${name}"]: name must be a valid identifier (^[a-zA-Z_][a-zA-Z0-9_]*$)
|
|
543
|
-
)
|
|
546
|
+
`clientTools["${name}"]: name must be a valid identifier (^[a-zA-Z_][a-zA-Z0-9_]*$)`
|
|
547
|
+
);
|
|
544
548
|
}
|
|
545
|
-
if (!def || typeof def !==
|
|
546
|
-
throw new Error(`clientTools["${name}"]: must be an object`)
|
|
549
|
+
if (!def || typeof def !== "object") {
|
|
550
|
+
throw new Error(`clientTools["${name}"]: must be an object`);
|
|
547
551
|
}
|
|
548
|
-
if (typeof def.description !==
|
|
549
|
-
throw new Error(`clientTools["${name}"]: must have a description`)
|
|
552
|
+
if (typeof def.description !== "string" || def.description.length === 0) {
|
|
553
|
+
throw new Error(`clientTools["${name}"]: must have a description`);
|
|
550
554
|
}
|
|
551
|
-
if (typeof def.handler !==
|
|
552
|
-
throw new Error(`clientTools["${name}"]: must have a handler function`)
|
|
555
|
+
if (typeof def.handler !== "function") {
|
|
556
|
+
throw new Error(`clientTools["${name}"]: must have a handler function`);
|
|
553
557
|
}
|
|
554
558
|
if (def.usage !== void 0 && def.usage.length > MAX_USAGE) {
|
|
555
|
-
throw new Error(`clientTools["${name}"]: usage must be \u2264500 chars`)
|
|
559
|
+
throw new Error(`clientTools["${name}"]: usage must be \u2264500 chars`);
|
|
556
560
|
}
|
|
557
|
-
if (
|
|
558
|
-
|
|
559
|
-
(!Number.isFinite(def.timeoutMs) || def.timeoutMs <= 0 || def.timeoutMs > MAX_TIMEOUT_MS)
|
|
560
|
-
) {
|
|
561
|
-
throw new Error(`clientTools["${name}"]: timeoutMs must be in (0, 30000]`)
|
|
561
|
+
if (def.timeoutMs !== void 0 && (!Number.isFinite(def.timeoutMs) || def.timeoutMs <= 0 || def.timeoutMs > MAX_TIMEOUT_MS)) {
|
|
562
|
+
throw new Error(`clientTools["${name}"]: timeoutMs must be in (0, 30000]`);
|
|
562
563
|
}
|
|
563
564
|
}
|
|
564
|
-
}
|
|
565
|
+
};
|
|
565
566
|
var buildRegisterFrame = (tools) => ({
|
|
566
|
-
type:
|
|
567
|
+
type: "client_tools_register",
|
|
567
568
|
tools: Object.entries(tools).map(([name, def]) => ({
|
|
568
569
|
name,
|
|
569
570
|
description: def.description,
|
|
570
571
|
parameters: def.parameters,
|
|
571
|
-
...
|
|
572
|
-
...
|
|
573
|
-
}))
|
|
574
|
-
})
|
|
572
|
+
...def.usage !== void 0 ? { usage: def.usage } : {},
|
|
573
|
+
...def.timeoutMs !== void 0 ? { timeoutMs: def.timeoutMs } : {}
|
|
574
|
+
}))
|
|
575
|
+
});
|
|
575
576
|
var dispatchClientToolCall = (send, tools, frame) => {
|
|
576
577
|
const safeSend = (payload) => {
|
|
577
578
|
try {
|
|
578
|
-
send(payload)
|
|
579
|
-
} catch {
|
|
580
|
-
|
|
581
|
-
|
|
579
|
+
send(payload);
|
|
580
|
+
} catch {
|
|
581
|
+
}
|
|
582
|
+
};
|
|
583
|
+
const tool = tools[frame.name];
|
|
582
584
|
if (!tool) {
|
|
583
585
|
safeSend({
|
|
584
|
-
type:
|
|
586
|
+
type: "client_tool_result",
|
|
585
587
|
toolCallId: frame.toolCallId,
|
|
586
|
-
error: `No handler for ${frame.name}
|
|
587
|
-
})
|
|
588
|
-
return
|
|
588
|
+
error: `No handler for ${frame.name}`
|
|
589
|
+
});
|
|
590
|
+
return;
|
|
589
591
|
}
|
|
590
592
|
void (async () => {
|
|
591
593
|
try {
|
|
592
|
-
const out = await tool.handler(frame.args)
|
|
594
|
+
const out = await tool.handler(frame.args);
|
|
593
595
|
safeSend({
|
|
594
|
-
type:
|
|
596
|
+
type: "client_tool_result",
|
|
595
597
|
toolCallId: frame.toolCallId,
|
|
596
|
-
result: typeof out ===
|
|
597
|
-
})
|
|
598
|
+
result: typeof out === "string" ? out : JSON.stringify(out)
|
|
599
|
+
});
|
|
598
600
|
} catch (err) {
|
|
599
601
|
safeSend({
|
|
600
|
-
type:
|
|
602
|
+
type: "client_tool_result",
|
|
601
603
|
toolCallId: frame.toolCallId,
|
|
602
|
-
error: err instanceof Error ? err.message : String(err)
|
|
603
|
-
})
|
|
604
|
+
error: err instanceof Error ? err.message : String(err)
|
|
605
|
+
});
|
|
604
606
|
}
|
|
605
|
-
})()
|
|
606
|
-
}
|
|
607
|
+
})();
|
|
608
|
+
};
|
|
607
609
|
|
|
608
610
|
// src/ClientMarksBuffer.ts
|
|
609
611
|
var createClientMarksBuffer = (args) => {
|
|
610
|
-
const now = args.now ?? (() => performance.now())
|
|
611
|
-
let pendingFirstOutboundAt = null
|
|
612
|
-
const inFlight = /* @__PURE__ */ new Map()
|
|
612
|
+
const now = args.now ?? (() => performance.now());
|
|
613
|
+
let pendingFirstOutboundAt = null;
|
|
614
|
+
const inFlight = /* @__PURE__ */ new Map();
|
|
613
615
|
const tryEmit = (seq) => {
|
|
614
|
-
const slot = inFlight.get(seq)
|
|
615
|
-
if (!slot) return
|
|
616
|
-
if (!slot.ended) return
|
|
617
|
-
const marks = {}
|
|
616
|
+
const slot = inFlight.get(seq);
|
|
617
|
+
if (!slot) return;
|
|
618
|
+
if (!slot.ended) return;
|
|
619
|
+
const marks = {};
|
|
618
620
|
if (slot.firstOutboundAt !== null && slot.firstAudibleAt !== null) {
|
|
619
|
-
marks.client_mic_to_first_audible_ms = slot.firstAudibleAt - slot.firstOutboundAt
|
|
621
|
+
marks.client_mic_to_first_audible_ms = slot.firstAudibleAt - slot.firstOutboundAt;
|
|
620
622
|
}
|
|
621
623
|
args.send({
|
|
622
|
-
type:
|
|
624
|
+
type: "client_marks",
|
|
623
625
|
seq,
|
|
624
626
|
marks,
|
|
625
|
-
clientNow: Date.now()
|
|
626
|
-
})
|
|
627
|
-
inFlight.delete(seq)
|
|
628
|
-
}
|
|
627
|
+
clientNow: Date.now()
|
|
628
|
+
});
|
|
629
|
+
inFlight.delete(seq);
|
|
630
|
+
};
|
|
629
631
|
const markFirstOutboundAudio = () => {
|
|
630
|
-
if (pendingFirstOutboundAt !== null) return
|
|
631
|
-
pendingFirstOutboundAt = now()
|
|
632
|
-
}
|
|
632
|
+
if (pendingFirstOutboundAt !== null) return;
|
|
633
|
+
pendingFirstOutboundAt = now();
|
|
634
|
+
};
|
|
633
635
|
const markFirstAudibleOutput = () => {
|
|
634
|
-
let target
|
|
636
|
+
let target;
|
|
635
637
|
for (const slot of inFlight.values()) {
|
|
636
638
|
if (!slot.ended) {
|
|
637
|
-
target = slot
|
|
639
|
+
target = slot;
|
|
638
640
|
}
|
|
639
641
|
}
|
|
640
|
-
if (!target) return
|
|
641
|
-
if (target.firstAudibleAt !== null) return
|
|
642
|
-
target.firstAudibleAt = now()
|
|
643
|
-
}
|
|
642
|
+
if (!target) return;
|
|
643
|
+
if (target.firstAudibleAt !== null) return;
|
|
644
|
+
target.firstAudibleAt = now();
|
|
645
|
+
};
|
|
644
646
|
const onAgentTurnStart = (seq) => {
|
|
645
647
|
inFlight.set(seq, {
|
|
646
648
|
firstOutboundAt: pendingFirstOutboundAt,
|
|
647
649
|
firstAudibleAt: null,
|
|
648
|
-
ended: false
|
|
649
|
-
})
|
|
650
|
-
pendingFirstOutboundAt = null
|
|
651
|
-
}
|
|
650
|
+
ended: false
|
|
651
|
+
});
|
|
652
|
+
pendingFirstOutboundAt = null;
|
|
653
|
+
};
|
|
652
654
|
const onAgentTurnEnd = (seq) => {
|
|
653
|
-
const slot = inFlight.get(seq)
|
|
655
|
+
const slot = inFlight.get(seq);
|
|
654
656
|
if (!slot) {
|
|
655
|
-
args.send({ type:
|
|
656
|
-
return
|
|
657
|
+
args.send({ type: "client_marks", seq, marks: {}, clientNow: Date.now() });
|
|
658
|
+
return;
|
|
657
659
|
}
|
|
658
|
-
slot.ended = true
|
|
659
|
-
tryEmit(seq)
|
|
660
|
-
}
|
|
660
|
+
slot.ended = true;
|
|
661
|
+
tryEmit(seq);
|
|
662
|
+
};
|
|
661
663
|
const flush = () => {
|
|
662
664
|
for (const seq of [...inFlight.keys()]) {
|
|
663
|
-
const slot = inFlight.get(seq)
|
|
664
|
-
slot.ended = true
|
|
665
|
-
tryEmit(seq)
|
|
665
|
+
const slot = inFlight.get(seq);
|
|
666
|
+
slot.ended = true;
|
|
667
|
+
tryEmit(seq);
|
|
666
668
|
}
|
|
667
|
-
pendingFirstOutboundAt = null
|
|
668
|
-
}
|
|
669
|
+
pendingFirstOutboundAt = null;
|
|
670
|
+
};
|
|
669
671
|
return {
|
|
670
672
|
markFirstOutboundAudio,
|
|
671
673
|
markFirstAudibleOutput,
|
|
672
674
|
onAgentTurnStart,
|
|
673
675
|
onAgentTurnEnd,
|
|
674
|
-
flush
|
|
675
|
-
}
|
|
676
|
-
}
|
|
676
|
+
flush
|
|
677
|
+
};
|
|
678
|
+
};
|
|
677
679
|
|
|
678
680
|
// src/VoiceClient.ts
|
|
679
681
|
var BrowserVoiceClient = class {
|
|
680
682
|
constructor(args) {
|
|
681
|
-
this.rws = null
|
|
682
|
-
this.capture = null
|
|
683
|
-
this.playback = null
|
|
684
|
-
this.muted = false
|
|
685
|
-
this.inputVolume = 0
|
|
686
|
-
this.outputVolume = 0
|
|
687
|
-
this.startedAt = null
|
|
688
|
-
this.endedFired = false
|
|
689
|
-
this.lastError = null
|
|
683
|
+
this.rws = null;
|
|
684
|
+
this.capture = null;
|
|
685
|
+
this.playback = null;
|
|
686
|
+
this.muted = false;
|
|
687
|
+
this.inputVolume = 0;
|
|
688
|
+
this.outputVolume = 0;
|
|
689
|
+
this.startedAt = null;
|
|
690
|
+
this.endedFired = false;
|
|
691
|
+
this.lastError = null;
|
|
690
692
|
this.end = () => {
|
|
691
|
-
this.teardown(
|
|
692
|
-
}
|
|
693
|
+
this.teardown("user_hangup");
|
|
694
|
+
};
|
|
693
695
|
this.mute = () => {
|
|
694
|
-
if (this.muted) return
|
|
695
|
-
this.muted = true
|
|
696
|
-
this.capture?.mute(true)
|
|
697
|
-
}
|
|
696
|
+
if (this.muted) return;
|
|
697
|
+
this.muted = true;
|
|
698
|
+
this.capture?.mute(true);
|
|
699
|
+
};
|
|
698
700
|
this.unmute = () => {
|
|
699
|
-
if (!this.muted) return
|
|
700
|
-
this.muted = false
|
|
701
|
-
this.capture?.mute(false)
|
|
702
|
-
}
|
|
701
|
+
if (!this.muted) return;
|
|
702
|
+
this.muted = false;
|
|
703
|
+
this.capture?.mute(false);
|
|
704
|
+
};
|
|
703
705
|
// ---------------------------------------------------------------
|
|
704
706
|
// Internal
|
|
705
707
|
// ---------------------------------------------------------------
|
|
706
708
|
this.sendClientToolsRegister = () => {
|
|
707
|
-
const frame = buildRegisterFrame(this.args.options.clientTools ?? {})
|
|
708
|
-
this.rws?.send(JSON.stringify(frame))
|
|
709
|
-
}
|
|
709
|
+
const frame = buildRegisterFrame(this.args.options.clientTools ?? {});
|
|
710
|
+
this.rws?.send(JSON.stringify(frame));
|
|
711
|
+
};
|
|
710
712
|
this.setState = (next) => {
|
|
711
|
-
if (this.proto.state === next) return
|
|
712
|
-
this.proto.state = next
|
|
713
|
-
this.args.options.onStateChange?.(next)
|
|
714
|
-
}
|
|
713
|
+
if (this.proto.state === next) return;
|
|
714
|
+
this.proto.state = next;
|
|
715
|
+
this.args.options.onStateChange?.(next);
|
|
716
|
+
};
|
|
715
717
|
this.emitError = (err) => {
|
|
716
|
-
this.lastError = err
|
|
717
|
-
this.args.options.onError?.(err)
|
|
718
|
-
}
|
|
718
|
+
this.lastError = err;
|
|
719
|
+
this.args.options.onError?.(err);
|
|
720
|
+
};
|
|
719
721
|
this.handleSocketEvent = (ev) => {
|
|
720
722
|
switch (ev.type) {
|
|
721
|
-
case
|
|
722
|
-
void this.startCapture()
|
|
723
|
-
break
|
|
724
|
-
case
|
|
725
|
-
this.proto.transcript = []
|
|
726
|
-
this.proto.agentBubbleId = null
|
|
727
|
-
this.args.options.onTranscript?.(this.proto.transcript)
|
|
728
|
-
void this.startCapture()
|
|
729
|
-
this.setState(
|
|
730
|
-
break
|
|
731
|
-
case
|
|
732
|
-
if (typeof ev.data ===
|
|
723
|
+
case "open":
|
|
724
|
+
void this.startCapture();
|
|
725
|
+
break;
|
|
726
|
+
case "reconnected":
|
|
727
|
+
this.proto.transcript = [];
|
|
728
|
+
this.proto.agentBubbleId = null;
|
|
729
|
+
this.args.options.onTranscript?.(this.proto.transcript);
|
|
730
|
+
void this.startCapture();
|
|
731
|
+
this.setState("listening");
|
|
732
|
+
break;
|
|
733
|
+
case "message":
|
|
734
|
+
if (typeof ev.data === "string") {
|
|
733
735
|
handleServerMessage(ev.data, this.proto, {
|
|
734
736
|
onState: this.setState,
|
|
735
737
|
onTranscript: (entries) => this.args.options.onTranscript?.(entries),
|
|
736
738
|
onError: this.emitError,
|
|
737
739
|
onInterrupt: () => {
|
|
738
|
-
this.playback?.flush()
|
|
739
|
-
this.args.options.onInterrupt?.()
|
|
740
|
+
this.playback?.flush();
|
|
741
|
+
this.args.options.onInterrupt?.();
|
|
740
742
|
},
|
|
741
743
|
onAgentTurnStart: (seq) => {
|
|
742
|
-
if (typeof seq ===
|
|
743
|
-
this.args.options.onAgentTurnStart?.()
|
|
744
|
+
if (typeof seq === "number") this.marks.onAgentTurnStart(seq);
|
|
745
|
+
this.args.options.onAgentTurnStart?.();
|
|
744
746
|
},
|
|
745
747
|
onAgentTurnEnd: (seq) => {
|
|
746
|
-
if (typeof seq ===
|
|
748
|
+
if (typeof seq === "number") this.marks.onAgentTurnEnd(seq);
|
|
747
749
|
},
|
|
748
750
|
onCallEnd: (reason) => this.teardown(reason),
|
|
749
751
|
onConnected: () => this.sendClientToolsRegister(),
|
|
750
|
-
onClientToolCall: (frame) =>
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
})
|
|
752
|
+
onClientToolCall: (frame) => dispatchClientToolCall(
|
|
753
|
+
(f) => this.rws?.send(JSON.stringify(f)),
|
|
754
|
+
this.args.options.clientTools ?? {},
|
|
755
|
+
frame
|
|
756
|
+
)
|
|
757
|
+
});
|
|
757
758
|
} else {
|
|
758
|
-
this.marks.markFirstAudibleOutput()
|
|
759
|
-
this.playback?.enqueue(ev.data)
|
|
759
|
+
this.marks.markFirstAudibleOutput();
|
|
760
|
+
this.playback?.enqueue(ev.data);
|
|
760
761
|
}
|
|
761
|
-
break
|
|
762
|
-
case
|
|
762
|
+
break;
|
|
763
|
+
case "close":
|
|
763
764
|
if (ev.permanent) {
|
|
764
|
-
const reason = this.proto.endReason ?? (this.lastError ?
|
|
765
|
-
this.teardown(reason)
|
|
765
|
+
const reason = this.proto.endReason ?? (this.lastError ? "error" : "user_hangup");
|
|
766
|
+
this.teardown(reason);
|
|
766
767
|
}
|
|
767
|
-
break
|
|
768
|
-
case
|
|
769
|
-
this.emitError({ code:
|
|
770
|
-
break
|
|
768
|
+
break;
|
|
769
|
+
case "error":
|
|
770
|
+
this.emitError({ code: "socket_error", message: ev.error.message });
|
|
771
|
+
break;
|
|
771
772
|
}
|
|
772
|
-
}
|
|
773
|
+
};
|
|
773
774
|
this.startCapture = async () => {
|
|
774
|
-
if (this.capture?.isCapturing()) return
|
|
775
|
+
if (this.capture?.isCapturing()) return;
|
|
775
776
|
this.capture = createAudioCapture({
|
|
776
777
|
onChunk: (pcm) => {
|
|
777
|
-
this.marks.markFirstOutboundAudio()
|
|
778
|
-
this.rws?.send(pcm)
|
|
778
|
+
this.marks.markFirstOutboundAudio();
|
|
779
|
+
this.rws?.send(pcm);
|
|
779
780
|
},
|
|
780
781
|
onVolume: (v) => {
|
|
781
|
-
this.inputVolume = v
|
|
782
|
-
this.args.options.onVolume?.({ input: v, output: this.outputVolume })
|
|
782
|
+
this.inputVolume = v;
|
|
783
|
+
this.args.options.onVolume?.({ input: v, output: this.outputVolume });
|
|
783
784
|
},
|
|
784
785
|
onError: (err) => {
|
|
785
786
|
this.emitError({
|
|
786
|
-
code: err.name ===
|
|
787
|
-
message: err.message
|
|
788
|
-
})
|
|
789
|
-
}
|
|
790
|
-
})
|
|
791
|
-
if (this.muted) this.capture.mute(true)
|
|
787
|
+
code: err.name === "NotAllowedError" ? "mic_denied" : "mic_start_failed",
|
|
788
|
+
message: err.message
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
});
|
|
792
|
+
if (this.muted) this.capture.mute(true);
|
|
792
793
|
try {
|
|
793
|
-
await this.capture.start()
|
|
794
|
-
} catch {
|
|
795
|
-
|
|
794
|
+
await this.capture.start();
|
|
795
|
+
} catch {
|
|
796
|
+
}
|
|
797
|
+
};
|
|
796
798
|
this.teardown = (reason) => {
|
|
797
799
|
try {
|
|
798
|
-
this.marks.flush()
|
|
799
|
-
} catch {
|
|
800
|
-
|
|
801
|
-
this.capture
|
|
802
|
-
this.
|
|
803
|
-
this.playback
|
|
800
|
+
this.marks.flush();
|
|
801
|
+
} catch {
|
|
802
|
+
}
|
|
803
|
+
this.capture?.stop();
|
|
804
|
+
this.capture = null;
|
|
805
|
+
this.playback?.close();
|
|
806
|
+
this.playback = null;
|
|
804
807
|
try {
|
|
805
|
-
this.rws?.close(1e3, reason)
|
|
806
|
-
} catch {
|
|
807
|
-
|
|
808
|
-
this.
|
|
809
|
-
this.
|
|
810
|
-
|
|
808
|
+
this.rws?.close(1e3, reason);
|
|
809
|
+
} catch {
|
|
810
|
+
}
|
|
811
|
+
this.rws = null;
|
|
812
|
+
this.setState("ended");
|
|
813
|
+
this.fireEndOnce(reason);
|
|
814
|
+
};
|
|
811
815
|
this.fireEndOnce = (reason) => {
|
|
812
|
-
if (this.endedFired) return
|
|
813
|
-
this.endedFired = true
|
|
814
|
-
const startedAt = this.startedAt ?? Date.now()
|
|
816
|
+
if (this.endedFired) return;
|
|
817
|
+
this.endedFired = true;
|
|
818
|
+
const startedAt = this.startedAt ?? Date.now();
|
|
815
819
|
this.args.options.onEnd?.({
|
|
816
820
|
reason,
|
|
817
|
-
errorCode: reason ===
|
|
818
|
-
durationMs: Date.now() - startedAt
|
|
819
|
-
})
|
|
820
|
-
}
|
|
821
|
-
this.args = args
|
|
822
|
-
this.proto = createProtocolState()
|
|
823
|
-
validateClientToolMap(args.options.clientTools)
|
|
821
|
+
errorCode: reason === "error" ? this.lastError?.code : void 0,
|
|
822
|
+
durationMs: Date.now() - startedAt
|
|
823
|
+
});
|
|
824
|
+
};
|
|
825
|
+
this.args = args;
|
|
826
|
+
this.proto = createProtocolState();
|
|
827
|
+
validateClientToolMap(args.options.clientTools);
|
|
824
828
|
this.marks = createClientMarksBuffer({
|
|
825
829
|
send: (frame) => {
|
|
826
830
|
try {
|
|
827
|
-
this.rws?.send(JSON.stringify(frame))
|
|
828
|
-
} catch {
|
|
829
|
-
|
|
830
|
-
|
|
831
|
+
this.rws?.send(JSON.stringify(frame));
|
|
832
|
+
} catch {
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
});
|
|
831
836
|
}
|
|
832
837
|
// ---------------------------------------------------------------
|
|
833
838
|
// Call interface
|
|
834
839
|
// ---------------------------------------------------------------
|
|
835
840
|
get state() {
|
|
836
|
-
return this.proto.state
|
|
841
|
+
return this.proto.state;
|
|
837
842
|
}
|
|
838
843
|
get transcript() {
|
|
839
|
-
return this.proto.transcript.slice()
|
|
844
|
+
return this.proto.transcript.slice();
|
|
840
845
|
}
|
|
841
846
|
get isMuted() {
|
|
842
|
-
return this.muted
|
|
847
|
+
return this.muted;
|
|
843
848
|
}
|
|
844
849
|
// ---------------------------------------------------------------
|
|
845
850
|
// Lifecycle — called by the factory immediately after construction.
|
|
@@ -847,52 +852,54 @@ var BrowserVoiceClient = class {
|
|
|
847
852
|
// failures arrive via `onError`.
|
|
848
853
|
// ---------------------------------------------------------------
|
|
849
854
|
async start() {
|
|
850
|
-
this.setState(
|
|
851
|
-
this.startedAt = Date.now()
|
|
855
|
+
this.setState("connecting");
|
|
856
|
+
this.startedAt = Date.now();
|
|
852
857
|
const url = buildWsUrl({
|
|
853
858
|
apiBase: this.args.config.apiBase,
|
|
854
859
|
agentId: this.args.options.agentId,
|
|
855
860
|
token: this.args.token,
|
|
856
|
-
bargeIn: this.args.options.bargeIn
|
|
857
|
-
})
|
|
861
|
+
bargeIn: this.args.options.bargeIn
|
|
862
|
+
});
|
|
858
863
|
this.playback = createAudioPlayback({
|
|
859
864
|
onVolume: (v) => {
|
|
860
|
-
this.outputVolume = v
|
|
861
|
-
this.args.options.onVolume?.({ input: this.inputVolume, output: v })
|
|
862
|
-
}
|
|
863
|
-
})
|
|
865
|
+
this.outputVolume = v;
|
|
866
|
+
this.args.options.onVolume?.({ input: this.inputVolume, output: v });
|
|
867
|
+
}
|
|
868
|
+
});
|
|
864
869
|
try {
|
|
865
|
-
await this.playback.resume()
|
|
866
|
-
} catch {
|
|
870
|
+
await this.playback.resume();
|
|
871
|
+
} catch {
|
|
872
|
+
}
|
|
867
873
|
this.rws = createReconnectingWebSocket(
|
|
868
874
|
{
|
|
869
875
|
url,
|
|
870
876
|
wsFactory: this.args.wsFactory,
|
|
871
|
-
maxRetries: 3
|
|
877
|
+
maxRetries: 3
|
|
872
878
|
},
|
|
873
|
-
(ev) => this.handleSocketEvent(ev)
|
|
874
|
-
)
|
|
879
|
+
(ev) => this.handleSocketEvent(ev)
|
|
880
|
+
);
|
|
875
881
|
}
|
|
876
|
-
}
|
|
882
|
+
};
|
|
877
883
|
|
|
878
884
|
// src/webrtc/createWebRtcCall.ts
|
|
879
885
|
async function createWebRtcCall(opts) {
|
|
880
|
-
validateClientToolMap(opts.clientTools)
|
|
881
|
-
const proto = createProtocolState()
|
|
882
|
-
let muted = false
|
|
883
|
-
let ended = false
|
|
884
|
-
const tools = opts.clientTools ?? {}
|
|
886
|
+
validateClientToolMap(opts.clientTools);
|
|
887
|
+
const proto = createProtocolState();
|
|
888
|
+
let muted = false;
|
|
889
|
+
let ended = false;
|
|
890
|
+
const tools = opts.clientTools ?? {};
|
|
885
891
|
const sendControl = (frame) => {
|
|
886
|
-
if (dc?.readyState !==
|
|
892
|
+
if (dc?.readyState !== "open") return;
|
|
887
893
|
try {
|
|
888
|
-
dc.send(JSON.stringify(frame))
|
|
889
|
-
} catch {
|
|
890
|
-
|
|
894
|
+
dc.send(JSON.stringify(frame));
|
|
895
|
+
} catch {
|
|
896
|
+
}
|
|
897
|
+
};
|
|
891
898
|
const fireState = (next) => {
|
|
892
|
-
if (proto.state === next) return
|
|
893
|
-
proto.state = next
|
|
894
|
-
opts.onStateChange?.(next)
|
|
895
|
-
}
|
|
899
|
+
if (proto.state === next) return;
|
|
900
|
+
proto.state = next;
|
|
901
|
+
opts.onStateChange?.(next);
|
|
902
|
+
};
|
|
896
903
|
const dispatch = (raw) => {
|
|
897
904
|
handleServerMessage(raw, proto, {
|
|
898
905
|
onState: fireState,
|
|
@@ -900,438 +907,555 @@ async function createWebRtcCall(opts) {
|
|
|
900
907
|
onError: (err) => opts.onError?.(err),
|
|
901
908
|
onInterrupt: () => opts.onInterrupt?.(),
|
|
902
909
|
onAgentTurnStart: () => opts.onAgentTurnStart?.(),
|
|
903
|
-
onAgentTurnEnd: () => {
|
|
910
|
+
onAgentTurnEnd: () => {
|
|
911
|
+
},
|
|
904
912
|
onCallEnd: () => teardown(),
|
|
905
913
|
onConnected: () => {
|
|
906
914
|
if (Object.keys(tools).length > 0) {
|
|
907
|
-
sendControl(buildRegisterFrame(tools))
|
|
915
|
+
sendControl(buildRegisterFrame(tools));
|
|
908
916
|
}
|
|
909
917
|
},
|
|
910
918
|
onClientToolCall: (frame) => {
|
|
911
|
-
dispatchClientToolCall(sendControl, tools, frame)
|
|
912
|
-
}
|
|
913
|
-
})
|
|
914
|
-
}
|
|
915
|
-
fireState(
|
|
919
|
+
dispatchClientToolCall(sendControl, tools, frame);
|
|
920
|
+
}
|
|
921
|
+
});
|
|
922
|
+
};
|
|
923
|
+
fireState("connecting");
|
|
916
924
|
const pc = new RTCPeerConnection({
|
|
917
|
-
iceServers: [{ urls:
|
|
918
|
-
})
|
|
919
|
-
const audioEl = document.createElement(
|
|
920
|
-
audioEl.autoplay = true
|
|
921
|
-
audioEl.style.display =
|
|
922
|
-
document.body.appendChild(audioEl)
|
|
925
|
+
iceServers: [{ urls: "stun:stun.l.google.com:19302" }]
|
|
926
|
+
});
|
|
927
|
+
const audioEl = document.createElement("audio");
|
|
928
|
+
audioEl.autoplay = true;
|
|
929
|
+
audioEl.style.display = "none";
|
|
930
|
+
document.body.appendChild(audioEl);
|
|
923
931
|
pc.ontrack = (event) => {
|
|
924
|
-
audioEl.srcObject = event.streams[0] ?? new MediaStream([event.track])
|
|
925
|
-
}
|
|
926
|
-
let mic
|
|
932
|
+
audioEl.srcObject = event.streams[0] ?? new MediaStream([event.track]);
|
|
933
|
+
};
|
|
934
|
+
let mic;
|
|
927
935
|
try {
|
|
928
|
-
mic = await navigator.mediaDevices.getUserMedia({ audio: true })
|
|
936
|
+
mic = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
929
937
|
} catch (err) {
|
|
930
|
-
const code =
|
|
931
|
-
err instanceof DOMException && err.name === 'NotAllowedError'
|
|
932
|
-
? 'mic_denied'
|
|
933
|
-
: 'mic_start_failed'
|
|
938
|
+
const code = err instanceof DOMException && err.name === "NotAllowedError" ? "mic_denied" : "mic_start_failed";
|
|
934
939
|
opts.onError?.({
|
|
935
940
|
code,
|
|
936
|
-
message: err instanceof Error ? err.message :
|
|
937
|
-
})
|
|
938
|
-
fireState(
|
|
939
|
-
pc.close()
|
|
940
|
-
audioEl.remove()
|
|
941
|
-
throw err
|
|
942
|
-
}
|
|
943
|
-
for (const track of mic.getAudioTracks()) pc.addTrack(track, mic)
|
|
944
|
-
const dc = pc.createDataChannel(
|
|
941
|
+
message: err instanceof Error ? err.message : "getUserMedia failed"
|
|
942
|
+
});
|
|
943
|
+
fireState("error");
|
|
944
|
+
pc.close();
|
|
945
|
+
audioEl.remove();
|
|
946
|
+
throw err;
|
|
947
|
+
}
|
|
948
|
+
for (const track of mic.getAudioTracks()) pc.addTrack(track, mic);
|
|
949
|
+
const dc = pc.createDataChannel("control", { ordered: true });
|
|
945
950
|
dc.onmessage = (e) => {
|
|
946
|
-
if (typeof e.data ===
|
|
947
|
-
}
|
|
951
|
+
if (typeof e.data === "string") dispatch(e.data);
|
|
952
|
+
};
|
|
948
953
|
dc.onerror = () => {
|
|
949
|
-
opts.onError?.({ code:
|
|
950
|
-
}
|
|
954
|
+
opts.onError?.({ code: "socket_error", message: "control channel error" });
|
|
955
|
+
};
|
|
951
956
|
dc.onopen = () => {
|
|
952
957
|
if (Object.keys(tools).length > 0) {
|
|
953
|
-
sendControl(buildRegisterFrame(tools))
|
|
958
|
+
sendControl(buildRegisterFrame(tools));
|
|
954
959
|
}
|
|
955
|
-
}
|
|
956
|
-
const gateway = opts.webrtcGatewayBase ||
|
|
957
|
-
const offerUrl = gateway
|
|
958
|
-
|
|
959
|
-
: `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/offer?token=${encodeURIComponent(opts.token)}`
|
|
960
|
-
const iceUrl = gateway
|
|
961
|
-
? `${gateway}/webrtc/ice?token=${encodeURIComponent(opts.token)}`
|
|
962
|
-
: `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/ice?token=${encodeURIComponent(opts.token)}`
|
|
960
|
+
};
|
|
961
|
+
const gateway = opts.webrtcGatewayBase || "";
|
|
962
|
+
const offerUrl = gateway ? `${gateway}/webrtc/offer?token=${encodeURIComponent(opts.token)}` : `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/offer?token=${encodeURIComponent(opts.token)}`;
|
|
963
|
+
const iceUrl = gateway ? `${gateway}/webrtc/ice?token=${encodeURIComponent(opts.token)}` : `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/ice?token=${encodeURIComponent(opts.token)}`;
|
|
963
964
|
const teardown = () => {
|
|
964
|
-
if (ended) return
|
|
965
|
-
ended = true
|
|
965
|
+
if (ended) return;
|
|
966
|
+
ended = true;
|
|
966
967
|
try {
|
|
967
|
-
mic.getTracks().forEach((t) => t.stop())
|
|
968
|
-
} catch {
|
|
968
|
+
mic.getTracks().forEach((t) => t.stop());
|
|
969
|
+
} catch {
|
|
970
|
+
}
|
|
969
971
|
try {
|
|
970
|
-
pc.close()
|
|
971
|
-
} catch {
|
|
972
|
+
pc.close();
|
|
973
|
+
} catch {
|
|
974
|
+
}
|
|
972
975
|
try {
|
|
973
|
-
audioEl.remove()
|
|
974
|
-
} catch {
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
976
|
+
audioEl.remove();
|
|
977
|
+
} catch {
|
|
978
|
+
}
|
|
979
|
+
fireState("ended");
|
|
980
|
+
opts.onEnd?.();
|
|
981
|
+
};
|
|
982
|
+
let callId = null;
|
|
983
|
+
const pendingCandidates = [];
|
|
980
984
|
const postCandidate = (candidate) => {
|
|
981
985
|
void fetch(iceUrl, {
|
|
982
|
-
method:
|
|
983
|
-
headers: {
|
|
984
|
-
body: JSON.stringify({ callId, candidate })
|
|
985
|
-
}).catch(() => {
|
|
986
|
-
|
|
986
|
+
method: "POST",
|
|
987
|
+
headers: { "content-type": "application/json" },
|
|
988
|
+
body: JSON.stringify({ callId, candidate })
|
|
989
|
+
}).catch(() => {
|
|
990
|
+
});
|
|
991
|
+
};
|
|
987
992
|
pc.onicecandidate = (e) => {
|
|
988
|
-
if (!e.candidate) return
|
|
989
|
-
if (callId) postCandidate(e.candidate)
|
|
990
|
-
else pendingCandidates.push(e.candidate)
|
|
991
|
-
}
|
|
993
|
+
if (!e.candidate) return;
|
|
994
|
+
if (callId) postCandidate(e.candidate);
|
|
995
|
+
else pendingCandidates.push(e.candidate);
|
|
996
|
+
};
|
|
992
997
|
pc.onconnectionstatechange = () => {
|
|
993
|
-
const s = pc.connectionState
|
|
994
|
-
if (s ===
|
|
995
|
-
if (s ===
|
|
996
|
-
opts.onError?.({ code:
|
|
997
|
-
teardown()
|
|
998
|
+
const s = pc.connectionState;
|
|
999
|
+
if (s === "connected") fireState("listening");
|
|
1000
|
+
if (s === "failed" || s === "disconnected") {
|
|
1001
|
+
opts.onError?.({ code: "socket_error", message: `webrtc connection ${s}` });
|
|
1002
|
+
teardown();
|
|
998
1003
|
}
|
|
999
|
-
if (s ===
|
|
1000
|
-
}
|
|
1001
|
-
await pc.setLocalDescription(await pc.createOffer())
|
|
1004
|
+
if (s === "closed" && !ended) teardown();
|
|
1005
|
+
};
|
|
1006
|
+
await pc.setLocalDescription(await pc.createOffer());
|
|
1002
1007
|
try {
|
|
1003
1008
|
const offerRes = await fetch(offerUrl, {
|
|
1004
|
-
method:
|
|
1005
|
-
headers: {
|
|
1006
|
-
body: JSON.stringify({ sdp: pc.localDescription.sdp, type:
|
|
1007
|
-
})
|
|
1009
|
+
method: "POST",
|
|
1010
|
+
headers: { "content-type": "application/json" },
|
|
1011
|
+
body: JSON.stringify({ sdp: pc.localDescription.sdp, type: "offer", agentId: opts.agentId })
|
|
1012
|
+
});
|
|
1008
1013
|
if (!offerRes.ok) {
|
|
1009
|
-
const code = offerRes.status === 401 ?
|
|
1010
|
-
opts.onError?.({ code, message: `signaling failed: HTTP ${offerRes.status}` })
|
|
1011
|
-
fireState(
|
|
1012
|
-
mic.getTracks().forEach((t) => t.stop())
|
|
1013
|
-
pc.close()
|
|
1014
|
-
audioEl.remove()
|
|
1015
|
-
throw new Error(`webrtc offer failed: ${offerRes.status}`)
|
|
1014
|
+
const code = offerRes.status === 401 ? "unauthorized" : "server_error";
|
|
1015
|
+
opts.onError?.({ code, message: `signaling failed: HTTP ${offerRes.status}` });
|
|
1016
|
+
fireState("error");
|
|
1017
|
+
mic.getTracks().forEach((t) => t.stop());
|
|
1018
|
+
pc.close();
|
|
1019
|
+
audioEl.remove();
|
|
1020
|
+
throw new Error(`webrtc offer failed: ${offerRes.status}`);
|
|
1016
1021
|
}
|
|
1017
|
-
const body = await offerRes.json()
|
|
1018
|
-
callId = body.callId
|
|
1019
|
-
await pc.setRemoteDescription({ type:
|
|
1020
|
-
while (pendingCandidates.length > 0) postCandidate(pendingCandidates.shift())
|
|
1022
|
+
const body = await offerRes.json();
|
|
1023
|
+
callId = body.callId;
|
|
1024
|
+
await pc.setRemoteDescription({ type: "answer", sdp: body.sdp });
|
|
1025
|
+
while (pendingCandidates.length > 0) postCandidate(pendingCandidates.shift());
|
|
1021
1026
|
} catch (err) {
|
|
1022
1027
|
if (!ended) {
|
|
1023
1028
|
opts.onError?.({
|
|
1024
|
-
code:
|
|
1025
|
-
message: err instanceof Error ? err.message :
|
|
1026
|
-
})
|
|
1027
|
-
fireState(
|
|
1028
|
-
mic.getTracks().forEach((t) => t.stop())
|
|
1029
|
-
pc.close()
|
|
1030
|
-
audioEl.remove()
|
|
1029
|
+
code: "network_unreachable",
|
|
1030
|
+
message: err instanceof Error ? err.message : "signaling failed"
|
|
1031
|
+
});
|
|
1032
|
+
fireState("error");
|
|
1033
|
+
mic.getTracks().forEach((t) => t.stop());
|
|
1034
|
+
pc.close();
|
|
1035
|
+
audioEl.remove();
|
|
1031
1036
|
}
|
|
1032
|
-
throw err
|
|
1037
|
+
throw err;
|
|
1033
1038
|
}
|
|
1034
1039
|
return {
|
|
1035
1040
|
get state() {
|
|
1036
|
-
return proto.state
|
|
1041
|
+
return proto.state;
|
|
1037
1042
|
},
|
|
1038
1043
|
get transcript() {
|
|
1039
|
-
return proto.transcript.slice()
|
|
1044
|
+
return proto.transcript.slice();
|
|
1040
1045
|
},
|
|
1041
1046
|
get isMuted() {
|
|
1042
|
-
return muted
|
|
1047
|
+
return muted;
|
|
1043
1048
|
},
|
|
1044
1049
|
end: () => teardown(),
|
|
1045
1050
|
mute: () => {
|
|
1046
|
-
if (muted) return
|
|
1047
|
-
muted = true
|
|
1048
|
-
mic.getAudioTracks().forEach((t) =>
|
|
1051
|
+
if (muted) return;
|
|
1052
|
+
muted = true;
|
|
1053
|
+
mic.getAudioTracks().forEach((t) => t.enabled = false);
|
|
1049
1054
|
},
|
|
1050
1055
|
unmute: () => {
|
|
1051
|
-
if (!muted) return
|
|
1052
|
-
muted = false
|
|
1053
|
-
mic.getAudioTracks().forEach((t) =>
|
|
1056
|
+
if (!muted) return;
|
|
1057
|
+
muted = false;
|
|
1058
|
+
mic.getAudioTracks().forEach((t) => t.enabled = true);
|
|
1059
|
+
}
|
|
1060
|
+
};
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
// src/textSession.ts
|
|
1064
|
+
async function startTextSession(opts) {
|
|
1065
|
+
const f = opts.fetch ?? fetch;
|
|
1066
|
+
const tokenQs = `?token=${encodeURIComponent(opts.token)}`;
|
|
1067
|
+
const startUrl = `${opts.baseUrl}/v1/agents/${opts.agentId}/chat${tokenQs}`;
|
|
1068
|
+
const startBody = opts.text ? JSON.stringify({ text: opts.text }) : "{}";
|
|
1069
|
+
const res = await f(startUrl, {
|
|
1070
|
+
method: "POST",
|
|
1071
|
+
headers: { "Content-Type": "application/json", Accept: "text/event-stream" },
|
|
1072
|
+
body: startBody
|
|
1073
|
+
});
|
|
1074
|
+
if (!res.ok || !res.body) {
|
|
1075
|
+
const text = await res.text().catch(() => "");
|
|
1076
|
+
throw new Error(`startTextSession failed: ${res.status} ${text}`);
|
|
1077
|
+
}
|
|
1078
|
+
const iter = parseSse(res.body);
|
|
1079
|
+
let chatId = "";
|
|
1080
|
+
let callId = "";
|
|
1081
|
+
const buffered = [];
|
|
1082
|
+
const it = iter[Symbol.asyncIterator]();
|
|
1083
|
+
while (true) {
|
|
1084
|
+
const { value, done } = await it.next();
|
|
1085
|
+
if (done) break;
|
|
1086
|
+
if (value.type === "chat.started") {
|
|
1087
|
+
chatId = value.chatId;
|
|
1088
|
+
callId = value.callId;
|
|
1089
|
+
break;
|
|
1090
|
+
}
|
|
1091
|
+
buffered.push(value);
|
|
1092
|
+
}
|
|
1093
|
+
return {
|
|
1094
|
+
id: chatId,
|
|
1095
|
+
callId,
|
|
1096
|
+
greeting: replayThen(buffered, { [Symbol.asyncIterator]: () => it }),
|
|
1097
|
+
async send(text) {
|
|
1098
|
+
const r = await f(`${opts.baseUrl}/v1/chats/${chatId}/messages${tokenQs}`, {
|
|
1099
|
+
method: "POST",
|
|
1100
|
+
headers: { "Content-Type": "application/json", Accept: "text/event-stream" },
|
|
1101
|
+
body: JSON.stringify({ text })
|
|
1102
|
+
});
|
|
1103
|
+
if (!r.ok || !r.body) {
|
|
1104
|
+
const errText = await r.text().catch(() => "");
|
|
1105
|
+
throw new Error(`send failed: ${r.status} ${errText}`);
|
|
1106
|
+
}
|
|
1107
|
+
return parseSse(r.body);
|
|
1054
1108
|
},
|
|
1109
|
+
async end() {
|
|
1110
|
+
await f(`${opts.baseUrl}/v1/calls/${callId}`, { method: "DELETE" });
|
|
1111
|
+
}
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
async function* parseSse(body) {
|
|
1115
|
+
const reader = body.getReader();
|
|
1116
|
+
const decoder = new TextDecoder();
|
|
1117
|
+
let buf = "";
|
|
1118
|
+
while (true) {
|
|
1119
|
+
const { value, done } = await reader.read();
|
|
1120
|
+
if (done) return;
|
|
1121
|
+
buf += decoder.decode(value, { stream: true });
|
|
1122
|
+
let idx;
|
|
1123
|
+
while ((idx = buf.indexOf("\n\n")) >= 0) {
|
|
1124
|
+
const chunk = buf.slice(0, idx);
|
|
1125
|
+
buf = buf.slice(idx + 2);
|
|
1126
|
+
let event = "message";
|
|
1127
|
+
let data = "";
|
|
1128
|
+
for (const line of chunk.split("\n")) {
|
|
1129
|
+
if (line.startsWith(":")) continue;
|
|
1130
|
+
if (line.startsWith("event:")) event = line.slice(6).trim();
|
|
1131
|
+
else if (line.startsWith("data:")) data += line.slice(5).trim();
|
|
1132
|
+
}
|
|
1133
|
+
if (!data) continue;
|
|
1134
|
+
try {
|
|
1135
|
+
const parsed = JSON.parse(data);
|
|
1136
|
+
yield { type: event, ...parsed };
|
|
1137
|
+
} catch {
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1055
1140
|
}
|
|
1056
1141
|
}
|
|
1142
|
+
async function* replayThen(buffered, rest) {
|
|
1143
|
+
for (const x of buffered) yield x;
|
|
1144
|
+
for await (const x of rest) yield x;
|
|
1145
|
+
}
|
|
1057
1146
|
|
|
1058
|
-
// src/
|
|
1059
|
-
var
|
|
1147
|
+
// src/incomingCall.ts
|
|
1148
|
+
var parseIncomingCall = (raw) => {
|
|
1149
|
+
if (typeof raw !== "object" || raw === null) {
|
|
1150
|
+
throw new Error("parseIncomingCall: payload must be an object");
|
|
1151
|
+
}
|
|
1152
|
+
const p = raw;
|
|
1153
|
+
if (typeof p.token !== "string" || !p.token.startsWith("ct_")) {
|
|
1154
|
+
throw new Error("parseIncomingCall: missing or invalid `token` (expected a ct_ string)");
|
|
1155
|
+
}
|
|
1156
|
+
if (typeof p.agentId !== "string" || p.agentId.length === 0) {
|
|
1157
|
+
throw new Error("parseIncomingCall: missing `agentId`");
|
|
1158
|
+
}
|
|
1159
|
+
const transport = p.transport === "webrtc" ? "webrtc" : "ws";
|
|
1160
|
+
const out = { token: p.token, agentId: p.agentId, transport };
|
|
1161
|
+
if (transport === "webrtc" && typeof p.webrtcGatewayBase === "string") {
|
|
1162
|
+
out.webrtcGatewayBase = p.webrtcGatewayBase;
|
|
1163
|
+
}
|
|
1164
|
+
if (typeof p.expiresAt === "number") out.expiresAt = p.expiresAt;
|
|
1165
|
+
if (typeof p.agentName === "string") out.agentName = p.agentName;
|
|
1166
|
+
if (typeof p.agentAvatarUrl === "string") out.agentAvatarUrl = p.agentAvatarUrl;
|
|
1167
|
+
return out;
|
|
1168
|
+
};
|
|
1169
|
+
|
|
1170
|
+
// src/assistant.ts
|
|
1171
|
+
var browserWsFactory = (url) => new globalThis.WebSocket(url);
|
|
1172
|
+
var AssistantVoiceFactory = class {
|
|
1173
|
+
constructor(config) {
|
|
1174
|
+
this.startCall = async (options) => {
|
|
1175
|
+
if (!options.agentId) {
|
|
1176
|
+
throw new Error("startCall: agentId is required");
|
|
1177
|
+
}
|
|
1178
|
+
const { context, metadata } = mergeStartCallContext(this.config, options);
|
|
1179
|
+
const fetchArgs = {
|
|
1180
|
+
agentId: options.agentId,
|
|
1181
|
+
userId: options.userId,
|
|
1182
|
+
context,
|
|
1183
|
+
metadata
|
|
1184
|
+
};
|
|
1185
|
+
let resolved;
|
|
1186
|
+
if (options.token) {
|
|
1187
|
+
resolved = { token: options.token, transport: "ws" };
|
|
1188
|
+
} else {
|
|
1189
|
+
const r = await this.config.fetchToken(fetchArgs);
|
|
1190
|
+
if (!r) {
|
|
1191
|
+
throw new Error("configureVoiceClient.fetchToken returned empty token");
|
|
1192
|
+
}
|
|
1193
|
+
resolved = typeof r === "string" ? { token: r, transport: "ws" } : r;
|
|
1194
|
+
if (!resolved.token) {
|
|
1195
|
+
throw new Error("configureVoiceClient.fetchToken returned an object without `token`");
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
if (resolved.transport === "webrtc") {
|
|
1199
|
+
return createWebRtcCall({
|
|
1200
|
+
agentId: options.agentId,
|
|
1201
|
+
apiBase: this.config.apiBase,
|
|
1202
|
+
token: resolved.token,
|
|
1203
|
+
webrtcGatewayBase: resolved.webrtcGatewayBase,
|
|
1204
|
+
onStateChange: options.onStateChange,
|
|
1205
|
+
onTranscript: options.onTranscript,
|
|
1206
|
+
onError: options.onError,
|
|
1207
|
+
// Synthesise a minimal CallEndEvent. WebRTC doesn't carry an end reason
|
|
1208
|
+
// from the server yet — use 'agent_ended' as placeholder. durationMs is
|
|
1209
|
+
// tracked at 0 until the followup lands (see spec Followups section).
|
|
1210
|
+
onEnd: options.onEnd ? () => options.onEnd({ reason: "agent_ended", durationMs: 0 }) : void 0,
|
|
1211
|
+
onInterrupt: options.onInterrupt,
|
|
1212
|
+
onAgentTurnStart: options.onAgentTurnStart,
|
|
1213
|
+
clientTools: options.clientTools
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
const client = new BrowserVoiceClient({
|
|
1217
|
+
config: this.config,
|
|
1218
|
+
// Carry merged context/metadata through to startCall so server can
|
|
1219
|
+
// see what the SDK saw.
|
|
1220
|
+
options: { ...options, context, metadata },
|
|
1221
|
+
token: resolved.token,
|
|
1222
|
+
wsFactory: browserWsFactory
|
|
1223
|
+
});
|
|
1224
|
+
await client.start();
|
|
1225
|
+
return client;
|
|
1226
|
+
};
|
|
1227
|
+
// Text-channel chat session (no microphone / audio).
|
|
1228
|
+
// Mint a `ct_` token with `channel: 'text'` on your backend, then call
|
|
1229
|
+
// this to open an SSE stream against the chat API.
|
|
1230
|
+
this.startTextSession = (opts) => startTextSession({
|
|
1231
|
+
...opts,
|
|
1232
|
+
baseUrl: this.config.apiBase
|
|
1233
|
+
});
|
|
1234
|
+
this.config = config;
|
|
1235
|
+
}
|
|
1236
|
+
};
|
|
1237
|
+
function configureVoiceClient(config) {
|
|
1238
|
+
return new AssistantVoiceFactory(normalizeConfig(config));
|
|
1239
|
+
}
|
|
1060
1240
|
|
|
1061
1241
|
// src/roomProtocol.ts
|
|
1062
|
-
var SYSTEM_TOPIC =
|
|
1063
|
-
var TRANSCRIPT_TOPIC =
|
|
1242
|
+
var SYSTEM_TOPIC = "system";
|
|
1243
|
+
var TRANSCRIPT_TOPIC = "transcript";
|
|
1244
|
+
var ANALYSIS_TOPIC = "analysis";
|
|
1064
1245
|
var decodeSystem = (bytes) => {
|
|
1065
1246
|
try {
|
|
1066
|
-
const v = JSON.parse(new TextDecoder().decode(bytes))
|
|
1067
|
-
if (v && typeof v.kind ===
|
|
1068
|
-
return null
|
|
1247
|
+
const v = JSON.parse(new TextDecoder().decode(bytes));
|
|
1248
|
+
if (v && typeof v.kind === "string") return v;
|
|
1249
|
+
return null;
|
|
1069
1250
|
} catch {
|
|
1070
|
-
return null
|
|
1251
|
+
return null;
|
|
1071
1252
|
}
|
|
1072
|
-
}
|
|
1253
|
+
};
|
|
1073
1254
|
var decodeTranscript = (bytes) => {
|
|
1074
1255
|
try {
|
|
1075
|
-
const v = JSON.parse(new TextDecoder().decode(bytes))
|
|
1076
|
-
if (v && v.kind ===
|
|
1077
|
-
return null
|
|
1256
|
+
const v = JSON.parse(new TextDecoder().decode(bytes));
|
|
1257
|
+
if (v && v.kind === "partial") return v;
|
|
1258
|
+
return null;
|
|
1078
1259
|
} catch {
|
|
1079
|
-
return null
|
|
1260
|
+
return null;
|
|
1080
1261
|
}
|
|
1081
|
-
}
|
|
1262
|
+
};
|
|
1263
|
+
var decodeAnalysis = (bytes) => {
|
|
1264
|
+
try {
|
|
1265
|
+
const v = JSON.parse(new TextDecoder().decode(bytes));
|
|
1266
|
+
if (v && v.kind === "analysis" && typeof v.participantId === "string" && typeof v.label === "string") {
|
|
1267
|
+
return v;
|
|
1268
|
+
}
|
|
1269
|
+
return null;
|
|
1270
|
+
} catch {
|
|
1271
|
+
return null;
|
|
1272
|
+
}
|
|
1273
|
+
};
|
|
1082
1274
|
|
|
1083
1275
|
// src/room.ts
|
|
1084
|
-
var
|
|
1085
|
-
|
|
1276
|
+
var lkPromise;
|
|
1277
|
+
var loadLiveKit = () => {
|
|
1278
|
+
lkPromise ?? (lkPromise = import("livekit-client").catch((e) => {
|
|
1279
|
+
lkPromise = void 0;
|
|
1280
|
+
const msg = "joinRoom requires the optional peer dependency 'livekit-client' \u2014 npm install livekit-client";
|
|
1281
|
+
const wrapped = new Error(msg);
|
|
1282
|
+
wrapped.cause = e;
|
|
1283
|
+
throw wrapped;
|
|
1284
|
+
}));
|
|
1285
|
+
return lkPromise;
|
|
1286
|
+
};
|
|
1287
|
+
var identityToPid = (identity) => identity.startsWith("guest:") ? identity.slice("guest:".length) : identity;
|
|
1086
1288
|
var joinRoom = async (opts) => {
|
|
1087
|
-
const exchangeUrl = `${opts.apiBase.replace(/\/+$/,
|
|
1088
|
-
opts.roomId
|
|
1089
|
-
)}/join
|
|
1289
|
+
const exchangeUrl = `${opts.apiBase.replace(/\/+$/, "")}/v1/rooms/${encodeURIComponent(
|
|
1290
|
+
opts.roomId
|
|
1291
|
+
)}/join`;
|
|
1090
1292
|
const exchangeRes = await fetch(exchangeUrl, {
|
|
1091
|
-
method:
|
|
1092
|
-
headers: {
|
|
1093
|
-
body: JSON.stringify({ code: opts.joinCode, name: opts.name })
|
|
1094
|
-
})
|
|
1293
|
+
method: "POST",
|
|
1294
|
+
headers: { "Content-Type": "application/json" },
|
|
1295
|
+
body: JSON.stringify({ code: opts.joinCode, name: opts.name })
|
|
1296
|
+
});
|
|
1095
1297
|
if (!exchangeRes.ok) {
|
|
1096
|
-
const err = await exchangeRes.json().catch(() => ({}))
|
|
1097
|
-
throw new Error(err.error?.code ?? `join_failed_${exchangeRes.status}`)
|
|
1298
|
+
const err = await exchangeRes.json().catch(() => ({}));
|
|
1299
|
+
throw new Error(err.error?.code ?? `join_failed_${exchangeRes.status}`);
|
|
1098
1300
|
}
|
|
1099
|
-
const exchange = await exchangeRes.json()
|
|
1100
|
-
const handlers = /* @__PURE__ */ new Map()
|
|
1301
|
+
const exchange = await exchangeRes.json();
|
|
1302
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
1101
1303
|
const emit = (e, payload) => {
|
|
1102
1304
|
handlers.get(e)?.forEach((h) => {
|
|
1103
1305
|
try {
|
|
1104
|
-
h(payload)
|
|
1105
|
-
} catch {
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1306
|
+
h(payload);
|
|
1307
|
+
} catch {
|
|
1308
|
+
}
|
|
1309
|
+
});
|
|
1310
|
+
};
|
|
1311
|
+
const lk = await loadLiveKit();
|
|
1312
|
+
const trackKind = (t) => t.kind === lk.Track.Kind.Video ? "video" : "audio";
|
|
1313
|
+
const trackSource = (s) => {
|
|
1314
|
+
switch (s) {
|
|
1315
|
+
case lk.Track.Source.Camera:
|
|
1316
|
+
return "camera";
|
|
1317
|
+
case lk.Track.Source.Microphone:
|
|
1318
|
+
return "microphone";
|
|
1319
|
+
case lk.Track.Source.ScreenShare:
|
|
1320
|
+
return "screen_share";
|
|
1321
|
+
case lk.Track.Source.ScreenShareAudio:
|
|
1322
|
+
return "screen_share_audio";
|
|
1323
|
+
default:
|
|
1324
|
+
return "unknown";
|
|
1325
|
+
}
|
|
1326
|
+
};
|
|
1327
|
+
const room = new lk.Room({ adaptiveStream: true, dynacast: true });
|
|
1328
|
+
room.on(
|
|
1329
|
+
lk.RoomEvent.ParticipantConnected,
|
|
1330
|
+
(p) => emit("participant.joined", {
|
|
1111
1331
|
participantId: identityToPid(p.identity),
|
|
1112
|
-
name: p.name ??
|
|
1113
|
-
})
|
|
1114
|
-
)
|
|
1115
|
-
room.on(
|
|
1116
|
-
|
|
1332
|
+
name: p.name ?? ""
|
|
1333
|
+
})
|
|
1334
|
+
);
|
|
1335
|
+
room.on(
|
|
1336
|
+
lk.RoomEvent.ParticipantDisconnected,
|
|
1337
|
+
(p) => emit("participant.left", {
|
|
1117
1338
|
participantId: identityToPid(p.identity),
|
|
1118
|
-
name: p.name ??
|
|
1119
|
-
})
|
|
1120
|
-
)
|
|
1121
|
-
room.on(
|
|
1122
|
-
room.on(
|
|
1339
|
+
name: p.name ?? ""
|
|
1340
|
+
})
|
|
1341
|
+
);
|
|
1342
|
+
room.on(lk.RoomEvent.Disconnected, () => emit("room.ended", void 0));
|
|
1343
|
+
room.on(lk.RoomEvent.DataReceived, (data, _participant, _kind, topic) => {
|
|
1123
1344
|
if (topic === SYSTEM_TOPIC) {
|
|
1124
|
-
const m = decodeSystem(data)
|
|
1125
|
-
if (m) emit(
|
|
1345
|
+
const m = decodeSystem(data);
|
|
1346
|
+
if (m) emit("system.message", m);
|
|
1126
1347
|
} else if (topic === TRANSCRIPT_TOPIC) {
|
|
1127
|
-
const m = decodeTranscript(data)
|
|
1128
|
-
if (m) emit(
|
|
1129
|
-
}
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
const trackSource = (s) => {
|
|
1133
|
-
switch (s) {
|
|
1134
|
-
case import_livekit_client.Track.Source.Camera:
|
|
1135
|
-
return 'camera'
|
|
1136
|
-
case import_livekit_client.Track.Source.Microphone:
|
|
1137
|
-
return 'microphone'
|
|
1138
|
-
case import_livekit_client.Track.Source.ScreenShare:
|
|
1139
|
-
return 'screen_share'
|
|
1140
|
-
case import_livekit_client.Track.Source.ScreenShareAudio:
|
|
1141
|
-
return 'screen_share_audio'
|
|
1142
|
-
default:
|
|
1143
|
-
return 'unknown'
|
|
1348
|
+
const m = decodeTranscript(data);
|
|
1349
|
+
if (m) emit("transcript.partial", m);
|
|
1350
|
+
} else if (topic === ANALYSIS_TOPIC) {
|
|
1351
|
+
const m = decodeAnalysis(data);
|
|
1352
|
+
if (m) emit("analysis", m);
|
|
1144
1353
|
}
|
|
1145
|
-
}
|
|
1146
|
-
room.on(
|
|
1147
|
-
|
|
1354
|
+
});
|
|
1355
|
+
room.on(
|
|
1356
|
+
lk.RoomEvent.TrackSubscribed,
|
|
1357
|
+
(track, pub, participant) => emit("track.subscribed", {
|
|
1148
1358
|
participantId: identityToPid(participant.identity),
|
|
1149
1359
|
kind: trackKind(track),
|
|
1150
1360
|
source: trackSource(pub.source),
|
|
1151
|
-
track
|
|
1152
|
-
})
|
|
1153
|
-
)
|
|
1154
|
-
room.on(
|
|
1155
|
-
|
|
1361
|
+
track
|
|
1362
|
+
})
|
|
1363
|
+
);
|
|
1364
|
+
room.on(
|
|
1365
|
+
lk.RoomEvent.TrackUnsubscribed,
|
|
1366
|
+
(track, pub, participant) => emit("track.unsubscribed", {
|
|
1156
1367
|
participantId: identityToPid(participant.identity),
|
|
1157
1368
|
kind: trackKind(track),
|
|
1158
1369
|
source: trackSource(pub.source),
|
|
1159
|
-
track
|
|
1160
|
-
})
|
|
1161
|
-
)
|
|
1162
|
-
room.on(
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
speakers
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1370
|
+
track
|
|
1371
|
+
})
|
|
1372
|
+
);
|
|
1373
|
+
room.on(
|
|
1374
|
+
lk.RoomEvent.ActiveSpeakersChanged,
|
|
1375
|
+
(speakers) => emit(
|
|
1376
|
+
"active.speakers",
|
|
1377
|
+
speakers.map((p) => identityToPid(p.identity))
|
|
1378
|
+
)
|
|
1379
|
+
);
|
|
1380
|
+
await room.connect(exchange.livekit.url, exchange.livekit.token);
|
|
1169
1381
|
return {
|
|
1170
1382
|
participantId: exchange.participantId,
|
|
1383
|
+
kind: exchange.kind ?? "video",
|
|
1171
1384
|
get participants() {
|
|
1172
1385
|
return [...room.remoteParticipants.values()].map((p) => ({
|
|
1173
1386
|
participantId: identityToPid(p.identity),
|
|
1174
|
-
name: p.name ??
|
|
1175
|
-
}))
|
|
1387
|
+
name: p.name ?? ""
|
|
1388
|
+
}));
|
|
1176
1389
|
},
|
|
1177
1390
|
on(event, handler) {
|
|
1178
|
-
const set = handlers.get(event) ?? /* @__PURE__ */ new Set()
|
|
1179
|
-
set.add(handler)
|
|
1180
|
-
handlers.set(event, set)
|
|
1391
|
+
const set = handlers.get(event) ?? /* @__PURE__ */ new Set();
|
|
1392
|
+
set.add(handler);
|
|
1393
|
+
handlers.set(event, set);
|
|
1181
1394
|
},
|
|
1182
1395
|
publishMic: async () => {
|
|
1183
|
-
await room.localParticipant.setMicrophoneEnabled(true)
|
|
1396
|
+
await room.localParticipant.setMicrophoneEnabled(true);
|
|
1184
1397
|
},
|
|
1185
1398
|
publishCamera: async () => {
|
|
1186
|
-
await room.localParticipant.setCameraEnabled(true)
|
|
1399
|
+
await room.localParticipant.setCameraEnabled(true);
|
|
1187
1400
|
},
|
|
1188
1401
|
setMicEnabled: async (on) => {
|
|
1189
|
-
await room.localParticipant.setMicrophoneEnabled(on)
|
|
1402
|
+
await room.localParticipant.setMicrophoneEnabled(on);
|
|
1190
1403
|
},
|
|
1191
1404
|
setCameraEnabled: async (on) => {
|
|
1192
|
-
await room.localParticipant.setCameraEnabled(on)
|
|
1405
|
+
await room.localParticipant.setCameraEnabled(on);
|
|
1193
1406
|
},
|
|
1194
1407
|
isMicEnabled: () => room.localParticipant.isMicrophoneEnabled,
|
|
1195
1408
|
isCameraEnabled: () => room.localParticipant.isCameraEnabled,
|
|
1196
|
-
getLocalCameraTrack: () =>
|
|
1197
|
-
room.localParticipant.getTrackPublication(import_livekit_client.Track.Source.Camera)
|
|
1198
|
-
?.videoTrack ?? null,
|
|
1409
|
+
getLocalCameraTrack: () => room.localParticipant.getTrackPublication(lk.Track.Source.Camera)?.videoTrack ?? null,
|
|
1199
1410
|
getRemoteTracks: () => {
|
|
1200
|
-
const out = []
|
|
1411
|
+
const out = [];
|
|
1201
1412
|
for (const p of room.remoteParticipants.values()) {
|
|
1202
1413
|
for (const pub of p.trackPublications.values()) {
|
|
1203
|
-
const track = pub.track
|
|
1204
|
-
if (!track) continue
|
|
1414
|
+
const track = pub.track;
|
|
1415
|
+
if (!track) continue;
|
|
1205
1416
|
out.push({
|
|
1206
1417
|
participantId: identityToPid(p.identity),
|
|
1207
1418
|
kind: trackKind(track),
|
|
1208
1419
|
source: trackSource(pub.source),
|
|
1209
|
-
track
|
|
1210
|
-
})
|
|
1420
|
+
track
|
|
1421
|
+
});
|
|
1211
1422
|
}
|
|
1212
1423
|
}
|
|
1213
|
-
return out
|
|
1424
|
+
return out;
|
|
1214
1425
|
},
|
|
1215
1426
|
setScreenShareEnabled: async (on, opts2) => {
|
|
1216
|
-
await room.localParticipant.setScreenShareEnabled(on, { audio: opts2?.audio ?? false })
|
|
1427
|
+
await room.localParticipant.setScreenShareEnabled(on, { audio: opts2?.audio ?? false });
|
|
1217
1428
|
},
|
|
1218
1429
|
isScreenShareEnabled: () => room.localParticipant.isScreenShareEnabled,
|
|
1219
|
-
getLocalScreenTrack: () =>
|
|
1220
|
-
room.localParticipant.getTrackPublication(import_livekit_client.Track.Source.ScreenShare)
|
|
1221
|
-
?.videoTrack ?? null,
|
|
1430
|
+
getLocalScreenTrack: () => room.localParticipant.getTrackPublication(lk.Track.Source.ScreenShare)?.videoTrack ?? null,
|
|
1222
1431
|
leave: async () => {
|
|
1223
|
-
await room.disconnect()
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
// src/incomingCall.ts
|
|
1229
|
-
var parseIncomingCall = (raw) => {
|
|
1230
|
-
if (typeof raw !== 'object' || raw === null) {
|
|
1231
|
-
throw new Error('parseIncomingCall: payload must be an object')
|
|
1232
|
-
}
|
|
1233
|
-
const p = raw
|
|
1234
|
-
if (typeof p.token !== 'string' || !p.token.startsWith('ct_')) {
|
|
1235
|
-
throw new Error('parseIncomingCall: missing or invalid `token` (expected a ct_ string)')
|
|
1236
|
-
}
|
|
1237
|
-
if (typeof p.agentId !== 'string' || p.agentId.length === 0) {
|
|
1238
|
-
throw new Error('parseIncomingCall: missing `agentId`')
|
|
1239
|
-
}
|
|
1240
|
-
const transport = p.transport === 'webrtc' ? 'webrtc' : 'ws'
|
|
1241
|
-
const out = { token: p.token, agentId: p.agentId, transport }
|
|
1242
|
-
if (transport === 'webrtc' && typeof p.webrtcGatewayBase === 'string') {
|
|
1243
|
-
out.webrtcGatewayBase = p.webrtcGatewayBase
|
|
1244
|
-
}
|
|
1245
|
-
if (typeof p.expiresAt === 'number') out.expiresAt = p.expiresAt
|
|
1246
|
-
if (typeof p.agentName === 'string') out.agentName = p.agentName
|
|
1247
|
-
if (typeof p.agentAvatarUrl === 'string') out.agentAvatarUrl = p.agentAvatarUrl
|
|
1248
|
-
return out
|
|
1249
|
-
}
|
|
1432
|
+
await room.disconnect();
|
|
1433
|
+
}
|
|
1434
|
+
};
|
|
1435
|
+
};
|
|
1250
1436
|
|
|
1251
1437
|
// src/browser.ts
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
agentId: options.agentId,
|
|
1262
|
-
userId: options.userId,
|
|
1263
|
-
context,
|
|
1264
|
-
metadata,
|
|
1265
|
-
}
|
|
1266
|
-
let resolved
|
|
1267
|
-
if (options.token) {
|
|
1268
|
-
resolved = { token: options.token, transport: 'ws' }
|
|
1269
|
-
} else {
|
|
1270
|
-
const r = await this.config.fetchToken(fetchArgs)
|
|
1271
|
-
if (!r) {
|
|
1272
|
-
throw new Error('configureVoiceClient.fetchToken returned empty token')
|
|
1273
|
-
}
|
|
1274
|
-
resolved = typeof r === 'string' ? { token: r, transport: 'ws' } : r
|
|
1275
|
-
if (!resolved.token) {
|
|
1276
|
-
throw new Error('configureVoiceClient.fetchToken returned an object without `token`')
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
if (resolved.transport === 'webrtc') {
|
|
1280
|
-
return createWebRtcCall({
|
|
1281
|
-
agentId: options.agentId,
|
|
1282
|
-
apiBase: this.config.apiBase,
|
|
1283
|
-
token: resolved.token,
|
|
1284
|
-
webrtcGatewayBase: resolved.webrtcGatewayBase,
|
|
1285
|
-
onStateChange: options.onStateChange,
|
|
1286
|
-
onTranscript: options.onTranscript,
|
|
1287
|
-
onError: options.onError,
|
|
1288
|
-
// Synthesise a minimal CallEndEvent. WebRTC doesn't carry an end reason
|
|
1289
|
-
// from the server yet — use 'agent_ended' as placeholder. durationMs is
|
|
1290
|
-
// tracked at 0 until the followup lands (see spec Followups section).
|
|
1291
|
-
onEnd: options.onEnd
|
|
1292
|
-
? () => options.onEnd({ reason: 'agent_ended', durationMs: 0 })
|
|
1293
|
-
: void 0,
|
|
1294
|
-
onInterrupt: options.onInterrupt,
|
|
1295
|
-
onAgentTurnStart: options.onAgentTurnStart,
|
|
1296
|
-
clientTools: options.clientTools,
|
|
1297
|
-
})
|
|
1298
|
-
}
|
|
1299
|
-
const client = new BrowserVoiceClient({
|
|
1300
|
-
config: this.config,
|
|
1301
|
-
// Carry merged context/metadata through to startCall so server can
|
|
1302
|
-
// see what the SDK saw.
|
|
1303
|
-
options: { ...options, context, metadata },
|
|
1304
|
-
token: resolved.token,
|
|
1305
|
-
wsFactory: browserWsFactory,
|
|
1306
|
-
})
|
|
1307
|
-
await client.start()
|
|
1308
|
-
return client
|
|
1309
|
-
}
|
|
1310
|
-
// Multi-party rooms (Phase 7 video).
|
|
1311
|
-
//
|
|
1312
|
-
// The guest's browser calls this with the roomId + joinCode it parsed
|
|
1313
|
-
// out of the invite link. The SDK exchanges the code for a LiveKit
|
|
1314
|
-
// JWT against `${apiBase}/v1/rooms/:roomId/join` (an AUTH-EXEMPT
|
|
1315
|
-
// endpoint — the opaque code is the only credential), then connects
|
|
1316
|
-
// to LiveKit and returns a typed event surface.
|
|
1317
|
-
this.joinRoom = (opts) => joinRoom({ apiBase: this.config.apiBase, ...opts })
|
|
1318
|
-
this.config = config
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
function configureVoiceClient(config) {
|
|
1322
|
-
return new BrowserVoiceFactory(normalizeConfig(config))
|
|
1438
|
+
function configureVoiceClient2(config) {
|
|
1439
|
+
const normalized = normalizeConfig(config);
|
|
1440
|
+
const factory = configureVoiceClient(normalized);
|
|
1441
|
+
return {
|
|
1442
|
+
...factory,
|
|
1443
|
+
// Re-compose joinRoom onto the factory so barrel consumers who call
|
|
1444
|
+
// `voice.joinRoom(...)` keep working unchanged.
|
|
1445
|
+
joinRoom: (opts) => joinRoom({ apiBase: normalized.apiBase, ...opts })
|
|
1446
|
+
};
|
|
1323
1447
|
}
|
|
1324
1448
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1325
|
-
0 &&
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
//# sourceMappingURL=browser.js.map
|
|
1449
|
+
0 && (module.exports = {
|
|
1450
|
+
buildWsUrl,
|
|
1451
|
+
configureVoiceClient,
|
|
1452
|
+
createAudioCapture,
|
|
1453
|
+
createAudioPlayback,
|
|
1454
|
+
createProtocolState,
|
|
1455
|
+
createReconnectingWebSocket,
|
|
1456
|
+
handleServerMessage,
|
|
1457
|
+
joinRoom,
|
|
1458
|
+
parseIncomingCall,
|
|
1459
|
+
startTextSession
|
|
1460
|
+
});
|
|
1461
|
+
//# sourceMappingURL=browser.js.map
|