@nightzeros/chatai-widget-core 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +205 -0
- package/README.md +72 -0
- package/dist/index.d.ts +301 -2
- package/dist/index.js +1162 -14
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,4 +1,999 @@
|
|
|
1
|
+
// src/control-health.ts
|
|
2
|
+
var CONTROL_HEALTH_DEFAULTS = {
|
|
3
|
+
intervalMs: 5e3,
|
|
4
|
+
reconnectIntervalMs: 2e3,
|
|
5
|
+
requestTimeoutMs: 4e3,
|
|
6
|
+
failuresToReconnect: 2,
|
|
7
|
+
silenceToReconnectMs: 12e3,
|
|
8
|
+
reconnectGraceMs: 2e4,
|
|
9
|
+
lostCloseWindowMs: 1e4,
|
|
10
|
+
tickMs: 500
|
|
11
|
+
};
|
|
12
|
+
function initialControlHealth(now) {
|
|
13
|
+
return {
|
|
14
|
+
status: "healthy",
|
|
15
|
+
failures: 0,
|
|
16
|
+
lastSuccessAt: now,
|
|
17
|
+
reconnectingSince: null,
|
|
18
|
+
closingSince: null,
|
|
19
|
+
endReason: null,
|
|
20
|
+
cause: null
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
function isControlMuted(status) {
|
|
24
|
+
return status === "reconnecting" || status === "closing";
|
|
25
|
+
}
|
|
26
|
+
function terminated(state, endReason, cause) {
|
|
27
|
+
return { ...state, status: "terminated", endReason, cause };
|
|
28
|
+
}
|
|
29
|
+
function reconnecting(state, now) {
|
|
30
|
+
return { ...state, status: "reconnecting", reconnectingSince: state.reconnectingSince ?? now };
|
|
31
|
+
}
|
|
32
|
+
function reduceControlHealth(state, input, now, settings = CONTROL_HEALTH_DEFAULTS) {
|
|
33
|
+
if (state.status === "terminated") return state;
|
|
34
|
+
switch (input.kind) {
|
|
35
|
+
case "tick":
|
|
36
|
+
if (state.status === "closing" && now - (state.closingSince ?? now) >= settings.lostCloseWindowMs) {
|
|
37
|
+
return terminated(state, "disconnected", "lost");
|
|
38
|
+
}
|
|
39
|
+
if (state.status === "reconnecting" && now - (state.reconnectingSince ?? now) >= settings.reconnectGraceMs) {
|
|
40
|
+
return terminated(state, "disconnected", "grace_expired");
|
|
41
|
+
}
|
|
42
|
+
if ((state.status === "healthy" || state.status === "suspect") && now - state.lastSuccessAt >= settings.silenceToReconnectMs) {
|
|
43
|
+
return reconnecting(state, now);
|
|
44
|
+
}
|
|
45
|
+
return state;
|
|
46
|
+
case "healthy":
|
|
47
|
+
if (state.status === "closing") return state;
|
|
48
|
+
return { ...state, status: "healthy", failures: 0, lastSuccessAt: now, reconnectingSince: null };
|
|
49
|
+
case "degraded":
|
|
50
|
+
if (state.status === "closing") return state;
|
|
51
|
+
return reconnecting({ ...state, failures: 0 }, now);
|
|
52
|
+
case "neutral":
|
|
53
|
+
return state;
|
|
54
|
+
case "failure": {
|
|
55
|
+
const failures = state.failures + 1;
|
|
56
|
+
if (state.status === "healthy" || state.status === "suspect") {
|
|
57
|
+
return failures >= settings.failuresToReconnect ? reconnecting({ ...state, failures }, now) : { ...state, status: "suspect", failures };
|
|
58
|
+
}
|
|
59
|
+
return { ...state, failures };
|
|
60
|
+
}
|
|
61
|
+
case "ended":
|
|
62
|
+
return terminated(state, input.endReason ?? "disconnected", "ended");
|
|
63
|
+
case "lost":
|
|
64
|
+
if (state.status === "closing") return state;
|
|
65
|
+
return { ...state, status: "closing", closingSince: now, failures: 0 };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function classifyHeartbeatResponse(status, body) {
|
|
69
|
+
const data = body && typeof body === "object" ? body : {};
|
|
70
|
+
const nextMs = typeof data.nextHeartbeatMs === "number" && data.nextHeartbeatMs > 0 ? data.nextHeartbeatMs : void 0;
|
|
71
|
+
if (status === 421 || status === 429) return { kind: "neutral", ...nextMs ? { nextMs } : {} };
|
|
72
|
+
if (status === 404) return { kind: "lost" };
|
|
73
|
+
if (status !== 200) return { kind: "failure" };
|
|
74
|
+
switch (data.state) {
|
|
75
|
+
case "healthy":
|
|
76
|
+
return { kind: "healthy", ...nextMs ? { nextMs } : {} };
|
|
77
|
+
case "degraded":
|
|
78
|
+
return { kind: "degraded", ...nextMs ? { nextMs } : {} };
|
|
79
|
+
case "ended":
|
|
80
|
+
return { kind: "ended", endReason: typeof data.endReason === "string" ? data.endReason : null };
|
|
81
|
+
case "lost":
|
|
82
|
+
return { kind: "lost" };
|
|
83
|
+
default:
|
|
84
|
+
return { kind: "failure" };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function createControlHealthMonitor(options) {
|
|
88
|
+
const now = options.now ?? (() => Date.now());
|
|
89
|
+
const settings = {
|
|
90
|
+
...CONTROL_HEALTH_DEFAULTS,
|
|
91
|
+
...options.settings,
|
|
92
|
+
...options.intervalMs && options.intervalMs > 0 ? { intervalMs: options.intervalMs } : {}
|
|
93
|
+
};
|
|
94
|
+
let state = initialControlHealth(now());
|
|
95
|
+
let running = false;
|
|
96
|
+
let beatTimer = null;
|
|
97
|
+
let tickTimer = null;
|
|
98
|
+
let inFlight = null;
|
|
99
|
+
let requestTimer = null;
|
|
100
|
+
const stop = () => {
|
|
101
|
+
running = false;
|
|
102
|
+
if (beatTimer) clearTimeout(beatTimer);
|
|
103
|
+
beatTimer = null;
|
|
104
|
+
if (requestTimer) clearTimeout(requestTimer);
|
|
105
|
+
requestTimer = null;
|
|
106
|
+
if (tickTimer) clearInterval(tickTimer);
|
|
107
|
+
tickTimer = null;
|
|
108
|
+
const request = inFlight;
|
|
109
|
+
inFlight = null;
|
|
110
|
+
request?.abort();
|
|
111
|
+
};
|
|
112
|
+
const apply = (input) => {
|
|
113
|
+
if (!running) return;
|
|
114
|
+
const previous = state;
|
|
115
|
+
const next = reduceControlHealth(previous, input, now(), settings);
|
|
116
|
+
state = next;
|
|
117
|
+
if (next.status === previous.status) return;
|
|
118
|
+
if (next.status === "terminated") stop();
|
|
119
|
+
options.onChange(next, previous);
|
|
120
|
+
};
|
|
121
|
+
const schedule = (delayMs) => {
|
|
122
|
+
if (!running) return;
|
|
123
|
+
if (beatTimer) clearTimeout(beatTimer);
|
|
124
|
+
beatTimer = setTimeout(() => void beat(), Math.max(0, delayMs));
|
|
125
|
+
};
|
|
126
|
+
const beat = async () => {
|
|
127
|
+
beatTimer = null;
|
|
128
|
+
if (!running || inFlight) return;
|
|
129
|
+
const controller = new AbortController();
|
|
130
|
+
inFlight = controller;
|
|
131
|
+
const abortedOrTimedOut = new Promise((resolve) => {
|
|
132
|
+
controller.signal.addEventListener("abort", () => resolve({ kind: "failure" }), { once: true });
|
|
133
|
+
requestTimer = setTimeout(() => controller.abort(), settings.requestTimeoutMs);
|
|
134
|
+
});
|
|
135
|
+
const outcome = await Promise.race([
|
|
136
|
+
options.send(controller.signal).catch(() => ({ kind: "failure" })),
|
|
137
|
+
abortedOrTimedOut
|
|
138
|
+
]);
|
|
139
|
+
if (inFlight !== controller) return;
|
|
140
|
+
if (requestTimer) clearTimeout(requestTimer);
|
|
141
|
+
requestTimer = null;
|
|
142
|
+
inFlight = null;
|
|
143
|
+
apply(outcome);
|
|
144
|
+
if (!running) return;
|
|
145
|
+
const hinted = "nextMs" in outcome ? outcome.nextMs : void 0;
|
|
146
|
+
const fast = isControlMuted(state.status);
|
|
147
|
+
schedule(fast ? Math.min(hinted ?? settings.reconnectIntervalMs, settings.reconnectIntervalMs) : hinted ?? settings.intervalMs);
|
|
148
|
+
};
|
|
149
|
+
return {
|
|
150
|
+
start() {
|
|
151
|
+
if (running || state.status === "terminated") return;
|
|
152
|
+
running = true;
|
|
153
|
+
state = initialControlHealth(now());
|
|
154
|
+
schedule(settings.intervalMs);
|
|
155
|
+
tickTimer = setInterval(() => apply({ kind: "tick" }), settings.tickMs);
|
|
156
|
+
},
|
|
157
|
+
stop,
|
|
158
|
+
state: () => state
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// src/voice-gate.ts
|
|
163
|
+
var PLAYBACK_GATE_DEFAULTS = {
|
|
164
|
+
reconnectMinMs: 250,
|
|
165
|
+
reconnectMaxMs: 4e3,
|
|
166
|
+
/** The server sends a keepalive every 10 s; a stream silent this long is dead. */
|
|
167
|
+
staleAfterMs: 25e3
|
|
168
|
+
};
|
|
169
|
+
function parseGateLine(line) {
|
|
170
|
+
let data;
|
|
171
|
+
try {
|
|
172
|
+
data = JSON.parse(line);
|
|
173
|
+
} catch {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
if (!data || typeof data !== "object") return null;
|
|
177
|
+
const event = data;
|
|
178
|
+
if (event.type === "keepalive" || event.type === "end") return { type: event.type };
|
|
179
|
+
if (event.type !== "gate" || typeof event.seq !== "number" || event.state !== "open" && event.state !== "closed") {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
type: "gate",
|
|
184
|
+
decision: {
|
|
185
|
+
seq: event.seq,
|
|
186
|
+
state: event.state,
|
|
187
|
+
inputEndMs: typeof event.inputEndMs === "number" ? event.inputEndMs : null
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
function createPlaybackGate(options) {
|
|
192
|
+
const settings = { ...PLAYBACK_GATE_DEFAULTS, ...options.settings };
|
|
193
|
+
let running = false;
|
|
194
|
+
let streamUp = false;
|
|
195
|
+
let decision = null;
|
|
196
|
+
let locallyClosed = false;
|
|
197
|
+
let lastInputStartMs = null;
|
|
198
|
+
let attempt = 0;
|
|
199
|
+
let inFlight = null;
|
|
200
|
+
let retryTimer = null;
|
|
201
|
+
let staleTimer = null;
|
|
202
|
+
let open = false;
|
|
203
|
+
const isOpen = () => running && streamUp && decision?.state === "open" && !locallyClosed;
|
|
204
|
+
const update = () => {
|
|
205
|
+
const next = isOpen();
|
|
206
|
+
if (next === open) return;
|
|
207
|
+
open = next;
|
|
208
|
+
options.onChange(open);
|
|
209
|
+
};
|
|
210
|
+
const spokeSince = (inputEndMs) => lastInputStartMs !== null && (inputEndMs === null || lastInputStartMs >= inputEndMs);
|
|
211
|
+
const apply = (next) => {
|
|
212
|
+
if (decision && next.seq <= decision.seq) return;
|
|
213
|
+
decision = next;
|
|
214
|
+
locallyClosed = next.state === "open" && spokeSince(next.inputEndMs);
|
|
215
|
+
};
|
|
216
|
+
const clearStale = () => {
|
|
217
|
+
if (staleTimer) clearTimeout(staleTimer);
|
|
218
|
+
staleTimer = null;
|
|
219
|
+
};
|
|
220
|
+
const armStale = (controller) => {
|
|
221
|
+
clearStale();
|
|
222
|
+
staleTimer = setTimeout(() => controller.abort(), settings.staleAfterMs);
|
|
223
|
+
};
|
|
224
|
+
const read = async (body, controller) => {
|
|
225
|
+
const reader = body.getReader();
|
|
226
|
+
const decoder = new TextDecoder();
|
|
227
|
+
let buffer = "";
|
|
228
|
+
armStale(controller);
|
|
229
|
+
try {
|
|
230
|
+
for (; ; ) {
|
|
231
|
+
const { value, done } = await reader.read();
|
|
232
|
+
if (done || inFlight !== controller) return false;
|
|
233
|
+
armStale(controller);
|
|
234
|
+
buffer += decoder.decode(value, { stream: true });
|
|
235
|
+
let newline = buffer.indexOf("\n");
|
|
236
|
+
while (newline >= 0) {
|
|
237
|
+
const event = parseGateLine(buffer.slice(0, newline));
|
|
238
|
+
buffer = buffer.slice(newline + 1);
|
|
239
|
+
newline = buffer.indexOf("\n");
|
|
240
|
+
if (event?.type === "end") return true;
|
|
241
|
+
if (event?.type === "gate") {
|
|
242
|
+
attempt = 0;
|
|
243
|
+
streamUp = true;
|
|
244
|
+
apply(event.decision);
|
|
245
|
+
update();
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
} finally {
|
|
250
|
+
reader.cancel().catch(() => void 0);
|
|
251
|
+
}
|
|
252
|
+
};
|
|
253
|
+
const connect = async () => {
|
|
254
|
+
retryTimer = null;
|
|
255
|
+
if (!running) return;
|
|
256
|
+
const controller = new AbortController();
|
|
257
|
+
inFlight = controller;
|
|
258
|
+
let finished = false;
|
|
259
|
+
try {
|
|
260
|
+
const response = await options.connect(controller.signal);
|
|
261
|
+
if (response.status === 404) finished = true;
|
|
262
|
+
else if (response.ok && response.body) finished = await read(response.body, controller);
|
|
263
|
+
} catch {
|
|
264
|
+
}
|
|
265
|
+
if (inFlight !== controller) return;
|
|
266
|
+
inFlight = null;
|
|
267
|
+
clearStale();
|
|
268
|
+
streamUp = false;
|
|
269
|
+
update();
|
|
270
|
+
if (finished || !running) return;
|
|
271
|
+
const delay = Math.min(settings.reconnectMaxMs, settings.reconnectMinMs * 2 ** attempt);
|
|
272
|
+
attempt += 1;
|
|
273
|
+
retryTimer = setTimeout(() => void connect(), delay);
|
|
274
|
+
};
|
|
275
|
+
return {
|
|
276
|
+
start() {
|
|
277
|
+
if (running) return;
|
|
278
|
+
running = true;
|
|
279
|
+
void connect();
|
|
280
|
+
},
|
|
281
|
+
stop() {
|
|
282
|
+
running = false;
|
|
283
|
+
if (retryTimer) clearTimeout(retryTimer);
|
|
284
|
+
retryTimer = null;
|
|
285
|
+
clearStale();
|
|
286
|
+
const request = inFlight;
|
|
287
|
+
inFlight = null;
|
|
288
|
+
request?.abort();
|
|
289
|
+
streamUp = false;
|
|
290
|
+
open = false;
|
|
291
|
+
},
|
|
292
|
+
isOpen,
|
|
293
|
+
noteInput(startMs) {
|
|
294
|
+
if (typeof startMs !== "number") return;
|
|
295
|
+
lastInputStartMs = Math.max(lastInputStartMs ?? startMs, startMs);
|
|
296
|
+
if (decision?.state === "open" && !locallyClosed && spokeSince(decision.inputEndMs)) {
|
|
297
|
+
locallyClosed = true;
|
|
298
|
+
update();
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// src/voice-state.ts
|
|
305
|
+
var VOICE_ACTIVITY_HOLD_MS = 450;
|
|
306
|
+
var VOICE_INTERRUPTED_HOLD_MS = 1500;
|
|
307
|
+
var VOICE_PHASE_LABELS = {
|
|
308
|
+
idle: "Voice is off",
|
|
309
|
+
requesting_mic: "Allow microphone access",
|
|
310
|
+
connecting: "Connecting\u2026",
|
|
311
|
+
listening: "Listening",
|
|
312
|
+
user_speaking: "Hearing you",
|
|
313
|
+
processing: "Looking that up\u2026",
|
|
314
|
+
assistant_speaking: "Speaking",
|
|
315
|
+
interrupted: "Interrupted \u2014 go ahead",
|
|
316
|
+
reconnecting: "Reconnecting\u2026",
|
|
317
|
+
error: "Voice stopped",
|
|
318
|
+
ended: "Voice ended"
|
|
319
|
+
};
|
|
320
|
+
function recent(at, now, windowMs) {
|
|
321
|
+
return at !== null && now - at < windowMs;
|
|
322
|
+
}
|
|
323
|
+
function isUserActive(signals) {
|
|
324
|
+
return recent(signals.userVoiceAt, signals.now, VOICE_ACTIVITY_HOLD_MS);
|
|
325
|
+
}
|
|
326
|
+
function isAssistantActive(signals) {
|
|
327
|
+
return recent(signals.assistantVoiceAt, signals.now, VOICE_ACTIVITY_HOLD_MS);
|
|
328
|
+
}
|
|
329
|
+
function deriveVoicePhase(signals) {
|
|
330
|
+
switch (signals.connection) {
|
|
331
|
+
case "idle":
|
|
332
|
+
return "idle";
|
|
333
|
+
case "requesting_mic":
|
|
334
|
+
return "requesting_mic";
|
|
335
|
+
case "connecting":
|
|
336
|
+
return "connecting";
|
|
337
|
+
case "reconnecting":
|
|
338
|
+
return "reconnecting";
|
|
339
|
+
case "failed":
|
|
340
|
+
return "error";
|
|
341
|
+
case "ended":
|
|
342
|
+
return "ended";
|
|
343
|
+
case "connected":
|
|
344
|
+
break;
|
|
345
|
+
}
|
|
346
|
+
if (recent(signals.interruptedAt, signals.now, VOICE_INTERRUPTED_HOLD_MS)) return "interrupted";
|
|
347
|
+
if (isUserActive(signals)) return "user_speaking";
|
|
348
|
+
if (signals.processing) return "processing";
|
|
349
|
+
if (isAssistantActive(signals)) return "assistant_speaking";
|
|
350
|
+
return "listening";
|
|
351
|
+
}
|
|
352
|
+
function detectInterruption(input) {
|
|
353
|
+
return !input.userWasActive && input.userIsActive && input.assistantIsActive;
|
|
354
|
+
}
|
|
355
|
+
function isVoiceActive(connection) {
|
|
356
|
+
return connection === "requesting_mic" || connection === "connecting" || connection === "connected" || connection === "reconnecting";
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// src/voice-transcript.ts
|
|
360
|
+
var BARGE_IN_MIN_WORDS = 2;
|
|
361
|
+
function words(text) {
|
|
362
|
+
return text.trim().split(/\s+/).filter(Boolean).length;
|
|
363
|
+
}
|
|
364
|
+
function createVoiceTranscript(idPrefix) {
|
|
365
|
+
const turns = [];
|
|
366
|
+
let pending = null;
|
|
367
|
+
let assistantEndMs = Number.NEGATIVE_INFINITY;
|
|
368
|
+
let counter = 0;
|
|
369
|
+
const push = (role, text) => {
|
|
370
|
+
turns.push({ id: `${idPrefix}:${counter++}`, role, text, interrupted: false });
|
|
371
|
+
};
|
|
372
|
+
const commitPending = () => {
|
|
373
|
+
if (!pending) return;
|
|
374
|
+
const last = turns.at(-1);
|
|
375
|
+
if (last?.role === "assistant") last.interrupted = true;
|
|
376
|
+
push("user", pending.text);
|
|
377
|
+
pending = null;
|
|
378
|
+
};
|
|
379
|
+
return {
|
|
380
|
+
input(delta, startMs) {
|
|
381
|
+
if (!delta) return;
|
|
382
|
+
const last = turns.at(-1);
|
|
383
|
+
if (pending) {
|
|
384
|
+
if (startMs >= assistantEndMs && pending.startMs < assistantEndMs && words(pending.text) < BARGE_IN_MIN_WORDS) {
|
|
385
|
+
pending = null;
|
|
386
|
+
push("user", delta);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
pending.text += delta;
|
|
390
|
+
if (words(pending.text) >= BARGE_IN_MIN_WORDS) commitPending();
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
if (last?.role === "user") {
|
|
394
|
+
last.text += delta;
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
if (last?.role === "assistant" && startMs < assistantEndMs) {
|
|
398
|
+
pending = { text: delta, startMs };
|
|
399
|
+
if (words(pending.text) >= BARGE_IN_MIN_WORDS) commitPending();
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
push("user", delta);
|
|
403
|
+
},
|
|
404
|
+
output(delta, endMs) {
|
|
405
|
+
if (!delta) return;
|
|
406
|
+
pending = null;
|
|
407
|
+
assistantEndMs = Math.max(assistantEndMs, endMs);
|
|
408
|
+
const last = turns.at(-1);
|
|
409
|
+
if (last?.role === "assistant") {
|
|
410
|
+
last.text += delta;
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
push("assistant", delta);
|
|
414
|
+
},
|
|
415
|
+
/** Committed turns in spoken order (copies). */
|
|
416
|
+
turns() {
|
|
417
|
+
return turns.filter((turn) => turn.text.trim()).map((turn) => ({ ...turn, text: turn.text.trim() }));
|
|
418
|
+
},
|
|
419
|
+
/** Ends the transcript; short speech still overlapping the assistant is dropped. */
|
|
420
|
+
finish() {
|
|
421
|
+
pending = null;
|
|
422
|
+
return this.turns();
|
|
423
|
+
}
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// src/voice.ts
|
|
428
|
+
var VOICE_END_NOTICES = {
|
|
429
|
+
voice_unavailable: "Voice is unavailable for the rest of this conversation, but we can keep chatting here.",
|
|
430
|
+
superseded: "Voice continued in another tab or window. You can keep typing here.",
|
|
431
|
+
disconnected: "Voice disconnected. You can keep typing here.",
|
|
432
|
+
idle: "Voice ended after a period of silence. You can keep typing here.",
|
|
433
|
+
max_duration: "Voice reached its maximum length. You can keep typing here."
|
|
434
|
+
};
|
|
435
|
+
function visitorEndReason(reported) {
|
|
436
|
+
switch (reported) {
|
|
437
|
+
case "voice_unavailable":
|
|
438
|
+
case "usage_limit":
|
|
439
|
+
return "voice_unavailable";
|
|
440
|
+
case "superseded":
|
|
441
|
+
return "superseded";
|
|
442
|
+
case "idle":
|
|
443
|
+
return "idle";
|
|
444
|
+
case "max_duration":
|
|
445
|
+
return "max_duration";
|
|
446
|
+
case "disconnected":
|
|
447
|
+
case "heartbeat_lost":
|
|
448
|
+
case "control_lost":
|
|
449
|
+
case "shutdown":
|
|
450
|
+
return "disconnected";
|
|
451
|
+
default:
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
var VOICE_ERROR_MESSAGES = {
|
|
456
|
+
unsupported: "Voice needs a browser with microphone and WebRTC support on a secure (HTTPS) page.",
|
|
457
|
+
mic_denied: "Microphone access is blocked. Allow it in your browser settings to talk, or keep typing.",
|
|
458
|
+
mic_unavailable: "No microphone is available. Check that one is connected and not in use by another app.",
|
|
459
|
+
voice_unavailable: "Voice isn't available right now. You can keep typing.",
|
|
460
|
+
rate_limited: "Too many voice requests. Please wait a moment and try again.",
|
|
461
|
+
network: "Unable to reach the voice service. Check your connection and try again.",
|
|
462
|
+
media_failed: "The voice connection dropped. Try again to continue.",
|
|
463
|
+
server: "Voice could not start. Please try again."
|
|
464
|
+
};
|
|
465
|
+
var MIC_RMS_THRESHOLD = 0.02;
|
|
466
|
+
var REMOTE_RMS_THRESHOLD = 0.01;
|
|
467
|
+
var TICK_MS = 100;
|
|
468
|
+
var ICE_GATHER_TIMEOUT_MS = 4e3;
|
|
469
|
+
var FILLER_WINDOW_MS = 1200;
|
|
470
|
+
var PROCESSING_TIMEOUT_MS = 2e4;
|
|
471
|
+
var DISCONNECT_GRACE_MS = 5e3;
|
|
472
|
+
var MOCK_SDP_MARKER = "a=chatai-mock-session:";
|
|
473
|
+
function browserVoiceMedia() {
|
|
474
|
+
if (typeof window === "undefined" || typeof navigator === "undefined") return null;
|
|
475
|
+
if (window.isSecureContext === false) return null;
|
|
476
|
+
const mediaDevices = navigator.mediaDevices;
|
|
477
|
+
if (!mediaDevices?.getUserMedia || typeof RTCPeerConnection === "undefined") return null;
|
|
478
|
+
return {
|
|
479
|
+
getUserMedia: (constraints) => mediaDevices.getUserMedia(constraints),
|
|
480
|
+
createPeerConnection: () => new RTCPeerConnection(),
|
|
481
|
+
createAudioContext: () => {
|
|
482
|
+
const Ctor = globalThis.AudioContext ?? globalThis.webkitAudioContext;
|
|
483
|
+
try {
|
|
484
|
+
return Ctor ? new Ctor() : null;
|
|
485
|
+
} catch {
|
|
486
|
+
return null;
|
|
487
|
+
}
|
|
488
|
+
},
|
|
489
|
+
createAudioElement: () => {
|
|
490
|
+
if (typeof document === "undefined") return null;
|
|
491
|
+
const audio = document.createElement("audio");
|
|
492
|
+
audio.autoplay = true;
|
|
493
|
+
audio.setAttribute("playsinline", "");
|
|
494
|
+
return audio;
|
|
495
|
+
}
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
function micError(error) {
|
|
499
|
+
const name = error instanceof Error || error instanceof DOMException ? error.name : "";
|
|
500
|
+
if (name === "NotAllowedError" || name === "SecurityError" || name === "PermissionDeniedError") {
|
|
501
|
+
return { code: "mic_denied", message: VOICE_ERROR_MESSAGES.mic_denied };
|
|
502
|
+
}
|
|
503
|
+
if (name === "NotFoundError" || name === "NotReadableError" || name === "OverconstrainedError") {
|
|
504
|
+
return { code: "mic_unavailable", message: VOICE_ERROR_MESSAGES.mic_unavailable };
|
|
505
|
+
}
|
|
506
|
+
return { code: "server", message: VOICE_ERROR_MESSAGES.server };
|
|
507
|
+
}
|
|
508
|
+
async function mintError(response) {
|
|
509
|
+
const body = await response.json().catch(() => ({}));
|
|
510
|
+
if (response.status === 402 || body.reason === "voice_unavailable" || body.reason === "voice_minutes_exhausted" || body.reason === "voice_concurrency_limit") {
|
|
511
|
+
return { code: "voice_unavailable", message: VOICE_ERROR_MESSAGES.voice_unavailable };
|
|
512
|
+
}
|
|
513
|
+
if (response.status === 429) return { code: "rate_limited", message: VOICE_ERROR_MESSAGES.rate_limited };
|
|
514
|
+
if (response.status >= 500) {
|
|
515
|
+
return { code: "voice_unavailable", message: VOICE_ERROR_MESSAGES.voice_unavailable };
|
|
516
|
+
}
|
|
517
|
+
if (response.status === 403 && /voice is not enabled/i.test(body.error ?? "")) {
|
|
518
|
+
return { code: "voice_unavailable", message: VOICE_ERROR_MESSAGES.voice_unavailable };
|
|
519
|
+
}
|
|
520
|
+
return { code: "server", message: body.error?.slice(0, 200) || VOICE_ERROR_MESSAGES.server };
|
|
521
|
+
}
|
|
522
|
+
function waitForIceGathering(pc) {
|
|
523
|
+
if (pc.iceGatheringState === "complete") return Promise.resolve();
|
|
524
|
+
return new Promise((resolve) => {
|
|
525
|
+
const timer = setTimeout(done, ICE_GATHER_TIMEOUT_MS);
|
|
526
|
+
function done() {
|
|
527
|
+
clearTimeout(timer);
|
|
528
|
+
pc.removeEventListener("icegatheringstatechange", check);
|
|
529
|
+
resolve();
|
|
530
|
+
}
|
|
531
|
+
function check() {
|
|
532
|
+
if (pc.iceGatheringState === "complete") done();
|
|
533
|
+
}
|
|
534
|
+
pc.addEventListener("icegatheringstatechange", check);
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
function rms(analyser) {
|
|
538
|
+
if (!analyser) return 0;
|
|
539
|
+
analyser.node.getFloatTimeDomainData(analyser.buffer);
|
|
540
|
+
let sum = 0;
|
|
541
|
+
for (const sample of analyser.buffer) sum += sample * sample;
|
|
542
|
+
return Math.sqrt(sum / analyser.buffer.length);
|
|
543
|
+
}
|
|
544
|
+
function analyse(ctx, stream) {
|
|
545
|
+
if (!ctx) return null;
|
|
546
|
+
try {
|
|
547
|
+
const node = ctx.createAnalyser();
|
|
548
|
+
node.fftSize = 512;
|
|
549
|
+
ctx.createMediaStreamSource(stream).connect(node);
|
|
550
|
+
return { node, buffer: new Float32Array(node.fftSize) };
|
|
551
|
+
} catch {
|
|
552
|
+
return null;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
function createVoiceSession(options) {
|
|
556
|
+
const fetcher = options.fetch;
|
|
557
|
+
const now = options.now ?? (() => Date.now());
|
|
558
|
+
const transcript = createVoiceTranscript(`voice_${now().toString(36)}`);
|
|
559
|
+
const signals = {
|
|
560
|
+
connection: "idle",
|
|
561
|
+
now: now(),
|
|
562
|
+
userVoiceAt: null,
|
|
563
|
+
assistantVoiceAt: null,
|
|
564
|
+
processing: false,
|
|
565
|
+
interruptedAt: null
|
|
566
|
+
};
|
|
567
|
+
let phase = "idle";
|
|
568
|
+
let error;
|
|
569
|
+
let ephemeral;
|
|
570
|
+
let transcriptSaved;
|
|
571
|
+
let conversationId;
|
|
572
|
+
let recording;
|
|
573
|
+
let endReason;
|
|
574
|
+
let mock = false;
|
|
575
|
+
let sessionId = null;
|
|
576
|
+
let lastHeaders = { "Content-Type": "application/json" };
|
|
577
|
+
let stopped = false;
|
|
578
|
+
let mic = null;
|
|
579
|
+
let pc = null;
|
|
580
|
+
let audio = null;
|
|
581
|
+
let audioCtx = null;
|
|
582
|
+
let micAnalyser = null;
|
|
583
|
+
let remoteAnalyser = null;
|
|
584
|
+
let tick = null;
|
|
585
|
+
let disconnectTimer = null;
|
|
586
|
+
let health = null;
|
|
587
|
+
let gate = null;
|
|
588
|
+
let controlMuted = false;
|
|
589
|
+
let processingSince = 0;
|
|
590
|
+
let userWasActive = false;
|
|
591
|
+
const levels = { input: 0, output: 0 };
|
|
592
|
+
const snapshot = () => ({
|
|
593
|
+
connection: signals.connection,
|
|
594
|
+
phase,
|
|
595
|
+
...error ? { error } : {},
|
|
596
|
+
...ephemeral !== void 0 ? { ephemeral } : {},
|
|
597
|
+
...transcriptSaved !== void 0 ? { transcriptSaved } : {},
|
|
598
|
+
...conversationId !== void 0 ? { conversationId } : {},
|
|
599
|
+
...recording !== void 0 ? { recording } : {},
|
|
600
|
+
...endReason ? { endReason, notice: VOICE_END_NOTICES[endReason] } : {},
|
|
601
|
+
mock,
|
|
602
|
+
turns: transcript.turns()
|
|
603
|
+
});
|
|
604
|
+
const emit = () => options.onChange(snapshot());
|
|
605
|
+
const gateClosed = () => gate !== null && !gate.isOpen();
|
|
606
|
+
const applyAudioMute = () => {
|
|
607
|
+
if (audio) audio.muted = controlMuted || gateClosed();
|
|
608
|
+
};
|
|
609
|
+
const refreshPhase = (force = false) => {
|
|
610
|
+
signals.now = now();
|
|
611
|
+
const next = deriveVoicePhase(signals);
|
|
612
|
+
if (next !== phase || force) {
|
|
613
|
+
phase = next;
|
|
614
|
+
emit();
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
const setConnection = (next) => {
|
|
618
|
+
signals.connection = next;
|
|
619
|
+
refreshPhase(true);
|
|
620
|
+
};
|
|
621
|
+
const onTick = () => {
|
|
622
|
+
const at = now();
|
|
623
|
+
if (signals.connection === "reconnecting") {
|
|
624
|
+
levels.input = 0;
|
|
625
|
+
levels.output = 0;
|
|
626
|
+
refreshPhase();
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
levels.input = rms(micAnalyser);
|
|
630
|
+
levels.output = gateClosed() ? 0 : rms(remoteAnalyser);
|
|
631
|
+
if (levels.input > MIC_RMS_THRESHOLD) signals.userVoiceAt = at;
|
|
632
|
+
if (levels.output > REMOTE_RMS_THRESHOLD) signals.assistantVoiceAt = at;
|
|
633
|
+
if (signals.processing && at - processingSince > PROCESSING_TIMEOUT_MS) signals.processing = false;
|
|
634
|
+
signals.now = at;
|
|
635
|
+
const userActive = isUserActive(signals);
|
|
636
|
+
if (detectInterruption({
|
|
637
|
+
userWasActive,
|
|
638
|
+
userIsActive: userActive,
|
|
639
|
+
assistantIsActive: isAssistantActive(signals)
|
|
640
|
+
})) {
|
|
641
|
+
signals.interruptedAt = at;
|
|
642
|
+
}
|
|
643
|
+
userWasActive = userActive;
|
|
644
|
+
refreshPhase();
|
|
645
|
+
};
|
|
646
|
+
const teardownMedia = () => {
|
|
647
|
+
health?.stop();
|
|
648
|
+
health = null;
|
|
649
|
+
gate?.stop();
|
|
650
|
+
gate = null;
|
|
651
|
+
if (tick) clearInterval(tick);
|
|
652
|
+
tick = null;
|
|
653
|
+
if (disconnectTimer) clearTimeout(disconnectTimer);
|
|
654
|
+
disconnectTimer = null;
|
|
655
|
+
pc?.getSenders().forEach((sender) => sender.track?.stop());
|
|
656
|
+
pc?.close();
|
|
657
|
+
pc = null;
|
|
658
|
+
mic?.getTracks().forEach((track) => track.stop());
|
|
659
|
+
mic = null;
|
|
660
|
+
if (audio) {
|
|
661
|
+
audio.pause?.();
|
|
662
|
+
audio.srcObject = null;
|
|
663
|
+
}
|
|
664
|
+
audio = null;
|
|
665
|
+
void audioCtx?.close().catch(() => void 0);
|
|
666
|
+
audioCtx = null;
|
|
667
|
+
micAnalyser = null;
|
|
668
|
+
remoteAnalyser = null;
|
|
669
|
+
levels.input = 0;
|
|
670
|
+
levels.output = 0;
|
|
671
|
+
signals.processing = false;
|
|
672
|
+
signals.userVoiceAt = null;
|
|
673
|
+
signals.assistantVoiceAt = null;
|
|
674
|
+
signals.interruptedAt = null;
|
|
675
|
+
};
|
|
676
|
+
const endServerSession = async (reason, keepalive) => {
|
|
677
|
+
const id = sessionId;
|
|
678
|
+
sessionId = null;
|
|
679
|
+
if (!id) return;
|
|
680
|
+
const headers = keepalive ? lastHeaders : await options.headers().catch(() => lastHeaders);
|
|
681
|
+
const response = await fetcher(
|
|
682
|
+
`${options.apiUrl}/api/v1/voice/sessions/${encodeURIComponent(id)}/end`,
|
|
683
|
+
{
|
|
684
|
+
method: "POST",
|
|
685
|
+
headers,
|
|
686
|
+
body: JSON.stringify({ reason, visitorId: options.visitorId, source: "widget" }),
|
|
687
|
+
keepalive
|
|
688
|
+
}
|
|
689
|
+
).catch(() => null);
|
|
690
|
+
if (!response?.ok) return;
|
|
691
|
+
const body = await response.json().catch(() => null);
|
|
692
|
+
const reported = visitorEndReason(body?.endReason);
|
|
693
|
+
if (reported) {
|
|
694
|
+
endReason = reported;
|
|
695
|
+
if (signals.connection === "failed") {
|
|
696
|
+
error = void 0;
|
|
697
|
+
signals.connection = "ended";
|
|
698
|
+
}
|
|
699
|
+
refreshPhase(true);
|
|
700
|
+
}
|
|
701
|
+
};
|
|
702
|
+
const fail = async (next, reason = "error") => {
|
|
703
|
+
if (stopped) return;
|
|
704
|
+
stopped = true;
|
|
705
|
+
error = next;
|
|
706
|
+
teardownMedia();
|
|
707
|
+
transcript.finish();
|
|
708
|
+
setConnection("failed");
|
|
709
|
+
await endServerSession(reason, false);
|
|
710
|
+
};
|
|
711
|
+
const setMediaMuted = (muted) => {
|
|
712
|
+
mic?.getTracks().forEach((track) => {
|
|
713
|
+
track.enabled = !muted;
|
|
714
|
+
});
|
|
715
|
+
controlMuted = muted;
|
|
716
|
+
applyAudioMute();
|
|
717
|
+
};
|
|
718
|
+
const startPlaybackGate = (id, token) => {
|
|
719
|
+
gate = createPlaybackGate({
|
|
720
|
+
connect: (signal) => fetcher(`${options.apiUrl}/api/v1/voice/sessions/${encodeURIComponent(id)}/gate`, {
|
|
721
|
+
method: "POST",
|
|
722
|
+
headers: { "Content-Type": "application/json" },
|
|
723
|
+
body: JSON.stringify({ token }),
|
|
724
|
+
signal
|
|
725
|
+
}),
|
|
726
|
+
onChange: () => {
|
|
727
|
+
if (stopped) return;
|
|
728
|
+
applyAudioMute();
|
|
729
|
+
refreshPhase();
|
|
730
|
+
},
|
|
731
|
+
...options.playbackGate ? { settings: options.playbackGate } : {}
|
|
732
|
+
});
|
|
733
|
+
applyAudioMute();
|
|
734
|
+
gate.start();
|
|
735
|
+
};
|
|
736
|
+
const terminateForControl = async (reported) => {
|
|
737
|
+
if (stopped) return;
|
|
738
|
+
stopped = true;
|
|
739
|
+
endReason = visitorEndReason(reported) ?? "disconnected";
|
|
740
|
+
teardownMedia();
|
|
741
|
+
transcript.finish();
|
|
742
|
+
setConnection("ended");
|
|
743
|
+
await endServerSession("connection_lost", false);
|
|
744
|
+
};
|
|
745
|
+
const onControlHealth = (next, previous) => {
|
|
746
|
+
if (stopped) return;
|
|
747
|
+
if (next.status === "terminated") {
|
|
748
|
+
void terminateForControl(next.endReason);
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
const muted = isControlMuted(next.status);
|
|
752
|
+
if (muted && !isControlMuted(previous.status)) {
|
|
753
|
+
setMediaMuted(true);
|
|
754
|
+
signals.processing = false;
|
|
755
|
+
signals.userVoiceAt = null;
|
|
756
|
+
signals.assistantVoiceAt = null;
|
|
757
|
+
signals.interruptedAt = null;
|
|
758
|
+
userWasActive = false;
|
|
759
|
+
setConnection("reconnecting");
|
|
760
|
+
} else if (!muted && signals.connection === "reconnecting") {
|
|
761
|
+
setMediaMuted(false);
|
|
762
|
+
setConnection("connected");
|
|
763
|
+
}
|
|
764
|
+
};
|
|
765
|
+
const startControlHealth = (id, token, intervalMs) => {
|
|
766
|
+
const { now: healthNow, ...settings } = options.controlHealth ?? {};
|
|
767
|
+
const send = async (signal) => {
|
|
768
|
+
const response = await fetcher(
|
|
769
|
+
`${options.apiUrl}/api/v1/voice/sessions/${encodeURIComponent(id)}/heartbeat`,
|
|
770
|
+
{
|
|
771
|
+
method: "POST",
|
|
772
|
+
headers: { "Content-Type": "application/json" },
|
|
773
|
+
body: JSON.stringify({ token }),
|
|
774
|
+
signal
|
|
775
|
+
}
|
|
776
|
+
);
|
|
777
|
+
return classifyHeartbeatResponse(response.status, await response.json().catch(() => null));
|
|
778
|
+
};
|
|
779
|
+
health = createControlHealthMonitor({
|
|
780
|
+
send,
|
|
781
|
+
onChange: onControlHealth,
|
|
782
|
+
intervalMs,
|
|
783
|
+
settings,
|
|
784
|
+
...healthNow ? { now: healthNow } : {}
|
|
785
|
+
});
|
|
786
|
+
health.start();
|
|
787
|
+
};
|
|
788
|
+
const mediaLost = () => {
|
|
789
|
+
if (signals.connection === "reconnecting") {
|
|
790
|
+
void terminateForControl("disconnected");
|
|
791
|
+
return;
|
|
792
|
+
}
|
|
793
|
+
void fail({ code: "media_failed", message: VOICE_ERROR_MESSAGES.media_failed }, "connection_lost");
|
|
794
|
+
};
|
|
795
|
+
const handleEvent = (event) => {
|
|
796
|
+
const at = now();
|
|
797
|
+
if (signals.connection === "reconnecting") {
|
|
798
|
+
if (event.type === "session.closed") void terminateForControl("disconnected");
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
switch (event.type) {
|
|
802
|
+
case "session.started":
|
|
803
|
+
if (signals.connection === "connecting") setConnection("connected");
|
|
804
|
+
return;
|
|
805
|
+
case "session.input_transcript.delta":
|
|
806
|
+
gate?.noteInput(event.start_ms);
|
|
807
|
+
signals.userVoiceAt = at;
|
|
808
|
+
transcript.input(event.delta ?? "", event.start_ms ?? at);
|
|
809
|
+
refreshPhase(true);
|
|
810
|
+
return;
|
|
811
|
+
case "session.output_transcript.delta":
|
|
812
|
+
if (gateClosed()) {
|
|
813
|
+
if (!signals.processing) {
|
|
814
|
+
signals.processing = true;
|
|
815
|
+
processingSince = at;
|
|
816
|
+
}
|
|
817
|
+
refreshPhase(true);
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
signals.assistantVoiceAt = at;
|
|
821
|
+
if (gate || signals.processing && at - processingSince >= FILLER_WINDOW_MS) signals.processing = false;
|
|
822
|
+
transcript.output(event.delta ?? "", event.end_ms ?? at);
|
|
823
|
+
refreshPhase(true);
|
|
824
|
+
return;
|
|
825
|
+
case "session.delegation.created":
|
|
826
|
+
signals.processing = true;
|
|
827
|
+
processingSince = at;
|
|
828
|
+
refreshPhase();
|
|
829
|
+
return;
|
|
830
|
+
case "session.closed":
|
|
831
|
+
void session.end("remote_hangup");
|
|
832
|
+
return;
|
|
833
|
+
}
|
|
834
|
+
};
|
|
835
|
+
const session = {
|
|
836
|
+
snapshot,
|
|
837
|
+
/** Mic/speaker levels (0–1-ish RMS) for visual feedback; read on animation frames. */
|
|
838
|
+
levels() {
|
|
839
|
+
return { ...levels };
|
|
840
|
+
},
|
|
841
|
+
async start() {
|
|
842
|
+
if (signals.connection !== "idle") return;
|
|
843
|
+
audio = options.media.createAudioElement();
|
|
844
|
+
audioCtx = options.media.createAudioContext();
|
|
845
|
+
setConnection("requesting_mic");
|
|
846
|
+
try {
|
|
847
|
+
mic = await options.media.getUserMedia({
|
|
848
|
+
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
|
|
849
|
+
});
|
|
850
|
+
} catch (caught) {
|
|
851
|
+
await fail(micError(caught));
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
if (stopped) {
|
|
855
|
+
teardownMedia();
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
setConnection("connecting");
|
|
859
|
+
void audioCtx?.resume?.().catch(() => void 0);
|
|
860
|
+
micAnalyser = analyse(audioCtx, mic);
|
|
861
|
+
let answer;
|
|
862
|
+
try {
|
|
863
|
+
const peer = options.media.createPeerConnection();
|
|
864
|
+
pc = peer;
|
|
865
|
+
mic.getTracks().forEach((track) => peer.addTrack(track, mic));
|
|
866
|
+
peer.addEventListener("track", (event) => {
|
|
867
|
+
const [remote] = event.streams;
|
|
868
|
+
if (!remote || pc !== peer) return;
|
|
869
|
+
if (audio) {
|
|
870
|
+
applyAudioMute();
|
|
871
|
+
audio.srcObject = remote;
|
|
872
|
+
void audio.play?.().catch(() => void 0);
|
|
873
|
+
}
|
|
874
|
+
remoteAnalyser = analyse(audioCtx, remote);
|
|
875
|
+
});
|
|
876
|
+
peer.addEventListener("connectionstatechange", () => {
|
|
877
|
+
if (pc !== peer || stopped) return;
|
|
878
|
+
const state = peer.connectionState;
|
|
879
|
+
if (state === "connected") {
|
|
880
|
+
if (disconnectTimer) clearTimeout(disconnectTimer);
|
|
881
|
+
disconnectTimer = null;
|
|
882
|
+
if (signals.connection === "connecting") setConnection("connected");
|
|
883
|
+
} else if (state === "failed") {
|
|
884
|
+
mediaLost();
|
|
885
|
+
} else if (state === "disconnected" && !disconnectTimer) {
|
|
886
|
+
disconnectTimer = setTimeout(() => {
|
|
887
|
+
disconnectTimer = null;
|
|
888
|
+
if (pc === peer && peer.connectionState !== "connected") mediaLost();
|
|
889
|
+
}, DISCONNECT_GRACE_MS);
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
const events = peer.createDataChannel("oai-events");
|
|
893
|
+
events.addEventListener("message", (message) => {
|
|
894
|
+
try {
|
|
895
|
+
handleEvent(JSON.parse(String(message.data)));
|
|
896
|
+
} catch {
|
|
897
|
+
}
|
|
898
|
+
});
|
|
899
|
+
await peer.setLocalDescription(await peer.createOffer());
|
|
900
|
+
await waitForIceGathering(peer);
|
|
901
|
+
if (stopped) {
|
|
902
|
+
teardownMedia();
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
905
|
+
lastHeaders = await options.headers();
|
|
906
|
+
const response = await fetcher(`${options.apiUrl}/api/v1/voice/sessions`, {
|
|
907
|
+
method: "POST",
|
|
908
|
+
headers: lastHeaders,
|
|
909
|
+
body: JSON.stringify({
|
|
910
|
+
assistantId: options.assistantId,
|
|
911
|
+
sdpOffer: peer.localDescription?.sdp ?? "",
|
|
912
|
+
visitorId: options.visitorId,
|
|
913
|
+
source: "widget",
|
|
914
|
+
...options.conversationId ? { conversationId: options.conversationId } : {},
|
|
915
|
+
...options.history.length ? { history: options.history } : {},
|
|
916
|
+
...options.recordingConsent ? { recordingConsent: true } : {},
|
|
917
|
+
capabilities: ["heartbeat", "playback_gate"]
|
|
918
|
+
})
|
|
919
|
+
});
|
|
920
|
+
if (!response.ok) {
|
|
921
|
+
await fail(await mintError(response));
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
924
|
+
answer = await response.json();
|
|
925
|
+
if (!answer.sessionId || !answer.sdpAnswer) {
|
|
926
|
+
await fail({ code: "server", message: VOICE_ERROR_MESSAGES.server });
|
|
927
|
+
return;
|
|
928
|
+
}
|
|
929
|
+
sessionId = answer.sessionId;
|
|
930
|
+
if (stopped) {
|
|
931
|
+
await endServerSession("close_requested", false);
|
|
932
|
+
return;
|
|
933
|
+
}
|
|
934
|
+
ephemeral = Boolean(answer.ephemeral);
|
|
935
|
+
transcriptSaved = answer.transcriptSaved ?? !ephemeral;
|
|
936
|
+
conversationId = answer.conversationId ?? null;
|
|
937
|
+
recording = Boolean(answer.recording);
|
|
938
|
+
mock = answer.sdpAnswer.includes(MOCK_SDP_MARKER);
|
|
939
|
+
if (answer.playbackGate && answer.controlToken) startPlaybackGate(sessionId, answer.controlToken);
|
|
940
|
+
if (!mock) await peer.setRemoteDescription({ type: "answer", sdp: answer.sdpAnswer });
|
|
941
|
+
} catch (caught) {
|
|
942
|
+
const offline = caught instanceof TypeError;
|
|
943
|
+
await fail(
|
|
944
|
+
offline ? { code: "network", message: VOICE_ERROR_MESSAGES.network } : { code: "server", message: VOICE_ERROR_MESSAGES.server },
|
|
945
|
+
"error"
|
|
946
|
+
);
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
if (stopped) return;
|
|
950
|
+
tick = setInterval(onTick, TICK_MS);
|
|
951
|
+
if (answer.controlToken && sessionId) {
|
|
952
|
+
startControlHealth(sessionId, answer.controlToken, answer.heartbeatIntervalMs);
|
|
953
|
+
}
|
|
954
|
+
if (mock) setConnection("connected");
|
|
955
|
+
else emit();
|
|
956
|
+
},
|
|
957
|
+
/**
|
|
958
|
+
* Stops the mic and media immediately, then finalizes the server session.
|
|
959
|
+
* `keepalive` lets the end request outlive a closing page.
|
|
960
|
+
*/
|
|
961
|
+
async end(reason = "close_requested", opts = {}) {
|
|
962
|
+
if (stopped) {
|
|
963
|
+
if (sessionId) await endServerSession(reason, Boolean(opts.keepalive));
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
stopped = true;
|
|
967
|
+
teardownMedia();
|
|
968
|
+
transcript.finish();
|
|
969
|
+
setConnection(signals.connection === "idle" ? "idle" : "ended");
|
|
970
|
+
await endServerSession(reason, Boolean(opts.keepalive));
|
|
971
|
+
}
|
|
972
|
+
};
|
|
973
|
+
return session;
|
|
974
|
+
}
|
|
975
|
+
|
|
1
976
|
// src/client.ts
|
|
977
|
+
var CLIENT_HISTORY_LIMIT = 12;
|
|
978
|
+
var CLIENT_HISTORY_MAX_CHARS = 1500;
|
|
979
|
+
var IDLE_VOICE = { connection: "idle", phase: "idle" };
|
|
980
|
+
function clientHistory(messages) {
|
|
981
|
+
return messages.filter((message) => message.content.trim() && !message.live && !message.unsaved).slice(-CLIENT_HISTORY_LIMIT).map((message) => ({
|
|
982
|
+
role: message.role,
|
|
983
|
+
content: message.content.slice(0, CLIENT_HISTORY_MAX_CHARS)
|
|
984
|
+
}));
|
|
985
|
+
}
|
|
986
|
+
function voiceMessages(turns, live, unsaved) {
|
|
987
|
+
return turns.map((turn, index) => ({
|
|
988
|
+
id: void 0,
|
|
989
|
+
role: turn.role,
|
|
990
|
+
content: turn.text,
|
|
991
|
+
modality: "voice",
|
|
992
|
+
...turn.interrupted ? { interrupted: true } : {},
|
|
993
|
+
...live && index === turns.length - 1 ? { live: true } : {},
|
|
994
|
+
...unsaved ? { unsaved: true } : {}
|
|
995
|
+
}));
|
|
996
|
+
}
|
|
2
997
|
function resolveApiUrl(value) {
|
|
3
998
|
let url;
|
|
4
999
|
try {
|
|
@@ -85,7 +1080,12 @@ function createWidgetController(options) {
|
|
|
85
1080
|
const visitorKey = `chatai.widget.${options.assistantId}.visitor`;
|
|
86
1081
|
const conversationKey = `chatai.widget.${options.assistantId}.conversation`;
|
|
87
1082
|
const listeners = /* @__PURE__ */ new Set();
|
|
88
|
-
let state = { status: "loading", messages: [] };
|
|
1083
|
+
let state = { status: "loading", messages: [], voice: IDLE_VOICE };
|
|
1084
|
+
const voiceMedia = options.voiceMedia === void 0 ? browserVoiceMedia() : options.voiceMedia;
|
|
1085
|
+
let voiceSession = null;
|
|
1086
|
+
let voiceBase = 0;
|
|
1087
|
+
let voiceUnavailable = false;
|
|
1088
|
+
let removePageHide = null;
|
|
89
1089
|
const origin = () => resolveApiUrl(options.apiUrl);
|
|
90
1090
|
const emit = () => listeners.forEach((listener) => listener(state));
|
|
91
1091
|
const setState = (next) => {
|
|
@@ -128,6 +1128,91 @@ function createWidgetController(options) {
|
|
|
128
1128
|
}
|
|
129
1129
|
return body.signature;
|
|
130
1130
|
};
|
|
1131
|
+
const requestHeaders = async (vid) => {
|
|
1132
|
+
const headers = { "Content-Type": "application/json" };
|
|
1133
|
+
const signature = await fetchSignatureHeader(vid);
|
|
1134
|
+
if (signature) headers["X-ChatAI-Signature"] = signature;
|
|
1135
|
+
return headers;
|
|
1136
|
+
};
|
|
1137
|
+
const applyVoiceSnapshot = (session, snap) => {
|
|
1138
|
+
if (voiceSession !== session) return;
|
|
1139
|
+
const active = isVoiceActive(snap.connection);
|
|
1140
|
+
if (snap.endReason === "voice_unavailable") voiceUnavailable = true;
|
|
1141
|
+
let conversationId = state.conversationId;
|
|
1142
|
+
if (snap.conversationId) {
|
|
1143
|
+
conversationId = snap.conversationId;
|
|
1144
|
+
storage?.setItem(conversationKey, snap.conversationId);
|
|
1145
|
+
}
|
|
1146
|
+
state = {
|
|
1147
|
+
...state,
|
|
1148
|
+
conversationId,
|
|
1149
|
+
messages: [
|
|
1150
|
+
...state.messages.slice(0, voiceBase),
|
|
1151
|
+
...voiceMessages(snap.turns, active, snap.ephemeral === false && snap.transcriptSaved === false)
|
|
1152
|
+
],
|
|
1153
|
+
voice: {
|
|
1154
|
+
connection: snap.connection,
|
|
1155
|
+
phase: snap.phase,
|
|
1156
|
+
...snap.error ? { error: snap.error } : {},
|
|
1157
|
+
...snap.ephemeral !== void 0 ? { ephemeral: snap.ephemeral } : {},
|
|
1158
|
+
...snap.transcriptSaved !== void 0 ? { transcriptSaved: snap.transcriptSaved } : {},
|
|
1159
|
+
...snap.recording ? { recording: true } : {},
|
|
1160
|
+
...snap.notice ? { notice: snap.notice } : {},
|
|
1161
|
+
...snap.mock ? { mock: true } : {}
|
|
1162
|
+
}
|
|
1163
|
+
};
|
|
1164
|
+
if (!active) {
|
|
1165
|
+
removePageHide?.();
|
|
1166
|
+
removePageHide = null;
|
|
1167
|
+
}
|
|
1168
|
+
emit();
|
|
1169
|
+
};
|
|
1170
|
+
const watchPageHide = (session) => {
|
|
1171
|
+
if (typeof window === "undefined") return;
|
|
1172
|
+
const onPageHide = () => void session.end("close_requested", { keepalive: true });
|
|
1173
|
+
window.addEventListener("pagehide", onPageHide);
|
|
1174
|
+
removePageHide = () => window.removeEventListener("pagehide", onPageHide);
|
|
1175
|
+
};
|
|
1176
|
+
const consentRequired = () => Boolean(state.config?.voice?.enabled && state.config.voice.recording?.consentRequired);
|
|
1177
|
+
const beginVoice = async (recordingConsent) => {
|
|
1178
|
+
if (!voiceMedia) {
|
|
1179
|
+
state = {
|
|
1180
|
+
...state,
|
|
1181
|
+
voice: {
|
|
1182
|
+
connection: "failed",
|
|
1183
|
+
phase: "error",
|
|
1184
|
+
error: { code: "unsupported", message: VOICE_ERROR_MESSAGES.unsupported }
|
|
1185
|
+
}
|
|
1186
|
+
};
|
|
1187
|
+
emit();
|
|
1188
|
+
return;
|
|
1189
|
+
}
|
|
1190
|
+
let apiUrl;
|
|
1191
|
+
try {
|
|
1192
|
+
apiUrl = origin();
|
|
1193
|
+
} catch (error) {
|
|
1194
|
+
setState({ ...state, error: humanizeNetworkError(error, "Unable to start voice.") });
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
const vid = visitorId();
|
|
1198
|
+
voiceBase = state.messages.length;
|
|
1199
|
+
const session = createVoiceSession({
|
|
1200
|
+
apiUrl,
|
|
1201
|
+
assistantId: options.assistantId,
|
|
1202
|
+
visitorId: vid,
|
|
1203
|
+
conversationId: state.conversationId,
|
|
1204
|
+
history: clientHistory(state.messages),
|
|
1205
|
+
recordingConsent,
|
|
1206
|
+
fetch: fetcher,
|
|
1207
|
+
media: voiceMedia,
|
|
1208
|
+
headers: () => requestHeaders(vid),
|
|
1209
|
+
onChange: (snap) => applyVoiceSnapshot(session, snap)
|
|
1210
|
+
});
|
|
1211
|
+
voiceSession = session;
|
|
1212
|
+
state = { ...state, error: void 0 };
|
|
1213
|
+
watchPageHide(session);
|
|
1214
|
+
await session.start();
|
|
1215
|
+
};
|
|
131
1216
|
return {
|
|
132
1217
|
getState() {
|
|
133
1218
|
return state;
|
|
@@ -161,9 +1246,10 @@ function createWidgetController(options) {
|
|
|
161
1246
|
},
|
|
162
1247
|
async send(message) {
|
|
163
1248
|
const content = message.trim();
|
|
164
|
-
if (!content || state.status === "streaming") return;
|
|
1249
|
+
if (!content || state.status === "streaming" || isVoiceActive(state.voice.connection)) return;
|
|
165
1250
|
const userMessage = { role: "user", content };
|
|
166
1251
|
let assistantMessage = { role: "assistant", content: "" };
|
|
1252
|
+
const history = clientHistory(state.messages);
|
|
167
1253
|
setState({
|
|
168
1254
|
...state,
|
|
169
1255
|
status: "streaming",
|
|
@@ -173,11 +1259,7 @@ function createWidgetController(options) {
|
|
|
173
1259
|
try {
|
|
174
1260
|
const apiUrl = origin();
|
|
175
1261
|
const vid = visitorId();
|
|
176
|
-
const headers =
|
|
177
|
-
const signature = await fetchSignatureHeader(vid);
|
|
178
|
-
if (signature) {
|
|
179
|
-
headers["X-ChatAI-Signature"] = signature;
|
|
180
|
-
}
|
|
1262
|
+
const headers = await requestHeaders(vid);
|
|
181
1263
|
const response = await fetcher(`${apiUrl}/api/v1/chat`, {
|
|
182
1264
|
method: "POST",
|
|
183
1265
|
headers,
|
|
@@ -186,7 +1268,9 @@ function createWidgetController(options) {
|
|
|
186
1268
|
message: content,
|
|
187
1269
|
conversationId: state.conversationId,
|
|
188
1270
|
visitorId: vid,
|
|
189
|
-
source: "widget"
|
|
1271
|
+
source: "widget",
|
|
1272
|
+
...history.length ? { history } : {},
|
|
1273
|
+
...voiceUnavailable ? { voiceUnavailable: true } : {}
|
|
190
1274
|
})
|
|
191
1275
|
});
|
|
192
1276
|
if (!response.ok || !response.body) {
|
|
@@ -230,11 +1314,7 @@ function createWidgetController(options) {
|
|
|
230
1314
|
try {
|
|
231
1315
|
const apiUrl = origin();
|
|
232
1316
|
const vid = visitorId();
|
|
233
|
-
const headers =
|
|
234
|
-
const signature = await fetchSignatureHeader(vid);
|
|
235
|
-
if (signature) {
|
|
236
|
-
headers["X-ChatAI-Signature"] = signature;
|
|
237
|
-
}
|
|
1317
|
+
const headers = await requestHeaders(vid);
|
|
238
1318
|
const response = await fetcher(`${apiUrl}/api/v1/feedback`, {
|
|
239
1319
|
method: "POST",
|
|
240
1320
|
headers,
|
|
@@ -259,14 +1339,82 @@ function createWidgetController(options) {
|
|
|
259
1339
|
throw error;
|
|
260
1340
|
}
|
|
261
1341
|
},
|
|
1342
|
+
/** Voice can be offered: enabled for this assistant and supported by this browser. */
|
|
1343
|
+
voiceAvailable() {
|
|
1344
|
+
return Boolean(state.config?.voice?.enabled);
|
|
1345
|
+
},
|
|
1346
|
+
voiceSupported() {
|
|
1347
|
+
return voiceMedia !== null;
|
|
1348
|
+
},
|
|
1349
|
+
/** Voice sessions are recorded, so starting one shows the disclosure first. */
|
|
1350
|
+
voiceConsentRequired() {
|
|
1351
|
+
return consentRequired();
|
|
1352
|
+
},
|
|
1353
|
+
/**
|
|
1354
|
+
* Start Voice. When sessions are recorded this only opens the disclosure;
|
|
1355
|
+
* the mic and the session wait for `acceptRecordingConsent()`.
|
|
1356
|
+
*/
|
|
1357
|
+
async startVoice() {
|
|
1358
|
+
if (!state.config?.voice?.enabled) return;
|
|
1359
|
+
if (isVoiceActive(state.voice.connection) || state.status === "streaming") return;
|
|
1360
|
+
if (consentRequired()) {
|
|
1361
|
+
setState({ ...state, error: void 0, voice: { ...IDLE_VOICE, consentPending: true } });
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1364
|
+
await beginVoice(false);
|
|
1365
|
+
},
|
|
1366
|
+
async acceptRecordingConsent() {
|
|
1367
|
+
if (!state.voice.consentPending) return;
|
|
1368
|
+
state = { ...state, voice: IDLE_VOICE };
|
|
1369
|
+
await beginVoice(true);
|
|
1370
|
+
},
|
|
1371
|
+
/** Back to text chat; nothing was requested or started. */
|
|
1372
|
+
declineRecordingConsent() {
|
|
1373
|
+
if (!state.voice.consentPending) return;
|
|
1374
|
+
setState({ ...state, voice: IDLE_VOICE });
|
|
1375
|
+
},
|
|
1376
|
+
async endVoice() {
|
|
1377
|
+
await voiceSession?.end("close_requested");
|
|
1378
|
+
},
|
|
1379
|
+
/** Live mic/speaker levels for the waveform; zeros when Voice is off. */
|
|
1380
|
+
voiceLevels() {
|
|
1381
|
+
return voiceSession?.levels() ?? { input: 0, output: 0 };
|
|
1382
|
+
},
|
|
1383
|
+
/** Clears a Voice error or end notice so the visitor can return to text or try again. */
|
|
1384
|
+
dismissVoiceError() {
|
|
1385
|
+
if (isVoiceActive(state.voice.connection)) return;
|
|
1386
|
+
setState({ ...state, voice: IDLE_VOICE });
|
|
1387
|
+
},
|
|
262
1388
|
destroy() {
|
|
1389
|
+
const session = voiceSession;
|
|
1390
|
+
voiceSession = null;
|
|
1391
|
+
removePageHide?.();
|
|
1392
|
+
removePageHide = null;
|
|
1393
|
+
void session?.end("close_requested", { keepalive: true });
|
|
263
1394
|
listeners.clear();
|
|
264
1395
|
}
|
|
265
1396
|
};
|
|
266
1397
|
}
|
|
267
1398
|
export {
|
|
1399
|
+
BARGE_IN_MIN_WORDS,
|
|
1400
|
+
CONTROL_HEALTH_DEFAULTS,
|
|
1401
|
+
VOICE_END_NOTICES,
|
|
1402
|
+
VOICE_ERROR_MESSAGES,
|
|
1403
|
+
VOICE_PHASE_LABELS,
|
|
1404
|
+
browserVoiceMedia,
|
|
1405
|
+
classifyHeartbeatResponse,
|
|
1406
|
+
clientHistory,
|
|
1407
|
+
createControlHealthMonitor,
|
|
268
1408
|
createSseParser,
|
|
1409
|
+
createVoiceSession,
|
|
1410
|
+
createVoiceTranscript,
|
|
269
1411
|
createWidgetController,
|
|
270
|
-
|
|
1412
|
+
deriveVoicePhase,
|
|
1413
|
+
initialControlHealth,
|
|
1414
|
+
isControlMuted,
|
|
1415
|
+
isVoiceActive,
|
|
1416
|
+
reduceControlHealth,
|
|
1417
|
+
resolveApiUrl,
|
|
1418
|
+
visitorEndReason
|
|
271
1419
|
};
|
|
272
1420
|
//# sourceMappingURL=index.js.map
|