@runtypelabs/voice 0.2.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -4
- package/dist/index.cjs +257 -25
- package/dist/index.d.cts +25 -2
- package/dist/index.d.ts +25 -2
- package/dist/index.js +257 -25
- package/dist/persona.cjs +319 -30
- package/dist/persona.d.cts +23 -4
- package/dist/persona.d.ts +23 -4
- package/dist/persona.js +319 -30
- package/dist/react.cjs +320 -32
- package/dist/react.d.cts +7 -4
- package/dist/react.d.ts +7 -4
- package/dist/react.js +321 -33
- package/dist/types-Dkh4LxEA.d.cts +100 -0
- package/dist/types-Dkh4LxEA.d.ts +100 -0
- package/package.json +1 -1
- package/dist/types-9SfA7oHc.d.cts +0 -40
- package/dist/types-9SfA7oHc.d.ts +0 -40
package/dist/react.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use client";
|
|
2
2
|
|
|
3
3
|
// src/react.ts
|
|
4
|
-
import { useEffect,
|
|
4
|
+
import { useEffect, useRef, useState, useSyncExternalStore } from "react";
|
|
5
5
|
|
|
6
6
|
// src/pcm-player.ts
|
|
7
7
|
var PCM_SAMPLE_RATE = 24e3;
|
|
@@ -14,13 +14,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
14
14
|
this.readOffset = 0
|
|
15
15
|
this.buffered = 0
|
|
16
16
|
this.waiting = true
|
|
17
|
-
// INVARIANT:
|
|
17
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
18
18
|
this.eosSeen = false
|
|
19
|
+
this.continuous = false
|
|
19
20
|
this.revision = 0
|
|
20
21
|
this.port.onmessage = (e) => {
|
|
21
22
|
const msg = e.data
|
|
22
23
|
this.revision = msg.revision
|
|
23
|
-
if (msg.type === '
|
|
24
|
+
if (msg.type === 'continuous') {
|
|
25
|
+
this.continuous = msg.enabled
|
|
26
|
+
} else if (msg.type === 'push') {
|
|
24
27
|
this.eosSeen = false
|
|
25
28
|
this.chunks.push(msg.samples)
|
|
26
29
|
this.buffered += msg.samples.length
|
|
@@ -60,7 +63,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
60
63
|
}
|
|
61
64
|
if (this.buffered === 0) {
|
|
62
65
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
63
|
-
if (this.eosSeen) {
|
|
66
|
+
if (this.eosSeen || this.continuous) {
|
|
64
67
|
this.eosSeen = false
|
|
65
68
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
66
69
|
}
|
|
@@ -104,6 +107,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
104
107
|
if (samples.length === 0) return;
|
|
105
108
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
106
109
|
},
|
|
110
|
+
setContinuousMode(enabled) {
|
|
111
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
112
|
+
},
|
|
107
113
|
endOfStream() {
|
|
108
114
|
node.port.postMessage({ type: "eos", revision });
|
|
109
115
|
},
|
|
@@ -148,9 +154,58 @@ function initialSnapshot() {
|
|
|
148
154
|
error: null,
|
|
149
155
|
errorDetails: void 0,
|
|
150
156
|
interruptionMode: "none",
|
|
151
|
-
canCancel: false
|
|
157
|
+
canCancel: false,
|
|
158
|
+
artifacts: [],
|
|
159
|
+
session: null
|
|
152
160
|
};
|
|
153
161
|
}
|
|
162
|
+
function readString(value) {
|
|
163
|
+
return typeof value === "string" && value ? value : void 0;
|
|
164
|
+
}
|
|
165
|
+
function readArtifactFile(value) {
|
|
166
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
167
|
+
const file = value;
|
|
168
|
+
const path = readString(file.path);
|
|
169
|
+
const mimeType = readString(file.mimeType);
|
|
170
|
+
if (!path || !mimeType) return void 0;
|
|
171
|
+
const language = readString(file.language);
|
|
172
|
+
return { path, mimeType, ...language ? { language } : {} };
|
|
173
|
+
}
|
|
174
|
+
function applyArtifactFrame(existing, frame, identity) {
|
|
175
|
+
if (frame.type === "artifact_start") {
|
|
176
|
+
const title = readString(frame.title);
|
|
177
|
+
const component = readString(frame.component);
|
|
178
|
+
const file = readArtifactFile(frame.file);
|
|
179
|
+
return {
|
|
180
|
+
...identity,
|
|
181
|
+
artifactType: frame.artifactType === "component" ? "component" : "markdown",
|
|
182
|
+
...title ? { title } : {},
|
|
183
|
+
...file ? { file } : {},
|
|
184
|
+
...component ? { component } : {},
|
|
185
|
+
content: "",
|
|
186
|
+
status: "streaming"
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
if (!existing) return void 0;
|
|
190
|
+
if (frame.type === "artifact_delta") {
|
|
191
|
+
const delta = typeof frame.delta === "string" ? frame.delta : "";
|
|
192
|
+
return delta ? { ...existing, content: existing.content + delta } : void 0;
|
|
193
|
+
}
|
|
194
|
+
if (frame.type === "artifact_update") {
|
|
195
|
+
const component = readString(frame.component) ?? existing.component;
|
|
196
|
+
const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
|
|
197
|
+
return {
|
|
198
|
+
...existing,
|
|
199
|
+
...component ? { component } : {},
|
|
200
|
+
...props ? { props } : {}
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
|
|
204
|
+
return void 0;
|
|
205
|
+
}
|
|
206
|
+
function readTurnId(msg) {
|
|
207
|
+
return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
|
|
208
|
+
}
|
|
154
209
|
var VoiceClient = class {
|
|
155
210
|
constructor(options) {
|
|
156
211
|
this.options = options;
|
|
@@ -171,6 +226,17 @@ var VoiceClient = class {
|
|
|
171
226
|
stoppedResponse = false;
|
|
172
227
|
responseEnded = true;
|
|
173
228
|
hasPendingAudio = false;
|
|
229
|
+
/**
|
|
230
|
+
* The assistant entry built from clause captions for the turn being spoken. The
|
|
231
|
+
* authoritative transcript replaces it; a stop commits it, because it captions
|
|
232
|
+
* audio the caller already heard.
|
|
233
|
+
*/
|
|
234
|
+
pendingAssistant = null;
|
|
235
|
+
/** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
|
|
236
|
+
committedAssistantTurnId = null;
|
|
237
|
+
activeTurnId = null;
|
|
238
|
+
suppressedArtifactTurnId = null;
|
|
239
|
+
fullDuplex = false;
|
|
174
240
|
getSnapshot = () => this.snapshot;
|
|
175
241
|
subscribe = (listener) => {
|
|
176
242
|
this.listeners.add(listener);
|
|
@@ -192,7 +258,7 @@ var VoiceClient = class {
|
|
|
192
258
|
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
193
259
|
this.cleanup();
|
|
194
260
|
const generation = this.generation;
|
|
195
|
-
this.update({ ...initialSnapshot(), status: "connecting" });
|
|
261
|
+
this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
|
|
196
262
|
try {
|
|
197
263
|
const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
|
|
198
264
|
if (generation !== this.generation) return;
|
|
@@ -207,10 +273,26 @@ var VoiceClient = class {
|
|
|
207
273
|
url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
|
|
208
274
|
url.search = "";
|
|
209
275
|
url.searchParams.set("voiceProtocol", "runtype-browser-v1");
|
|
276
|
+
const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
|
|
277
|
+
url.searchParams.set("clientCapabilities", capabilities);
|
|
278
|
+
const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
|
|
279
|
+
if (sessionId) url.searchParams.set("sessionId", sessionId);
|
|
280
|
+
const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
|
|
281
|
+
if (generation !== this.generation) return;
|
|
282
|
+
if (this.options.visitorToken !== void 0 && !visitorToken) {
|
|
283
|
+
throw new Error(
|
|
284
|
+
"Voice visitor credential unavailable. Initialize the client session first."
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
288
|
+
throw new Error("Invalid voice visitor credential.");
|
|
289
|
+
}
|
|
290
|
+
if (this.options.fullDuplex !== false)
|
|
291
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
210
292
|
url.hash = "";
|
|
211
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
293
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
212
294
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
213
|
-
});
|
|
295
|
+
}));
|
|
214
296
|
if (generation !== this.generation) {
|
|
215
297
|
stream.getTracks().forEach((track) => track.stop());
|
|
216
298
|
return;
|
|
@@ -230,7 +312,11 @@ var VoiceClient = class {
|
|
|
230
312
|
return;
|
|
231
313
|
}
|
|
232
314
|
this.player = player;
|
|
233
|
-
const socket = new WebSocket(url.toString(), [
|
|
315
|
+
const socket = new WebSocket(url.toString(), [
|
|
316
|
+
"runtype.bearer",
|
|
317
|
+
token,
|
|
318
|
+
...visitorToken ? [visitorToken] : []
|
|
319
|
+
]);
|
|
234
320
|
socket.binaryType = "arraybuffer";
|
|
235
321
|
this.socket = socket;
|
|
236
322
|
socket.onopen = () => {
|
|
@@ -261,6 +347,11 @@ var VoiceClient = class {
|
|
|
261
347
|
this.cleanup();
|
|
262
348
|
this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
|
|
263
349
|
};
|
|
350
|
+
/** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
|
|
351
|
+
resetSession = () => {
|
|
352
|
+
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
353
|
+
this.update({ session: null });
|
|
354
|
+
};
|
|
264
355
|
toggleMute = () => {
|
|
265
356
|
this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
|
|
266
357
|
};
|
|
@@ -275,9 +366,11 @@ var VoiceClient = class {
|
|
|
275
366
|
this.clearPlayback();
|
|
276
367
|
if (this.snapshot.interruptionMode === "none") {
|
|
277
368
|
this.stoppedResponse = !this.responseEnded;
|
|
369
|
+
this.commitClauseCaptions();
|
|
278
370
|
this.setStatus("listening");
|
|
279
371
|
return;
|
|
280
372
|
}
|
|
373
|
+
this.suppressedArtifactTurnId = this.activeTurnId;
|
|
281
374
|
this.awaitingClear = true;
|
|
282
375
|
this.setStatus("listening");
|
|
283
376
|
this.socket.send(JSON.stringify({ type: "cancel" }));
|
|
@@ -294,6 +387,7 @@ var VoiceClient = class {
|
|
|
294
387
|
});
|
|
295
388
|
}
|
|
296
389
|
cleanup() {
|
|
390
|
+
this.fullDuplex = false;
|
|
297
391
|
this.generation += 1;
|
|
298
392
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
299
393
|
this.processor?.disconnect();
|
|
@@ -319,6 +413,82 @@ var VoiceClient = class {
|
|
|
319
413
|
this.awaitingClear = false;
|
|
320
414
|
this.stoppedResponse = false;
|
|
321
415
|
this.responseEnded = true;
|
|
416
|
+
this.commitClauseCaptions();
|
|
417
|
+
this.committedAssistantTurnId = null;
|
|
418
|
+
this.activeTurnId = null;
|
|
419
|
+
this.suppressedArtifactTurnId = null;
|
|
420
|
+
const settledArtifacts = this.snapshot.artifacts.filter(
|
|
421
|
+
(artifact) => artifact.status === "complete"
|
|
422
|
+
);
|
|
423
|
+
if (settledArtifacts.length !== this.snapshot.artifacts.length)
|
|
424
|
+
this.update({ artifacts: settledArtifacts });
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Folds an `artifact` message into the snapshot. Frames of a turn the caller
|
|
428
|
+
* cancelled are dropped; a playback-only stop leaves the turn running, so its
|
|
429
|
+
* artifacts keep arriving and are kept.
|
|
430
|
+
*/
|
|
431
|
+
applyArtifactMessage(msg) {
|
|
432
|
+
if (this.awaitingClear) return;
|
|
433
|
+
const frame = msg.event;
|
|
434
|
+
if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
|
|
435
|
+
const event = frame;
|
|
436
|
+
const id = readString(event.id);
|
|
437
|
+
if (!id) return;
|
|
438
|
+
const turnId = readTurnId(msg);
|
|
439
|
+
if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
|
|
440
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
441
|
+
const artifacts = this.snapshot.artifacts;
|
|
442
|
+
const index = artifacts.findIndex((artifact) => artifact.id === id);
|
|
443
|
+
const executionId = readString(msg.executionId);
|
|
444
|
+
const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
|
|
445
|
+
id,
|
|
446
|
+
turnId,
|
|
447
|
+
...executionId ? { executionId } : {}
|
|
448
|
+
});
|
|
449
|
+
if (!next) return;
|
|
450
|
+
this.update({
|
|
451
|
+
artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
/** Settles the spoken clauses in place and ends the turn's claim on them. */
|
|
455
|
+
commitClauseCaptions() {
|
|
456
|
+
this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
|
|
457
|
+
const settled = this.pendingAssistant !== null;
|
|
458
|
+
this.pendingAssistant = null;
|
|
459
|
+
if (!settled) return;
|
|
460
|
+
const transcript = this.snapshot.transcript;
|
|
461
|
+
const last = transcript[transcript.length - 1];
|
|
462
|
+
if (last?.role !== "assistant" || last.partial !== true) return;
|
|
463
|
+
const { partial: _partial, ...committed } = last;
|
|
464
|
+
this.update({ transcript: [...transcript.slice(0, -1), committed] });
|
|
465
|
+
}
|
|
466
|
+
/** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
|
|
467
|
+
appendClauseCaption(text, turnId) {
|
|
468
|
+
const transcript = this.snapshot.transcript;
|
|
469
|
+
const last = transcript[transcript.length - 1];
|
|
470
|
+
if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
|
|
471
|
+
return {
|
|
472
|
+
interimTranscript: null,
|
|
473
|
+
transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
|
|
474
|
+
status: "speaking"
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
this.pendingAssistant = { turnId };
|
|
478
|
+
return {
|
|
479
|
+
interimTranscript: null,
|
|
480
|
+
transcript: [
|
|
481
|
+
...transcript,
|
|
482
|
+
{
|
|
483
|
+
role: "assistant",
|
|
484
|
+
content: text,
|
|
485
|
+
timestamp: Date.now(),
|
|
486
|
+
partial: true,
|
|
487
|
+
...turnId ? { turnId } : {}
|
|
488
|
+
}
|
|
489
|
+
],
|
|
490
|
+
status: "speaking"
|
|
491
|
+
};
|
|
322
492
|
}
|
|
323
493
|
clearPlayback() {
|
|
324
494
|
this.playbackRevision += 1;
|
|
@@ -353,32 +523,89 @@ var VoiceClient = class {
|
|
|
353
523
|
return;
|
|
354
524
|
}
|
|
355
525
|
switch (msg.type) {
|
|
356
|
-
case "session_config":
|
|
526
|
+
case "session_config": {
|
|
527
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
528
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
357
529
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
358
530
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
359
531
|
}
|
|
532
|
+
const sessionId = readString(msg.sessionId);
|
|
533
|
+
const conversationId = readString(msg.conversationId);
|
|
534
|
+
if (sessionId && conversationId) {
|
|
535
|
+
const session = { sessionId, conversationId };
|
|
536
|
+
this.update({ session });
|
|
537
|
+
this.options.onSession?.(session);
|
|
538
|
+
}
|
|
539
|
+
break;
|
|
540
|
+
}
|
|
541
|
+
case "artifact":
|
|
542
|
+
this.applyArtifactMessage(msg);
|
|
543
|
+
break;
|
|
544
|
+
case "transcript_update": {
|
|
545
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
546
|
+
break;
|
|
547
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
548
|
+
const entries = [...this.snapshot.transcript];
|
|
549
|
+
const index = entries.findIndex(
|
|
550
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
551
|
+
);
|
|
552
|
+
const entry = {
|
|
553
|
+
role: msg.role,
|
|
554
|
+
content: msg.text,
|
|
555
|
+
turnId: msg.turnId,
|
|
556
|
+
isFinal: msg.final === true,
|
|
557
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
558
|
+
};
|
|
559
|
+
if (index >= 0) entries[index] = entry;
|
|
560
|
+
else entries.push(entry);
|
|
561
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
564
|
+
case "delegation_started":
|
|
565
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
566
|
+
break;
|
|
567
|
+
case "delegation_completed":
|
|
568
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
360
569
|
break;
|
|
361
570
|
case "transcript_interim":
|
|
362
571
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
363
572
|
break;
|
|
364
|
-
case "
|
|
573
|
+
case "transcript_partial": {
|
|
574
|
+
if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
|
|
575
|
+
this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
|
|
576
|
+
if (this.awaitingClear || this.stoppedResponse) break;
|
|
577
|
+
this.responseEnded = false;
|
|
578
|
+
this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
|
|
579
|
+
break;
|
|
580
|
+
}
|
|
581
|
+
case "transcript_final": {
|
|
365
582
|
if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
|
|
366
|
-
|
|
583
|
+
const role = msg.role;
|
|
584
|
+
const text = msg.text;
|
|
585
|
+
if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
|
|
586
|
+
const turnId = readTurnId(msg);
|
|
587
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
588
|
+
if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
|
|
589
|
+
break;
|
|
367
590
|
this.responseEnded = false;
|
|
591
|
+
const transcript = this.snapshot.transcript;
|
|
592
|
+
const last = transcript[transcript.length - 1];
|
|
593
|
+
const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
|
|
594
|
+
this.pendingAssistant = null;
|
|
595
|
+
const entry = {
|
|
596
|
+
role,
|
|
597
|
+
content: text,
|
|
598
|
+
// INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
|
|
599
|
+
timestamp: supersedes && last ? last.timestamp : Date.now(),
|
|
600
|
+
...turnId ? { turnId } : {}
|
|
601
|
+
};
|
|
368
602
|
this.update({
|
|
369
603
|
interimTranscript: null,
|
|
370
|
-
transcript: [
|
|
371
|
-
|
|
372
|
-
{
|
|
373
|
-
role: msg.role,
|
|
374
|
-
content: msg.text,
|
|
375
|
-
timestamp: Date.now(),
|
|
376
|
-
...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
|
|
377
|
-
}
|
|
378
|
-
],
|
|
379
|
-
status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
|
|
604
|
+
transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
|
|
605
|
+
status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
|
|
380
606
|
});
|
|
381
607
|
break;
|
|
608
|
+
}
|
|
382
609
|
case "audio_end": {
|
|
383
610
|
this.responseEnded = true;
|
|
384
611
|
if (this.stoppedResponse) {
|
|
@@ -395,10 +622,12 @@ var VoiceClient = class {
|
|
|
395
622
|
break;
|
|
396
623
|
}
|
|
397
624
|
case "audio_clear":
|
|
625
|
+
this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
|
|
398
626
|
this.clearPlayback();
|
|
399
627
|
this.awaitingClear = false;
|
|
400
628
|
this.stoppedResponse = false;
|
|
401
629
|
this.responseEnded = true;
|
|
630
|
+
this.commitClauseCaptions();
|
|
402
631
|
this.setStatus("listening");
|
|
403
632
|
break;
|
|
404
633
|
case "metrics": {
|
|
@@ -411,8 +640,11 @@ var VoiceClient = class {
|
|
|
411
640
|
// @snake-case-ok: Existing voice wire contract.
|
|
412
641
|
firstAudioMs: number(msg.first_audio_ms),
|
|
413
642
|
// @snake-case-ok: Existing voice wire contract.
|
|
414
|
-
totalMs: number(msg.total_ms)
|
|
643
|
+
totalMs: number(msg.total_ms),
|
|
644
|
+
// @snake-case-ok: Existing voice wire contract.
|
|
645
|
+
firstSynthesisMs: number(msg.first_synthesis_ms),
|
|
415
646
|
// @snake-case-ok: Existing voice wire contract.
|
|
647
|
+
incremental: msg.incremental === true
|
|
416
648
|
}
|
|
417
649
|
});
|
|
418
650
|
break;
|
|
@@ -444,14 +676,14 @@ var VoiceClient = class {
|
|
|
444
676
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
445
677
|
this.processor = processor;
|
|
446
678
|
processor.onaudioprocess = (event) => {
|
|
447
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
679
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
448
680
|
const input = event.inputBuffer.getChannelData(0);
|
|
449
681
|
let sum = 0;
|
|
450
682
|
for (const sample of input) sum += sample * sample;
|
|
451
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
683
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
452
684
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
453
685
|
const pcm = new Int16Array(input.length);
|
|
454
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
686
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
455
687
|
socket.send(pcm.buffer);
|
|
456
688
|
return;
|
|
457
689
|
}
|
|
@@ -467,25 +699,81 @@ var VoiceClient = class {
|
|
|
467
699
|
};
|
|
468
700
|
|
|
469
701
|
// src/react.ts
|
|
470
|
-
function useVoiceClient({
|
|
702
|
+
function useVoiceClient({
|
|
703
|
+
agentId,
|
|
704
|
+
apiUrl,
|
|
705
|
+
clientToken,
|
|
706
|
+
artifacts,
|
|
707
|
+
sessionId,
|
|
708
|
+
visitorToken,
|
|
709
|
+
onSession,
|
|
710
|
+
fullDuplex,
|
|
711
|
+
audioSource
|
|
712
|
+
}) {
|
|
713
|
+
const sessionRef = useRef(sessionId);
|
|
714
|
+
useEffect(() => {
|
|
715
|
+
sessionRef.current = sessionId;
|
|
716
|
+
}, [sessionId]);
|
|
471
717
|
const tokenRef = useRef(clientToken);
|
|
718
|
+
const visitorRef = useRef(visitorToken);
|
|
719
|
+
const sessionCallbackRef = useRef(onSession);
|
|
720
|
+
const audioSourceRef = useRef(audioSource);
|
|
721
|
+
const hasAudioSource = audioSource !== void 0;
|
|
722
|
+
useEffect(() => {
|
|
723
|
+
visitorRef.current = visitorToken;
|
|
724
|
+
sessionCallbackRef.current = onSession;
|
|
725
|
+
}, [visitorToken, onSession]);
|
|
726
|
+
useEffect(() => {
|
|
727
|
+
audioSourceRef.current = audioSource;
|
|
728
|
+
}, [audioSource]);
|
|
472
729
|
useEffect(() => {
|
|
473
730
|
tokenRef.current = clientToken;
|
|
474
731
|
}, [clientToken]);
|
|
475
|
-
const
|
|
476
|
-
|
|
732
|
+
const createClient = () => new VoiceClient({
|
|
733
|
+
agentId,
|
|
734
|
+
apiUrl,
|
|
735
|
+
artifacts,
|
|
736
|
+
fullDuplex,
|
|
737
|
+
...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
|
|
738
|
+
get sessionId() {
|
|
739
|
+
return sessionRef.current;
|
|
740
|
+
},
|
|
741
|
+
get visitorToken() {
|
|
742
|
+
return visitorRef.current;
|
|
743
|
+
},
|
|
744
|
+
onSession: (session) => sessionCallbackRef.current?.(session),
|
|
745
|
+
clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
|
|
746
|
+
});
|
|
747
|
+
const [binding, setBinding] = useState(() => ({
|
|
748
|
+
agentId,
|
|
749
|
+
apiUrl,
|
|
750
|
+
artifacts,
|
|
751
|
+
fullDuplex,
|
|
752
|
+
hasAudioSource,
|
|
753
|
+
sessionId,
|
|
754
|
+
client: createClient()
|
|
755
|
+
}));
|
|
756
|
+
const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
|
|
757
|
+
if (configurationChanged || binding.sessionId !== sessionId) {
|
|
758
|
+
const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
|
|
759
|
+
setBinding({
|
|
477
760
|
agentId,
|
|
478
761
|
apiUrl,
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
762
|
+
artifacts,
|
|
763
|
+
fullDuplex,
|
|
764
|
+
hasAudioSource,
|
|
765
|
+
sessionId,
|
|
766
|
+
client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
const client = binding.client;
|
|
483
770
|
const snapshot = useSyncExternalStore(client.subscribe, client.getSnapshot, client.getSnapshot);
|
|
484
771
|
useEffect(() => () => client.endCall(), [client]);
|
|
485
772
|
return {
|
|
486
773
|
...snapshot,
|
|
487
774
|
startCall: client.startCall,
|
|
488
775
|
endCall: client.endCall,
|
|
776
|
+
resetSession: client.resetSession,
|
|
489
777
|
toggleMute: client.toggleMute,
|
|
490
778
|
cancelResponse: client.cancelResponse
|
|
491
779
|
};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
|
|
2
|
+
type InterruptionMode = 'none' | 'cancel' | 'barge-in';
|
|
3
|
+
interface TranscriptEntry {
|
|
4
|
+
readonly role: 'user' | 'assistant';
|
|
5
|
+
readonly content: string;
|
|
6
|
+
readonly timestamp: number;
|
|
7
|
+
readonly isFinal?: boolean;
|
|
8
|
+
readonly turnId?: string;
|
|
9
|
+
/**
|
|
10
|
+
* The reply is still being spoken, so `content` holds the clauses committed so
|
|
11
|
+
* far. A later snapshot replaces this entry with the authoritative transcript,
|
|
12
|
+
* or clears the flag in place when the caller stops the reply.
|
|
13
|
+
*/
|
|
14
|
+
readonly partial?: boolean;
|
|
15
|
+
}
|
|
16
|
+
interface VoiceMetrics {
|
|
17
|
+
readonly llmMs?: number;
|
|
18
|
+
readonly ttsMs?: number;
|
|
19
|
+
readonly firstAudioMs?: number;
|
|
20
|
+
readonly totalMs?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Turn start to the first synthesis request. Below `llmMs` when the engine spoke
|
|
23
|
+
* clauses while the reply was still generating, and equal to it otherwise.
|
|
24
|
+
*/
|
|
25
|
+
readonly firstSynthesisMs?: number;
|
|
26
|
+
/** True when the reply was synthesized clause by clause, so `ttsMs` covers overlapped work. */
|
|
27
|
+
readonly incremental?: boolean;
|
|
28
|
+
}
|
|
29
|
+
/** File metadata for a previewable artifact, mirroring the unified `artifact_start.file` field. */
|
|
30
|
+
interface VoiceArtifactFile {
|
|
31
|
+
readonly path: string;
|
|
32
|
+
readonly mimeType: string;
|
|
33
|
+
readonly language?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* An artifact the agent produced during a call, assembled from the additive
|
|
37
|
+
* `artifact` wire message. Artifacts arrive beside speech and are never spoken.
|
|
38
|
+
*/
|
|
39
|
+
interface VoiceArtifact {
|
|
40
|
+
readonly id: string;
|
|
41
|
+
readonly turnId: string | null;
|
|
42
|
+
readonly executionId?: string;
|
|
43
|
+
readonly artifactType: 'markdown' | 'component';
|
|
44
|
+
readonly title?: string;
|
|
45
|
+
readonly file?: VoiceArtifactFile;
|
|
46
|
+
readonly content: string;
|
|
47
|
+
readonly component?: string;
|
|
48
|
+
readonly props?: Record<string, unknown>;
|
|
49
|
+
readonly status: 'streaming' | 'complete';
|
|
50
|
+
}
|
|
51
|
+
/** The conversation this call is attached to, reported by the server once the call opens. */
|
|
52
|
+
interface VoiceSession {
|
|
53
|
+
readonly sessionId: string;
|
|
54
|
+
readonly conversationId: string;
|
|
55
|
+
}
|
|
56
|
+
interface VoiceSnapshot {
|
|
57
|
+
readonly status: VoiceStatus;
|
|
58
|
+
readonly transcript: readonly TranscriptEntry[];
|
|
59
|
+
readonly interimTranscript: string | null;
|
|
60
|
+
readonly metrics: VoiceMetrics | null;
|
|
61
|
+
readonly audioLevel: number;
|
|
62
|
+
readonly isMuted: boolean;
|
|
63
|
+
readonly error: string | null;
|
|
64
|
+
readonly errorDetails?: {
|
|
65
|
+
code: string;
|
|
66
|
+
serverId?: string;
|
|
67
|
+
serverName: string;
|
|
68
|
+
diagnosticId: string;
|
|
69
|
+
};
|
|
70
|
+
readonly interruptionMode: InterruptionMode;
|
|
71
|
+
readonly canCancel: boolean;
|
|
72
|
+
readonly artifacts: readonly VoiceArtifact[];
|
|
73
|
+
/** Survives `endCall` so a host can reopen the same conversation; null until the server reports it. */
|
|
74
|
+
readonly session: VoiceSession | null;
|
|
75
|
+
}
|
|
76
|
+
interface VoiceClientOptions {
|
|
77
|
+
agentId: string;
|
|
78
|
+
/** Disable when the transcript consumer only supports sequential pipeline turns. */
|
|
79
|
+
fullDuplex?: boolean;
|
|
80
|
+
/** Optional custom audio source; the client owns and stops the returned tracks. */
|
|
81
|
+
audioSource?: () => Promise<MediaStream>;
|
|
82
|
+
/** Browser client token, or a getter called once per call to obtain a fresh token. */
|
|
83
|
+
clientToken: string | (() => string | Promise<string>);
|
|
84
|
+
/** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
|
|
85
|
+
apiUrl?: string;
|
|
86
|
+
/**
|
|
87
|
+
* Declares the `artifacts` capability, so the server sends `artifact` messages
|
|
88
|
+
* and the client assembles `snapshot.artifacts`. Off by default, which keeps an
|
|
89
|
+
* existing embed's wire byte-identical.
|
|
90
|
+
*/
|
|
91
|
+
artifacts?: boolean;
|
|
92
|
+
/** Reuses an existing client session's conversation. Overrides the id remembered from a previous call. */
|
|
93
|
+
sessionId?: string;
|
|
94
|
+
/** Proof minted by client/init; resolve from the host's token-scoped visitor store on every call. */
|
|
95
|
+
visitorToken?: string | (() => string | undefined | Promise<string | undefined>);
|
|
96
|
+
/** Observe the server's conversation so a host can reuse it for text. Never contains visitor credentials. */
|
|
97
|
+
onSession?: (session: VoiceSession) => void;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceArtifact as b, VoiceArtifactFile as c, VoiceMetrics as d, VoiceSession as e, VoiceStatus as f };
|