@runtypelabs/voice 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -3
- package/dist/index.cjs +46 -8
- package/dist/index.d.cts +3 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +46 -8
- package/dist/persona.cjs +48 -10
- package/dist/persona.d.cts +1 -1
- package/dist/persona.d.ts +1 -1
- package/dist/persona.js +48 -10
- package/dist/react.cjs +61 -10
- package/dist/react.d.cts +3 -3
- package/dist/react.d.ts +3 -3
- package/dist/react.js +61 -10
- package/dist/{types-DaXyPeiV.d.cts → types-Dkh4LxEA.d.cts} +5 -0
- package/dist/{types-DaXyPeiV.d.ts → types-Dkh4LxEA.d.ts} +5 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -54,6 +54,11 @@ alongside `transcript`.
|
|
|
54
54
|
Changing `clientToken` affects the next call without interrupting the current one.
|
|
55
55
|
Startup and connection failures populate `error` and `status: 'error'`.
|
|
56
56
|
|
|
57
|
+
GPT-Live requires the default full-duplex browser protocol. Set `fullDuplex: false`
|
|
58
|
+
only for a client that cannot handle overlapping transcripts. GPT-Live does not
|
|
59
|
+
resume shared text/voice sessions; a call with `sessionId` or `visitorToken`
|
|
60
|
+
receives `VOICE_SHARED_SESSION_UNSUPPORTED`.
|
|
61
|
+
|
|
57
62
|
## Plain JavaScript
|
|
58
63
|
|
|
59
64
|
```ts
|
|
@@ -85,6 +90,10 @@ The adapter implements Persona 4.22's `VoiceProvider` API and delivers transcrip
|
|
|
85
90
|
through `onTranscript`, avoiding duplicate text dispatches. Use it as the custom
|
|
86
91
|
provider in the widget's `config`:
|
|
87
92
|
|
|
93
|
+
Persona 4.22 cannot handle GPT-Live's overlapping transcripts. The adapter does
|
|
94
|
+
not advertise full duplex, so it cannot connect to a GPT-Live agent. Use the
|
|
95
|
+
plain `VoiceClient` or the React hook for GPT-Live calls.
|
|
96
|
+
|
|
88
97
|
```ts
|
|
89
98
|
import { createPersonaVoiceProvider } from '@runtypelabs/voice/persona'
|
|
90
99
|
|
|
@@ -125,8 +134,8 @@ embed's wire byte-identical.
|
|
|
125
134
|
`event` is a unified `artifact_start`, `artifact_delta`, `artifact_update`, or
|
|
126
135
|
`artifact_complete` frame. A start opens a `streaming` record, deltas append to
|
|
127
136
|
its `content`, an update carries the component payload, and a complete settles
|
|
128
|
-
it.
|
|
129
|
-
|
|
137
|
+
it. Cancelling a turn drops its later artifacts. Playback-only Stop keeps
|
|
138
|
+
artifact delivery active. Ending the call keeps completed artifacts
|
|
130
139
|
and drops half-streamed ones. Artifacts are never spoken: file bodies arrive
|
|
131
140
|
only through these frames.
|
|
132
141
|
|
|
@@ -136,15 +145,32 @@ widget through its `upsertArtifact` handle:
|
|
|
136
145
|
|
|
137
146
|
```ts
|
|
138
147
|
import { bindVoiceArtifactsToPersona, createPersonaVoiceProvider } from '@runtypelabs/voice/persona'
|
|
148
|
+
import { initAgentWidget } from '@runtypelabs/persona'
|
|
139
149
|
|
|
140
150
|
const provider = createPersonaVoiceProvider({ agentId, clientToken, artifacts: true })
|
|
141
|
-
const widget = initAgentWidget({
|
|
151
|
+
const widget = initAgentWidget({
|
|
152
|
+
target: document.getElementById('chat')!,
|
|
153
|
+
config: {
|
|
154
|
+
...config,
|
|
155
|
+
features: { artifacts: { enabled: true } },
|
|
156
|
+
voiceRecognition: {
|
|
157
|
+
enabled: true,
|
|
158
|
+
provider: { type: 'custom', custom: () => provider },
|
|
159
|
+
},
|
|
160
|
+
},
|
|
161
|
+
})
|
|
142
162
|
const release = bindVoiceArtifactsToPersona(provider, widget)
|
|
163
|
+
|
|
164
|
+
function dispose() {
|
|
165
|
+
release()
|
|
166
|
+
widget.destroy()
|
|
167
|
+
}
|
|
143
168
|
```
|
|
144
169
|
|
|
145
170
|
The artifact id is the upsert id, so every frame updates the same record in
|
|
146
171
|
place, and only the settled record writes a transcript block. The pane opens
|
|
147
172
|
once on the first artifact; pass `{ showOnFirst: false }` to leave it closed.
|
|
173
|
+
Call `dispose()` when the host removes the widget.
|
|
148
174
|
Persona's `features.artifacts.enabled` must be on, because `showArtifacts()` is
|
|
149
175
|
a no-op otherwise. Call the returned function to stop delivery; the adapter's
|
|
150
176
|
`disconnect()` releases artifact callbacks along with the rest.
|
package/dist/index.cjs
CHANGED
|
@@ -35,13 +35,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
35
35
|
this.readOffset = 0
|
|
36
36
|
this.buffered = 0
|
|
37
37
|
this.waiting = true
|
|
38
|
-
// INVARIANT:
|
|
38
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
39
39
|
this.eosSeen = false
|
|
40
|
+
this.continuous = false
|
|
40
41
|
this.revision = 0
|
|
41
42
|
this.port.onmessage = (e) => {
|
|
42
43
|
const msg = e.data
|
|
43
44
|
this.revision = msg.revision
|
|
44
|
-
if (msg.type === '
|
|
45
|
+
if (msg.type === 'continuous') {
|
|
46
|
+
this.continuous = msg.enabled
|
|
47
|
+
} else if (msg.type === 'push') {
|
|
45
48
|
this.eosSeen = false
|
|
46
49
|
this.chunks.push(msg.samples)
|
|
47
50
|
this.buffered += msg.samples.length
|
|
@@ -81,7 +84,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
81
84
|
}
|
|
82
85
|
if (this.buffered === 0) {
|
|
83
86
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
84
|
-
if (this.eosSeen) {
|
|
87
|
+
if (this.eosSeen || this.continuous) {
|
|
85
88
|
this.eosSeen = false
|
|
86
89
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
87
90
|
}
|
|
@@ -125,6 +128,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
125
128
|
if (samples.length === 0) return;
|
|
126
129
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
127
130
|
},
|
|
131
|
+
setContinuousMode(enabled) {
|
|
132
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
133
|
+
},
|
|
128
134
|
endOfStream() {
|
|
129
135
|
node.port.postMessage({ type: "eos", revision });
|
|
130
136
|
},
|
|
@@ -251,6 +257,7 @@ var VoiceClient = class {
|
|
|
251
257
|
committedAssistantTurnId = null;
|
|
252
258
|
activeTurnId = null;
|
|
253
259
|
suppressedArtifactTurnId = null;
|
|
260
|
+
fullDuplex = false;
|
|
254
261
|
getSnapshot = () => this.snapshot;
|
|
255
262
|
subscribe = (listener) => {
|
|
256
263
|
this.listeners.add(listener);
|
|
@@ -301,10 +308,12 @@ var VoiceClient = class {
|
|
|
301
308
|
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
302
309
|
throw new Error("Invalid voice visitor credential.");
|
|
303
310
|
}
|
|
311
|
+
if (this.options.fullDuplex !== false)
|
|
312
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
304
313
|
url.hash = "";
|
|
305
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
314
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
306
315
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
307
|
-
});
|
|
316
|
+
}));
|
|
308
317
|
if (generation !== this.generation) {
|
|
309
318
|
stream.getTracks().forEach((track) => track.stop());
|
|
310
319
|
return;
|
|
@@ -399,6 +408,7 @@ var VoiceClient = class {
|
|
|
399
408
|
});
|
|
400
409
|
}
|
|
401
410
|
cleanup() {
|
|
411
|
+
this.fullDuplex = false;
|
|
402
412
|
this.generation += 1;
|
|
403
413
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
404
414
|
this.processor?.disconnect();
|
|
@@ -535,6 +545,8 @@ var VoiceClient = class {
|
|
|
535
545
|
}
|
|
536
546
|
switch (msg.type) {
|
|
537
547
|
case "session_config": {
|
|
548
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
549
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
538
550
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
539
551
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
540
552
|
}
|
|
@@ -550,6 +562,32 @@ var VoiceClient = class {
|
|
|
550
562
|
case "artifact":
|
|
551
563
|
this.applyArtifactMessage(msg);
|
|
552
564
|
break;
|
|
565
|
+
case "transcript_update": {
|
|
566
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
567
|
+
break;
|
|
568
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
569
|
+
const entries = [...this.snapshot.transcript];
|
|
570
|
+
const index = entries.findIndex(
|
|
571
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
572
|
+
);
|
|
573
|
+
const entry = {
|
|
574
|
+
role: msg.role,
|
|
575
|
+
content: msg.text,
|
|
576
|
+
turnId: msg.turnId,
|
|
577
|
+
isFinal: msg.final === true,
|
|
578
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
579
|
+
};
|
|
580
|
+
if (index >= 0) entries[index] = entry;
|
|
581
|
+
else entries.push(entry);
|
|
582
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
583
|
+
break;
|
|
584
|
+
}
|
|
585
|
+
case "delegation_started":
|
|
586
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
587
|
+
break;
|
|
588
|
+
case "delegation_completed":
|
|
589
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
590
|
+
break;
|
|
553
591
|
case "transcript_interim":
|
|
554
592
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
555
593
|
break;
|
|
@@ -659,14 +697,14 @@ var VoiceClient = class {
|
|
|
659
697
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
660
698
|
this.processor = processor;
|
|
661
699
|
processor.onaudioprocess = (event) => {
|
|
662
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
700
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
663
701
|
const input = event.inputBuffer.getChannelData(0);
|
|
664
702
|
let sum = 0;
|
|
665
703
|
for (const sample of input) sum += sample * sample;
|
|
666
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
704
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
667
705
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
668
706
|
const pcm = new Int16Array(input.length);
|
|
669
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
707
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
670
708
|
socket.send(pcm.buffer);
|
|
671
709
|
return;
|
|
672
710
|
}
|
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-
|
|
2
|
-
export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-
|
|
1
|
+
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-Dkh4LxEA.cjs';
|
|
2
|
+
export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-Dkh4LxEA.cjs';
|
|
3
3
|
|
|
4
4
|
/** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
|
|
5
5
|
declare class VoiceClient {
|
|
@@ -29,6 +29,7 @@ declare class VoiceClient {
|
|
|
29
29
|
private committedAssistantTurnId;
|
|
30
30
|
private activeTurnId;
|
|
31
31
|
private suppressedArtifactTurnId;
|
|
32
|
+
private fullDuplex;
|
|
32
33
|
constructor(options: VoiceClientOptions);
|
|
33
34
|
getSnapshot: () => VoiceSnapshot;
|
|
34
35
|
subscribe: (listener: () => void) => (() => void);
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-
|
|
2
|
-
export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-
|
|
1
|
+
import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-Dkh4LxEA.js';
|
|
2
|
+
export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-Dkh4LxEA.js';
|
|
3
3
|
|
|
4
4
|
/** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
|
|
5
5
|
declare class VoiceClient {
|
|
@@ -29,6 +29,7 @@ declare class VoiceClient {
|
|
|
29
29
|
private committedAssistantTurnId;
|
|
30
30
|
private activeTurnId;
|
|
31
31
|
private suppressedArtifactTurnId;
|
|
32
|
+
private fullDuplex;
|
|
32
33
|
constructor(options: VoiceClientOptions);
|
|
33
34
|
getSnapshot: () => VoiceSnapshot;
|
|
34
35
|
subscribe: (listener: () => void) => (() => void);
|
package/dist/index.js
CHANGED
|
@@ -9,13 +9,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
9
9
|
this.readOffset = 0
|
|
10
10
|
this.buffered = 0
|
|
11
11
|
this.waiting = true
|
|
12
|
-
// INVARIANT:
|
|
12
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
13
13
|
this.eosSeen = false
|
|
14
|
+
this.continuous = false
|
|
14
15
|
this.revision = 0
|
|
15
16
|
this.port.onmessage = (e) => {
|
|
16
17
|
const msg = e.data
|
|
17
18
|
this.revision = msg.revision
|
|
18
|
-
if (msg.type === '
|
|
19
|
+
if (msg.type === 'continuous') {
|
|
20
|
+
this.continuous = msg.enabled
|
|
21
|
+
} else if (msg.type === 'push') {
|
|
19
22
|
this.eosSeen = false
|
|
20
23
|
this.chunks.push(msg.samples)
|
|
21
24
|
this.buffered += msg.samples.length
|
|
@@ -55,7 +58,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
55
58
|
}
|
|
56
59
|
if (this.buffered === 0) {
|
|
57
60
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
58
|
-
if (this.eosSeen) {
|
|
61
|
+
if (this.eosSeen || this.continuous) {
|
|
59
62
|
this.eosSeen = false
|
|
60
63
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
61
64
|
}
|
|
@@ -99,6 +102,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
99
102
|
if (samples.length === 0) return;
|
|
100
103
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
101
104
|
},
|
|
105
|
+
setContinuousMode(enabled) {
|
|
106
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
107
|
+
},
|
|
102
108
|
endOfStream() {
|
|
103
109
|
node.port.postMessage({ type: "eos", revision });
|
|
104
110
|
},
|
|
@@ -225,6 +231,7 @@ var VoiceClient = class {
|
|
|
225
231
|
committedAssistantTurnId = null;
|
|
226
232
|
activeTurnId = null;
|
|
227
233
|
suppressedArtifactTurnId = null;
|
|
234
|
+
fullDuplex = false;
|
|
228
235
|
getSnapshot = () => this.snapshot;
|
|
229
236
|
subscribe = (listener) => {
|
|
230
237
|
this.listeners.add(listener);
|
|
@@ -275,10 +282,12 @@ var VoiceClient = class {
|
|
|
275
282
|
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
276
283
|
throw new Error("Invalid voice visitor credential.");
|
|
277
284
|
}
|
|
285
|
+
if (this.options.fullDuplex !== false)
|
|
286
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
278
287
|
url.hash = "";
|
|
279
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
288
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
280
289
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
281
|
-
});
|
|
290
|
+
}));
|
|
282
291
|
if (generation !== this.generation) {
|
|
283
292
|
stream.getTracks().forEach((track) => track.stop());
|
|
284
293
|
return;
|
|
@@ -373,6 +382,7 @@ var VoiceClient = class {
|
|
|
373
382
|
});
|
|
374
383
|
}
|
|
375
384
|
cleanup() {
|
|
385
|
+
this.fullDuplex = false;
|
|
376
386
|
this.generation += 1;
|
|
377
387
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
378
388
|
this.processor?.disconnect();
|
|
@@ -509,6 +519,8 @@ var VoiceClient = class {
|
|
|
509
519
|
}
|
|
510
520
|
switch (msg.type) {
|
|
511
521
|
case "session_config": {
|
|
522
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
523
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
512
524
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
513
525
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
514
526
|
}
|
|
@@ -524,6 +536,32 @@ var VoiceClient = class {
|
|
|
524
536
|
case "artifact":
|
|
525
537
|
this.applyArtifactMessage(msg);
|
|
526
538
|
break;
|
|
539
|
+
case "transcript_update": {
|
|
540
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
541
|
+
break;
|
|
542
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
543
|
+
const entries = [...this.snapshot.transcript];
|
|
544
|
+
const index = entries.findIndex(
|
|
545
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
546
|
+
);
|
|
547
|
+
const entry = {
|
|
548
|
+
role: msg.role,
|
|
549
|
+
content: msg.text,
|
|
550
|
+
turnId: msg.turnId,
|
|
551
|
+
isFinal: msg.final === true,
|
|
552
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
553
|
+
};
|
|
554
|
+
if (index >= 0) entries[index] = entry;
|
|
555
|
+
else entries.push(entry);
|
|
556
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
557
|
+
break;
|
|
558
|
+
}
|
|
559
|
+
case "delegation_started":
|
|
560
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
561
|
+
break;
|
|
562
|
+
case "delegation_completed":
|
|
563
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
564
|
+
break;
|
|
527
565
|
case "transcript_interim":
|
|
528
566
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
529
567
|
break;
|
|
@@ -633,14 +671,14 @@ var VoiceClient = class {
|
|
|
633
671
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
634
672
|
this.processor = processor;
|
|
635
673
|
processor.onaudioprocess = (event) => {
|
|
636
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
674
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
637
675
|
const input = event.inputBuffer.getChannelData(0);
|
|
638
676
|
let sum = 0;
|
|
639
677
|
for (const sample of input) sum += sample * sample;
|
|
640
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
678
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
641
679
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
642
680
|
const pcm = new Int16Array(input.length);
|
|
643
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
681
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
644
682
|
socket.send(pcm.buffer);
|
|
645
683
|
return;
|
|
646
684
|
}
|
package/dist/persona.cjs
CHANGED
|
@@ -36,13 +36,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
36
36
|
this.readOffset = 0
|
|
37
37
|
this.buffered = 0
|
|
38
38
|
this.waiting = true
|
|
39
|
-
// INVARIANT:
|
|
39
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
40
40
|
this.eosSeen = false
|
|
41
|
+
this.continuous = false
|
|
41
42
|
this.revision = 0
|
|
42
43
|
this.port.onmessage = (e) => {
|
|
43
44
|
const msg = e.data
|
|
44
45
|
this.revision = msg.revision
|
|
45
|
-
if (msg.type === '
|
|
46
|
+
if (msg.type === 'continuous') {
|
|
47
|
+
this.continuous = msg.enabled
|
|
48
|
+
} else if (msg.type === 'push') {
|
|
46
49
|
this.eosSeen = false
|
|
47
50
|
this.chunks.push(msg.samples)
|
|
48
51
|
this.buffered += msg.samples.length
|
|
@@ -82,7 +85,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
82
85
|
}
|
|
83
86
|
if (this.buffered === 0) {
|
|
84
87
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
85
|
-
if (this.eosSeen) {
|
|
88
|
+
if (this.eosSeen || this.continuous) {
|
|
86
89
|
this.eosSeen = false
|
|
87
90
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
88
91
|
}
|
|
@@ -126,6 +129,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
126
129
|
if (samples.length === 0) return;
|
|
127
130
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
128
131
|
},
|
|
132
|
+
setContinuousMode(enabled) {
|
|
133
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
134
|
+
},
|
|
129
135
|
endOfStream() {
|
|
130
136
|
node.port.postMessage({ type: "eos", revision });
|
|
131
137
|
},
|
|
@@ -252,6 +258,7 @@ var VoiceClient = class {
|
|
|
252
258
|
committedAssistantTurnId = null;
|
|
253
259
|
activeTurnId = null;
|
|
254
260
|
suppressedArtifactTurnId = null;
|
|
261
|
+
fullDuplex = false;
|
|
255
262
|
getSnapshot = () => this.snapshot;
|
|
256
263
|
subscribe = (listener) => {
|
|
257
264
|
this.listeners.add(listener);
|
|
@@ -302,10 +309,12 @@ var VoiceClient = class {
|
|
|
302
309
|
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
303
310
|
throw new Error("Invalid voice visitor credential.");
|
|
304
311
|
}
|
|
312
|
+
if (this.options.fullDuplex !== false)
|
|
313
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
305
314
|
url.hash = "";
|
|
306
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
315
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
307
316
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
308
|
-
});
|
|
317
|
+
}));
|
|
309
318
|
if (generation !== this.generation) {
|
|
310
319
|
stream.getTracks().forEach((track) => track.stop());
|
|
311
320
|
return;
|
|
@@ -400,6 +409,7 @@ var VoiceClient = class {
|
|
|
400
409
|
});
|
|
401
410
|
}
|
|
402
411
|
cleanup() {
|
|
412
|
+
this.fullDuplex = false;
|
|
403
413
|
this.generation += 1;
|
|
404
414
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
405
415
|
this.processor?.disconnect();
|
|
@@ -536,6 +546,8 @@ var VoiceClient = class {
|
|
|
536
546
|
}
|
|
537
547
|
switch (msg.type) {
|
|
538
548
|
case "session_config": {
|
|
549
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
550
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
539
551
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
540
552
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
541
553
|
}
|
|
@@ -551,6 +563,32 @@ var VoiceClient = class {
|
|
|
551
563
|
case "artifact":
|
|
552
564
|
this.applyArtifactMessage(msg);
|
|
553
565
|
break;
|
|
566
|
+
case "transcript_update": {
|
|
567
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
568
|
+
break;
|
|
569
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
570
|
+
const entries = [...this.snapshot.transcript];
|
|
571
|
+
const index = entries.findIndex(
|
|
572
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
573
|
+
);
|
|
574
|
+
const entry = {
|
|
575
|
+
role: msg.role,
|
|
576
|
+
content: msg.text,
|
|
577
|
+
turnId: msg.turnId,
|
|
578
|
+
isFinal: msg.final === true,
|
|
579
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
580
|
+
};
|
|
581
|
+
if (index >= 0) entries[index] = entry;
|
|
582
|
+
else entries.push(entry);
|
|
583
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
584
|
+
break;
|
|
585
|
+
}
|
|
586
|
+
case "delegation_started":
|
|
587
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
588
|
+
break;
|
|
589
|
+
case "delegation_completed":
|
|
590
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
591
|
+
break;
|
|
554
592
|
case "transcript_interim":
|
|
555
593
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
556
594
|
break;
|
|
@@ -660,14 +698,14 @@ var VoiceClient = class {
|
|
|
660
698
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
661
699
|
this.processor = processor;
|
|
662
700
|
processor.onaudioprocess = (event) => {
|
|
663
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
701
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
664
702
|
const input = event.inputBuffer.getChannelData(0);
|
|
665
703
|
let sum = 0;
|
|
666
704
|
for (const sample of input) sum += sample * sample;
|
|
667
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
705
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
668
706
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
669
707
|
const pcm = new Int16Array(input.length);
|
|
670
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
708
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
671
709
|
socket.send(pcm.buffer);
|
|
672
710
|
return;
|
|
673
711
|
}
|
|
@@ -684,7 +722,7 @@ var VoiceClient = class {
|
|
|
684
722
|
|
|
685
723
|
// src/persona.ts
|
|
686
724
|
function createPersonaVoiceProvider(options) {
|
|
687
|
-
const client = new VoiceClient(options);
|
|
725
|
+
const client = new VoiceClient({ ...options, fullDuplex: false });
|
|
688
726
|
const statusCallbacks = /* @__PURE__ */ new Set();
|
|
689
727
|
const errorCallbacks = /* @__PURE__ */ new Set();
|
|
690
728
|
const transcriptCallbacks = /* @__PURE__ */ new Set();
|
|
@@ -720,7 +758,7 @@ function createPersonaVoiceProvider(options) {
|
|
|
720
758
|
});
|
|
721
759
|
snapshot.transcript.forEach((entry, index) => {
|
|
722
760
|
if (entry === before.transcript[index]) return;
|
|
723
|
-
const isFinal = entry.partial !== true;
|
|
761
|
+
const isFinal = entry.partial !== true && entry.isFinal !== false;
|
|
724
762
|
for (const callback of transcriptCallbacks) {
|
|
725
763
|
if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
|
|
726
764
|
else callback(entry.role, entry.content, isFinal);
|
package/dist/persona.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AgentWidgetInitHandle, VoiceProvider } from '@runtypelabs/persona';
|
|
2
|
-
import { b as VoiceArtifact, V as VoiceClientOptions } from './types-
|
|
2
|
+
import { b as VoiceArtifact, V as VoiceClientOptions } from './types-Dkh4LxEA.cjs';
|
|
3
3
|
|
|
4
4
|
type ArtifactCallback = (artifact: VoiceArtifact) => void;
|
|
5
5
|
/**
|
package/dist/persona.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AgentWidgetInitHandle, VoiceProvider } from '@runtypelabs/persona';
|
|
2
|
-
import { b as VoiceArtifact, V as VoiceClientOptions } from './types-
|
|
2
|
+
import { b as VoiceArtifact, V as VoiceClientOptions } from './types-Dkh4LxEA.js';
|
|
3
3
|
|
|
4
4
|
type ArtifactCallback = (artifact: VoiceArtifact) => void;
|
|
5
5
|
/**
|
package/dist/persona.js
CHANGED
|
@@ -9,13 +9,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
9
9
|
this.readOffset = 0
|
|
10
10
|
this.buffered = 0
|
|
11
11
|
this.waiting = true
|
|
12
|
-
// INVARIANT:
|
|
12
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
13
13
|
this.eosSeen = false
|
|
14
|
+
this.continuous = false
|
|
14
15
|
this.revision = 0
|
|
15
16
|
this.port.onmessage = (e) => {
|
|
16
17
|
const msg = e.data
|
|
17
18
|
this.revision = msg.revision
|
|
18
|
-
if (msg.type === '
|
|
19
|
+
if (msg.type === 'continuous') {
|
|
20
|
+
this.continuous = msg.enabled
|
|
21
|
+
} else if (msg.type === 'push') {
|
|
19
22
|
this.eosSeen = false
|
|
20
23
|
this.chunks.push(msg.samples)
|
|
21
24
|
this.buffered += msg.samples.length
|
|
@@ -55,7 +58,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
55
58
|
}
|
|
56
59
|
if (this.buffered === 0) {
|
|
57
60
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
58
|
-
if (this.eosSeen) {
|
|
61
|
+
if (this.eosSeen || this.continuous) {
|
|
59
62
|
this.eosSeen = false
|
|
60
63
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
61
64
|
}
|
|
@@ -99,6 +102,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
99
102
|
if (samples.length === 0) return;
|
|
100
103
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
101
104
|
},
|
|
105
|
+
setContinuousMode(enabled) {
|
|
106
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
107
|
+
},
|
|
102
108
|
endOfStream() {
|
|
103
109
|
node.port.postMessage({ type: "eos", revision });
|
|
104
110
|
},
|
|
@@ -225,6 +231,7 @@ var VoiceClient = class {
|
|
|
225
231
|
committedAssistantTurnId = null;
|
|
226
232
|
activeTurnId = null;
|
|
227
233
|
suppressedArtifactTurnId = null;
|
|
234
|
+
fullDuplex = false;
|
|
228
235
|
getSnapshot = () => this.snapshot;
|
|
229
236
|
subscribe = (listener) => {
|
|
230
237
|
this.listeners.add(listener);
|
|
@@ -275,10 +282,12 @@ var VoiceClient = class {
|
|
|
275
282
|
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
276
283
|
throw new Error("Invalid voice visitor credential.");
|
|
277
284
|
}
|
|
285
|
+
if (this.options.fullDuplex !== false)
|
|
286
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
278
287
|
url.hash = "";
|
|
279
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
288
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
280
289
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
281
|
-
});
|
|
290
|
+
}));
|
|
282
291
|
if (generation !== this.generation) {
|
|
283
292
|
stream.getTracks().forEach((track) => track.stop());
|
|
284
293
|
return;
|
|
@@ -373,6 +382,7 @@ var VoiceClient = class {
|
|
|
373
382
|
});
|
|
374
383
|
}
|
|
375
384
|
cleanup() {
|
|
385
|
+
this.fullDuplex = false;
|
|
376
386
|
this.generation += 1;
|
|
377
387
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
378
388
|
this.processor?.disconnect();
|
|
@@ -509,6 +519,8 @@ var VoiceClient = class {
|
|
|
509
519
|
}
|
|
510
520
|
switch (msg.type) {
|
|
511
521
|
case "session_config": {
|
|
522
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
523
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
512
524
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
513
525
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
514
526
|
}
|
|
@@ -524,6 +536,32 @@ var VoiceClient = class {
|
|
|
524
536
|
case "artifact":
|
|
525
537
|
this.applyArtifactMessage(msg);
|
|
526
538
|
break;
|
|
539
|
+
case "transcript_update": {
|
|
540
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
541
|
+
break;
|
|
542
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
543
|
+
const entries = [...this.snapshot.transcript];
|
|
544
|
+
const index = entries.findIndex(
|
|
545
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
546
|
+
);
|
|
547
|
+
const entry = {
|
|
548
|
+
role: msg.role,
|
|
549
|
+
content: msg.text,
|
|
550
|
+
turnId: msg.turnId,
|
|
551
|
+
isFinal: msg.final === true,
|
|
552
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
553
|
+
};
|
|
554
|
+
if (index >= 0) entries[index] = entry;
|
|
555
|
+
else entries.push(entry);
|
|
556
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
557
|
+
break;
|
|
558
|
+
}
|
|
559
|
+
case "delegation_started":
|
|
560
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
561
|
+
break;
|
|
562
|
+
case "delegation_completed":
|
|
563
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
564
|
+
break;
|
|
527
565
|
case "transcript_interim":
|
|
528
566
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
529
567
|
break;
|
|
@@ -633,14 +671,14 @@ var VoiceClient = class {
|
|
|
633
671
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
634
672
|
this.processor = processor;
|
|
635
673
|
processor.onaudioprocess = (event) => {
|
|
636
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
674
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
637
675
|
const input = event.inputBuffer.getChannelData(0);
|
|
638
676
|
let sum = 0;
|
|
639
677
|
for (const sample of input) sum += sample * sample;
|
|
640
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
678
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
641
679
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
642
680
|
const pcm = new Int16Array(input.length);
|
|
643
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
681
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
644
682
|
socket.send(pcm.buffer);
|
|
645
683
|
return;
|
|
646
684
|
}
|
|
@@ -657,7 +695,7 @@ var VoiceClient = class {
|
|
|
657
695
|
|
|
658
696
|
// src/persona.ts
|
|
659
697
|
function createPersonaVoiceProvider(options) {
|
|
660
|
-
const client = new VoiceClient(options);
|
|
698
|
+
const client = new VoiceClient({ ...options, fullDuplex: false });
|
|
661
699
|
const statusCallbacks = /* @__PURE__ */ new Set();
|
|
662
700
|
const errorCallbacks = /* @__PURE__ */ new Set();
|
|
663
701
|
const transcriptCallbacks = /* @__PURE__ */ new Set();
|
|
@@ -693,7 +731,7 @@ function createPersonaVoiceProvider(options) {
|
|
|
693
731
|
});
|
|
694
732
|
snapshot.transcript.forEach((entry, index) => {
|
|
695
733
|
if (entry === before.transcript[index]) return;
|
|
696
|
-
const isFinal = entry.partial !== true;
|
|
734
|
+
const isFinal = entry.partial !== true && entry.isFinal !== false;
|
|
697
735
|
for (const callback of transcriptCallbacks) {
|
|
698
736
|
if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
|
|
699
737
|
else callback(entry.role, entry.content, isFinal);
|
package/dist/react.cjs
CHANGED
|
@@ -37,13 +37,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
37
37
|
this.readOffset = 0
|
|
38
38
|
this.buffered = 0
|
|
39
39
|
this.waiting = true
|
|
40
|
-
// INVARIANT:
|
|
40
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
41
41
|
this.eosSeen = false
|
|
42
|
+
this.continuous = false
|
|
42
43
|
this.revision = 0
|
|
43
44
|
this.port.onmessage = (e) => {
|
|
44
45
|
const msg = e.data
|
|
45
46
|
this.revision = msg.revision
|
|
46
|
-
if (msg.type === '
|
|
47
|
+
if (msg.type === 'continuous') {
|
|
48
|
+
this.continuous = msg.enabled
|
|
49
|
+
} else if (msg.type === 'push') {
|
|
47
50
|
this.eosSeen = false
|
|
48
51
|
this.chunks.push(msg.samples)
|
|
49
52
|
this.buffered += msg.samples.length
|
|
@@ -83,7 +86,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
83
86
|
}
|
|
84
87
|
if (this.buffered === 0) {
|
|
85
88
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
86
|
-
if (this.eosSeen) {
|
|
89
|
+
if (this.eosSeen || this.continuous) {
|
|
87
90
|
this.eosSeen = false
|
|
88
91
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
89
92
|
}
|
|
@@ -127,6 +130,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
127
130
|
if (samples.length === 0) return;
|
|
128
131
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
129
132
|
},
|
|
133
|
+
setContinuousMode(enabled) {
|
|
134
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
135
|
+
},
|
|
130
136
|
endOfStream() {
|
|
131
137
|
node.port.postMessage({ type: "eos", revision });
|
|
132
138
|
},
|
|
@@ -253,6 +259,7 @@ var VoiceClient = class {
|
|
|
253
259
|
committedAssistantTurnId = null;
|
|
254
260
|
activeTurnId = null;
|
|
255
261
|
suppressedArtifactTurnId = null;
|
|
262
|
+
fullDuplex = false;
|
|
256
263
|
getSnapshot = () => this.snapshot;
|
|
257
264
|
subscribe = (listener) => {
|
|
258
265
|
this.listeners.add(listener);
|
|
@@ -303,10 +310,12 @@ var VoiceClient = class {
|
|
|
303
310
|
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
304
311
|
throw new Error("Invalid voice visitor credential.");
|
|
305
312
|
}
|
|
313
|
+
if (this.options.fullDuplex !== false)
|
|
314
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
306
315
|
url.hash = "";
|
|
307
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
316
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
308
317
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
309
|
-
});
|
|
318
|
+
}));
|
|
310
319
|
if (generation !== this.generation) {
|
|
311
320
|
stream.getTracks().forEach((track) => track.stop());
|
|
312
321
|
return;
|
|
@@ -401,6 +410,7 @@ var VoiceClient = class {
|
|
|
401
410
|
});
|
|
402
411
|
}
|
|
403
412
|
cleanup() {
|
|
413
|
+
this.fullDuplex = false;
|
|
404
414
|
this.generation += 1;
|
|
405
415
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
406
416
|
this.processor?.disconnect();
|
|
@@ -537,6 +547,8 @@ var VoiceClient = class {
|
|
|
537
547
|
}
|
|
538
548
|
switch (msg.type) {
|
|
539
549
|
case "session_config": {
|
|
550
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
551
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
540
552
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
541
553
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
542
554
|
}
|
|
@@ -552,6 +564,32 @@ var VoiceClient = class {
|
|
|
552
564
|
case "artifact":
|
|
553
565
|
this.applyArtifactMessage(msg);
|
|
554
566
|
break;
|
|
567
|
+
case "transcript_update": {
|
|
568
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
569
|
+
break;
|
|
570
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
571
|
+
const entries = [...this.snapshot.transcript];
|
|
572
|
+
const index = entries.findIndex(
|
|
573
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
574
|
+
);
|
|
575
|
+
const entry = {
|
|
576
|
+
role: msg.role,
|
|
577
|
+
content: msg.text,
|
|
578
|
+
turnId: msg.turnId,
|
|
579
|
+
isFinal: msg.final === true,
|
|
580
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
581
|
+
};
|
|
582
|
+
if (index >= 0) entries[index] = entry;
|
|
583
|
+
else entries.push(entry);
|
|
584
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
585
|
+
break;
|
|
586
|
+
}
|
|
587
|
+
case "delegation_started":
|
|
588
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
589
|
+
break;
|
|
590
|
+
case "delegation_completed":
|
|
591
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
592
|
+
break;
|
|
555
593
|
case "transcript_interim":
|
|
556
594
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
557
595
|
break;
|
|
@@ -661,14 +699,14 @@ var VoiceClient = class {
|
|
|
661
699
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
662
700
|
this.processor = processor;
|
|
663
701
|
processor.onaudioprocess = (event) => {
|
|
664
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
702
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
665
703
|
const input = event.inputBuffer.getChannelData(0);
|
|
666
704
|
let sum = 0;
|
|
667
705
|
for (const sample of input) sum += sample * sample;
|
|
668
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
706
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
669
707
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
670
708
|
const pcm = new Int16Array(input.length);
|
|
671
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
709
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
672
710
|
socket.send(pcm.buffer);
|
|
673
711
|
return;
|
|
674
712
|
}
|
|
@@ -691,7 +729,9 @@ function useVoiceClient({
|
|
|
691
729
|
artifacts,
|
|
692
730
|
sessionId,
|
|
693
731
|
visitorToken,
|
|
694
|
-
onSession
|
|
732
|
+
onSession,
|
|
733
|
+
fullDuplex,
|
|
734
|
+
audioSource
|
|
695
735
|
}) {
|
|
696
736
|
const sessionRef = (0, import_react.useRef)(sessionId);
|
|
697
737
|
(0, import_react.useEffect)(() => {
|
|
@@ -700,10 +740,15 @@ function useVoiceClient({
|
|
|
700
740
|
const tokenRef = (0, import_react.useRef)(clientToken);
|
|
701
741
|
const visitorRef = (0, import_react.useRef)(visitorToken);
|
|
702
742
|
const sessionCallbackRef = (0, import_react.useRef)(onSession);
|
|
743
|
+
const audioSourceRef = (0, import_react.useRef)(audioSource);
|
|
744
|
+
const hasAudioSource = audioSource !== void 0;
|
|
703
745
|
(0, import_react.useEffect)(() => {
|
|
704
746
|
visitorRef.current = visitorToken;
|
|
705
747
|
sessionCallbackRef.current = onSession;
|
|
706
748
|
}, [visitorToken, onSession]);
|
|
749
|
+
(0, import_react.useEffect)(() => {
|
|
750
|
+
audioSourceRef.current = audioSource;
|
|
751
|
+
}, [audioSource]);
|
|
707
752
|
(0, import_react.useEffect)(() => {
|
|
708
753
|
tokenRef.current = clientToken;
|
|
709
754
|
}, [clientToken]);
|
|
@@ -711,6 +756,8 @@ function useVoiceClient({
|
|
|
711
756
|
agentId,
|
|
712
757
|
apiUrl,
|
|
713
758
|
artifacts,
|
|
759
|
+
fullDuplex,
|
|
760
|
+
...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
|
|
714
761
|
get sessionId() {
|
|
715
762
|
return sessionRef.current;
|
|
716
763
|
},
|
|
@@ -724,16 +771,20 @@ function useVoiceClient({
|
|
|
724
771
|
agentId,
|
|
725
772
|
apiUrl,
|
|
726
773
|
artifacts,
|
|
774
|
+
fullDuplex,
|
|
775
|
+
hasAudioSource,
|
|
727
776
|
sessionId,
|
|
728
777
|
client: createClient()
|
|
729
778
|
}));
|
|
730
|
-
const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts;
|
|
779
|
+
const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
|
|
731
780
|
if (configurationChanged || binding.sessionId !== sessionId) {
|
|
732
781
|
const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
|
|
733
782
|
setBinding({
|
|
734
783
|
agentId,
|
|
735
784
|
apiUrl,
|
|
736
785
|
artifacts,
|
|
786
|
+
fullDuplex,
|
|
787
|
+
hasAudioSource,
|
|
737
788
|
sessionId,
|
|
738
789
|
client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
|
|
739
790
|
});
|
package/dist/react.d.cts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-
|
|
2
|
-
export { a as VoiceSnapshot } from './types-
|
|
1
|
+
import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-Dkh4LxEA.cjs';
|
|
2
|
+
export { a as VoiceSnapshot } from './types-Dkh4LxEA.cjs';
|
|
3
3
|
|
|
4
4
|
/** Subscribe to a voice client and end its call when the component unmounts or changes agent. */
|
|
5
|
-
declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, }: VoiceClientOptions): {
|
|
5
|
+
declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, fullDuplex, audioSource, }: VoiceClientOptions): {
|
|
6
6
|
startCall: (tokenOverride?: string) => Promise<void>;
|
|
7
7
|
endCall: () => void;
|
|
8
8
|
resetSession: () => void;
|
package/dist/react.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-
|
|
2
|
-
export { a as VoiceSnapshot } from './types-
|
|
1
|
+
import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-Dkh4LxEA.js';
|
|
2
|
+
export { a as VoiceSnapshot } from './types-Dkh4LxEA.js';
|
|
3
3
|
|
|
4
4
|
/** Subscribe to a voice client and end its call when the component unmounts or changes agent. */
|
|
5
|
-
declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, }: VoiceClientOptions): {
|
|
5
|
+
declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, fullDuplex, audioSource, }: VoiceClientOptions): {
|
|
6
6
|
startCall: (tokenOverride?: string) => Promise<void>;
|
|
7
7
|
endCall: () => void;
|
|
8
8
|
resetSession: () => void;
|
package/dist/react.js
CHANGED
|
@@ -14,13 +14,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
14
14
|
this.readOffset = 0
|
|
15
15
|
this.buffered = 0
|
|
16
16
|
this.waiting = true
|
|
17
|
-
// INVARIANT:
|
|
17
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
18
18
|
this.eosSeen = false
|
|
19
|
+
this.continuous = false
|
|
19
20
|
this.revision = 0
|
|
20
21
|
this.port.onmessage = (e) => {
|
|
21
22
|
const msg = e.data
|
|
22
23
|
this.revision = msg.revision
|
|
23
|
-
if (msg.type === '
|
|
24
|
+
if (msg.type === 'continuous') {
|
|
25
|
+
this.continuous = msg.enabled
|
|
26
|
+
} else if (msg.type === 'push') {
|
|
24
27
|
this.eosSeen = false
|
|
25
28
|
this.chunks.push(msg.samples)
|
|
26
29
|
this.buffered += msg.samples.length
|
|
@@ -60,7 +63,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
60
63
|
}
|
|
61
64
|
if (this.buffered === 0) {
|
|
62
65
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
63
|
-
if (this.eosSeen) {
|
|
66
|
+
if (this.eosSeen || this.continuous) {
|
|
64
67
|
this.eosSeen = false
|
|
65
68
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
66
69
|
}
|
|
@@ -104,6 +107,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
104
107
|
if (samples.length === 0) return;
|
|
105
108
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
106
109
|
},
|
|
110
|
+
setContinuousMode(enabled) {
|
|
111
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
112
|
+
},
|
|
107
113
|
endOfStream() {
|
|
108
114
|
node.port.postMessage({ type: "eos", revision });
|
|
109
115
|
},
|
|
@@ -230,6 +236,7 @@ var VoiceClient = class {
|
|
|
230
236
|
committedAssistantTurnId = null;
|
|
231
237
|
activeTurnId = null;
|
|
232
238
|
suppressedArtifactTurnId = null;
|
|
239
|
+
fullDuplex = false;
|
|
233
240
|
getSnapshot = () => this.snapshot;
|
|
234
241
|
subscribe = (listener) => {
|
|
235
242
|
this.listeners.add(listener);
|
|
@@ -280,10 +287,12 @@ var VoiceClient = class {
|
|
|
280
287
|
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
281
288
|
throw new Error("Invalid voice visitor credential.");
|
|
282
289
|
}
|
|
290
|
+
if (this.options.fullDuplex !== false)
|
|
291
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
283
292
|
url.hash = "";
|
|
284
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
293
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
285
294
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
286
|
-
});
|
|
295
|
+
}));
|
|
287
296
|
if (generation !== this.generation) {
|
|
288
297
|
stream.getTracks().forEach((track) => track.stop());
|
|
289
298
|
return;
|
|
@@ -378,6 +387,7 @@ var VoiceClient = class {
|
|
|
378
387
|
});
|
|
379
388
|
}
|
|
380
389
|
cleanup() {
|
|
390
|
+
this.fullDuplex = false;
|
|
381
391
|
this.generation += 1;
|
|
382
392
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
383
393
|
this.processor?.disconnect();
|
|
@@ -514,6 +524,8 @@ var VoiceClient = class {
|
|
|
514
524
|
}
|
|
515
525
|
switch (msg.type) {
|
|
516
526
|
case "session_config": {
|
|
527
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
528
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
517
529
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
518
530
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
519
531
|
}
|
|
@@ -529,6 +541,32 @@ var VoiceClient = class {
|
|
|
529
541
|
case "artifact":
|
|
530
542
|
this.applyArtifactMessage(msg);
|
|
531
543
|
break;
|
|
544
|
+
case "transcript_update": {
|
|
545
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
546
|
+
break;
|
|
547
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
548
|
+
const entries = [...this.snapshot.transcript];
|
|
549
|
+
const index = entries.findIndex(
|
|
550
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
551
|
+
);
|
|
552
|
+
const entry = {
|
|
553
|
+
role: msg.role,
|
|
554
|
+
content: msg.text,
|
|
555
|
+
turnId: msg.turnId,
|
|
556
|
+
isFinal: msg.final === true,
|
|
557
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
558
|
+
};
|
|
559
|
+
if (index >= 0) entries[index] = entry;
|
|
560
|
+
else entries.push(entry);
|
|
561
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
564
|
+
case "delegation_started":
|
|
565
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
566
|
+
break;
|
|
567
|
+
case "delegation_completed":
|
|
568
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
569
|
+
break;
|
|
532
570
|
case "transcript_interim":
|
|
533
571
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
534
572
|
break;
|
|
@@ -638,14 +676,14 @@ var VoiceClient = class {
|
|
|
638
676
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
639
677
|
this.processor = processor;
|
|
640
678
|
processor.onaudioprocess = (event) => {
|
|
641
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
679
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
642
680
|
const input = event.inputBuffer.getChannelData(0);
|
|
643
681
|
let sum = 0;
|
|
644
682
|
for (const sample of input) sum += sample * sample;
|
|
645
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
683
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
646
684
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
647
685
|
const pcm = new Int16Array(input.length);
|
|
648
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
686
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
649
687
|
socket.send(pcm.buffer);
|
|
650
688
|
return;
|
|
651
689
|
}
|
|
@@ -668,7 +706,9 @@ function useVoiceClient({
|
|
|
668
706
|
artifacts,
|
|
669
707
|
sessionId,
|
|
670
708
|
visitorToken,
|
|
671
|
-
onSession
|
|
709
|
+
onSession,
|
|
710
|
+
fullDuplex,
|
|
711
|
+
audioSource
|
|
672
712
|
}) {
|
|
673
713
|
const sessionRef = useRef(sessionId);
|
|
674
714
|
useEffect(() => {
|
|
@@ -677,10 +717,15 @@ function useVoiceClient({
|
|
|
677
717
|
const tokenRef = useRef(clientToken);
|
|
678
718
|
const visitorRef = useRef(visitorToken);
|
|
679
719
|
const sessionCallbackRef = useRef(onSession);
|
|
720
|
+
const audioSourceRef = useRef(audioSource);
|
|
721
|
+
const hasAudioSource = audioSource !== void 0;
|
|
680
722
|
useEffect(() => {
|
|
681
723
|
visitorRef.current = visitorToken;
|
|
682
724
|
sessionCallbackRef.current = onSession;
|
|
683
725
|
}, [visitorToken, onSession]);
|
|
726
|
+
useEffect(() => {
|
|
727
|
+
audioSourceRef.current = audioSource;
|
|
728
|
+
}, [audioSource]);
|
|
684
729
|
useEffect(() => {
|
|
685
730
|
tokenRef.current = clientToken;
|
|
686
731
|
}, [clientToken]);
|
|
@@ -688,6 +733,8 @@ function useVoiceClient({
|
|
|
688
733
|
agentId,
|
|
689
734
|
apiUrl,
|
|
690
735
|
artifacts,
|
|
736
|
+
fullDuplex,
|
|
737
|
+
...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
|
|
691
738
|
get sessionId() {
|
|
692
739
|
return sessionRef.current;
|
|
693
740
|
},
|
|
@@ -701,16 +748,20 @@ function useVoiceClient({
|
|
|
701
748
|
agentId,
|
|
702
749
|
apiUrl,
|
|
703
750
|
artifacts,
|
|
751
|
+
fullDuplex,
|
|
752
|
+
hasAudioSource,
|
|
704
753
|
sessionId,
|
|
705
754
|
client: createClient()
|
|
706
755
|
}));
|
|
707
|
-
const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts;
|
|
756
|
+
const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
|
|
708
757
|
if (configurationChanged || binding.sessionId !== sessionId) {
|
|
709
758
|
const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
|
|
710
759
|
setBinding({
|
|
711
760
|
agentId,
|
|
712
761
|
apiUrl,
|
|
713
762
|
artifacts,
|
|
763
|
+
fullDuplex,
|
|
764
|
+
hasAudioSource,
|
|
714
765
|
sessionId,
|
|
715
766
|
client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
|
|
716
767
|
});
|
|
@@ -4,6 +4,7 @@ interface TranscriptEntry {
|
|
|
4
4
|
readonly role: 'user' | 'assistant';
|
|
5
5
|
readonly content: string;
|
|
6
6
|
readonly timestamp: number;
|
|
7
|
+
readonly isFinal?: boolean;
|
|
7
8
|
readonly turnId?: string;
|
|
8
9
|
/**
|
|
9
10
|
* The reply is still being spoken, so `content` holds the clauses committed so
|
|
@@ -74,6 +75,10 @@ interface VoiceSnapshot {
|
|
|
74
75
|
}
|
|
75
76
|
interface VoiceClientOptions {
|
|
76
77
|
agentId: string;
|
|
78
|
+
/** Disable when the transcript consumer only supports sequential pipeline turns. */
|
|
79
|
+
fullDuplex?: boolean;
|
|
80
|
+
/** Optional custom audio source; the client owns and stops the returned tracks. */
|
|
81
|
+
audioSource?: () => Promise<MediaStream>;
|
|
77
82
|
/** Browser client token, or a getter called once per call to obtain a fresh token. */
|
|
78
83
|
clientToken: string | (() => string | Promise<string>);
|
|
79
84
|
/** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
|
|
@@ -4,6 +4,7 @@ interface TranscriptEntry {
|
|
|
4
4
|
readonly role: 'user' | 'assistant';
|
|
5
5
|
readonly content: string;
|
|
6
6
|
readonly timestamp: number;
|
|
7
|
+
readonly isFinal?: boolean;
|
|
7
8
|
readonly turnId?: string;
|
|
8
9
|
/**
|
|
9
10
|
* The reply is still being spoken, so `content` holds the clauses committed so
|
|
@@ -74,6 +75,10 @@ interface VoiceSnapshot {
|
|
|
74
75
|
}
|
|
75
76
|
interface VoiceClientOptions {
|
|
76
77
|
agentId: string;
|
|
78
|
+
/** Disable when the transcript consumer only supports sequential pipeline turns. */
|
|
79
|
+
fullDuplex?: boolean;
|
|
80
|
+
/** Optional custom audio source; the client owns and stops the returned tracks. */
|
|
81
|
+
audioSource?: () => Promise<MediaStream>;
|
|
77
82
|
/** Browser client token, or a getter called once per call to obtain a fresh token. */
|
|
78
83
|
clientToken: string | (() => string | Promise<string>);
|
|
79
84
|
/** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
|