@runtypelabs/voice 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,6 +54,11 @@ alongside `transcript`.
54
54
  Changing `clientToken` affects the next call without interrupting the current one.
55
55
  Startup and connection failures populate `error` and `status: 'error'`.
56
56
 
57
+ GPT-Live requires the default full-duplex browser protocol. Set `fullDuplex: false`
58
+ only for a client that cannot handle overlapping transcripts. GPT-Live does not
59
+ resume shared text/voice sessions; a call with `sessionId` or `visitorToken`
60
+ receives `VOICE_SHARED_SESSION_UNSUPPORTED`.
61
+
57
62
  ## Plain JavaScript
58
63
 
59
64
  ```ts
@@ -85,6 +90,10 @@ The adapter implements Persona 4.22's `VoiceProvider` API and delivers transcrip
85
90
  through `onTranscript`, avoiding duplicate text dispatches. Use it as the custom
86
91
  provider in the widget's `config`:
87
92
 
93
+ Persona 4.22 cannot handle GPT-Live's overlapping transcripts. The adapter does
94
+ not advertise full duplex, so it cannot connect to a GPT-Live agent. Use the
95
+ plain `VoiceClient` or the React hook for GPT-Live calls.
96
+
88
97
  ```ts
89
98
  import { createPersonaVoiceProvider } from '@runtypelabs/voice/persona'
90
99
 
@@ -125,8 +134,8 @@ embed's wire byte-identical.
125
134
  `event` is a unified `artifact_start`, `artifact_delta`, `artifact_update`, or
126
135
  `artifact_complete` frame. A start opens a `streaming` record, deltas append to
127
136
  its `content`, an update carries the component payload, and a complete settles
128
- it. Artifacts of a turn the caller stopped or cancelled are dropped, the same
129
- suppression the transcript applies. Ending the call keeps completed artifacts
137
+ it. Cancelling a turn drops its later artifacts. Playback-only Stop keeps
138
+ artifact delivery active. Ending the call keeps completed artifacts
130
139
  and drops half-streamed ones. Artifacts are never spoken: file bodies arrive
131
140
  only through these frames.
132
141
 
@@ -136,15 +145,32 @@ widget through its `upsertArtifact` handle:
136
145
 
137
146
  ```ts
138
147
  import { bindVoiceArtifactsToPersona, createPersonaVoiceProvider } from '@runtypelabs/voice/persona'
148
+ import { initAgentWidget } from '@runtypelabs/persona'
139
149
 
140
150
  const provider = createPersonaVoiceProvider({ agentId, clientToken, artifacts: true })
141
- const widget = initAgentWidget({ ...config, features: { artifacts: { enabled: true } } })
151
+ const widget = initAgentWidget({
152
+ target: document.getElementById('chat')!,
153
+ config: {
154
+ ...config,
155
+ features: { artifacts: { enabled: true } },
156
+ voiceRecognition: {
157
+ enabled: true,
158
+ provider: { type: 'custom', custom: () => provider },
159
+ },
160
+ },
161
+ })
142
162
  const release = bindVoiceArtifactsToPersona(provider, widget)
163
+
164
+ function dispose() {
165
+ release()
166
+ widget.destroy()
167
+ }
143
168
  ```
144
169
 
145
170
  The artifact id is the upsert id, so every frame updates the same record in
146
171
  place, and only the settled record writes a transcript block. The pane opens
147
172
  once on the first artifact; pass `{ showOnFirst: false }` to leave it closed.
173
+ Call `dispose()` when the host removes the widget.
148
174
  Persona's `features.artifacts.enabled` must be on, because `showArtifacts()` is
149
175
  a no-op otherwise. Call the returned function to stop delivery; the adapter's
150
176
  `disconnect()` releases artifact callbacks along with the rest.
package/dist/index.cjs CHANGED
@@ -35,13 +35,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
35
35
  this.readOffset = 0
36
36
  this.buffered = 0
37
37
  this.waiting = true
38
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
38
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
39
39
  this.eosSeen = false
40
+ this.continuous = false
40
41
  this.revision = 0
41
42
  this.port.onmessage = (e) => {
42
43
  const msg = e.data
43
44
  this.revision = msg.revision
44
- if (msg.type === 'push') {
45
+ if (msg.type === 'continuous') {
46
+ this.continuous = msg.enabled
47
+ } else if (msg.type === 'push') {
45
48
  this.eosSeen = false
46
49
  this.chunks.push(msg.samples)
47
50
  this.buffered += msg.samples.length
@@ -81,7 +84,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
81
84
  }
82
85
  if (this.buffered === 0) {
83
86
  this.waiting = true // mid-reply underrun: re-buffer silently
84
- if (this.eosSeen) {
87
+ if (this.eosSeen || this.continuous) {
85
88
  this.eosSeen = false
86
89
  this.port.postMessage({ type: 'drained', revision: this.revision })
87
90
  }
@@ -125,6 +128,9 @@ async function createPcmPlayer(onDrained) {
125
128
  if (samples.length === 0) return;
126
129
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
127
130
  },
131
+ setContinuousMode(enabled) {
132
+ node.port.postMessage({ type: "continuous", enabled, revision });
133
+ },
128
134
  endOfStream() {
129
135
  node.port.postMessage({ type: "eos", revision });
130
136
  },
@@ -251,6 +257,7 @@ var VoiceClient = class {
251
257
  committedAssistantTurnId = null;
252
258
  activeTurnId = null;
253
259
  suppressedArtifactTurnId = null;
260
+ fullDuplex = false;
254
261
  getSnapshot = () => this.snapshot;
255
262
  subscribe = (listener) => {
256
263
  this.listeners.add(listener);
@@ -301,10 +308,12 @@ var VoiceClient = class {
301
308
  if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
302
309
  throw new Error("Invalid voice visitor credential.");
303
310
  }
311
+ if (this.options.fullDuplex !== false)
312
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
304
313
  url.hash = "";
305
- const stream = await navigator.mediaDevices.getUserMedia({
314
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
306
315
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
307
- });
316
+ }));
308
317
  if (generation !== this.generation) {
309
318
  stream.getTracks().forEach((track) => track.stop());
310
319
  return;
@@ -399,6 +408,7 @@ var VoiceClient = class {
399
408
  });
400
409
  }
401
410
  cleanup() {
411
+ this.fullDuplex = false;
402
412
  this.generation += 1;
403
413
  if (this.processor) this.processor.onaudioprocess = null;
404
414
  this.processor?.disconnect();
@@ -535,6 +545,8 @@ var VoiceClient = class {
535
545
  }
536
546
  switch (msg.type) {
537
547
  case "session_config": {
548
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
549
+ if (this.fullDuplex) player.setContinuousMode?.(true);
538
550
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
539
551
  this.update({ interruptionMode: msg.interruptionMode });
540
552
  }
@@ -550,6 +562,32 @@ var VoiceClient = class {
550
562
  case "artifact":
551
563
  this.applyArtifactMessage(msg);
552
564
  break;
565
+ case "transcript_update": {
566
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
567
+ break;
568
+ if (msg.role === "assistant" && this.awaitingClear) break;
569
+ const entries = [...this.snapshot.transcript];
570
+ const index = entries.findIndex(
571
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
572
+ );
573
+ const entry = {
574
+ role: msg.role,
575
+ content: msg.text,
576
+ turnId: msg.turnId,
577
+ isFinal: msg.final === true,
578
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
579
+ };
580
+ if (index >= 0) entries[index] = entry;
581
+ else entries.push(entry);
582
+ this.update({ transcript: entries, interimTranscript: null });
583
+ break;
584
+ }
585
+ case "delegation_started":
586
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
587
+ break;
588
+ case "delegation_completed":
589
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
590
+ break;
553
591
  case "transcript_interim":
554
592
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
555
593
  break;
@@ -659,14 +697,14 @@ var VoiceClient = class {
659
697
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
660
698
  this.processor = processor;
661
699
  processor.onaudioprocess = (event) => {
662
- if (generation !== this.generation || this.snapshot.isMuted) return;
700
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
663
701
  const input = event.inputBuffer.getChannelData(0);
664
702
  let sum = 0;
665
703
  for (const sample of input) sum += sample * sample;
666
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
704
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
667
705
  if (socket.readyState !== WebSocket.OPEN) return;
668
706
  const pcm = new Int16Array(input.length);
669
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
707
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
670
708
  socket.send(pcm.buffer);
671
709
  return;
672
710
  }
package/dist/index.d.cts CHANGED
@@ -1,5 +1,5 @@
1
- import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-DaXyPeiV.cjs';
2
- export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-DaXyPeiV.cjs';
1
+ import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-Dkh4LxEA.cjs';
2
+ export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-Dkh4LxEA.cjs';
3
3
 
4
4
  /** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
5
5
  declare class VoiceClient {
@@ -29,6 +29,7 @@ declare class VoiceClient {
29
29
  private committedAssistantTurnId;
30
30
  private activeTurnId;
31
31
  private suppressedArtifactTurnId;
32
+ private fullDuplex;
32
33
  constructor(options: VoiceClientOptions);
33
34
  getSnapshot: () => VoiceSnapshot;
34
35
  subscribe: (listener: () => void) => (() => void);
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-DaXyPeiV.js';
2
- export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-DaXyPeiV.js';
1
+ import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-Dkh4LxEA.js';
2
+ export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-Dkh4LxEA.js';
3
3
 
4
4
  /** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
5
5
  declare class VoiceClient {
@@ -29,6 +29,7 @@ declare class VoiceClient {
29
29
  private committedAssistantTurnId;
30
30
  private activeTurnId;
31
31
  private suppressedArtifactTurnId;
32
+ private fullDuplex;
32
33
  constructor(options: VoiceClientOptions);
33
34
  getSnapshot: () => VoiceSnapshot;
34
35
  subscribe: (listener: () => void) => (() => void);
package/dist/index.js CHANGED
@@ -9,13 +9,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
9
9
  this.readOffset = 0
10
10
  this.buffered = 0
11
11
  this.waiting = true
12
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
12
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
13
13
  this.eosSeen = false
14
+ this.continuous = false
14
15
  this.revision = 0
15
16
  this.port.onmessage = (e) => {
16
17
  const msg = e.data
17
18
  this.revision = msg.revision
18
- if (msg.type === 'push') {
19
+ if (msg.type === 'continuous') {
20
+ this.continuous = msg.enabled
21
+ } else if (msg.type === 'push') {
19
22
  this.eosSeen = false
20
23
  this.chunks.push(msg.samples)
21
24
  this.buffered += msg.samples.length
@@ -55,7 +58,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
55
58
  }
56
59
  if (this.buffered === 0) {
57
60
  this.waiting = true // mid-reply underrun: re-buffer silently
58
- if (this.eosSeen) {
61
+ if (this.eosSeen || this.continuous) {
59
62
  this.eosSeen = false
60
63
  this.port.postMessage({ type: 'drained', revision: this.revision })
61
64
  }
@@ -99,6 +102,9 @@ async function createPcmPlayer(onDrained) {
99
102
  if (samples.length === 0) return;
100
103
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
101
104
  },
105
+ setContinuousMode(enabled) {
106
+ node.port.postMessage({ type: "continuous", enabled, revision });
107
+ },
102
108
  endOfStream() {
103
109
  node.port.postMessage({ type: "eos", revision });
104
110
  },
@@ -225,6 +231,7 @@ var VoiceClient = class {
225
231
  committedAssistantTurnId = null;
226
232
  activeTurnId = null;
227
233
  suppressedArtifactTurnId = null;
234
+ fullDuplex = false;
228
235
  getSnapshot = () => this.snapshot;
229
236
  subscribe = (listener) => {
230
237
  this.listeners.add(listener);
@@ -275,10 +282,12 @@ var VoiceClient = class {
275
282
  if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
276
283
  throw new Error("Invalid voice visitor credential.");
277
284
  }
285
+ if (this.options.fullDuplex !== false)
286
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
278
287
  url.hash = "";
279
- const stream = await navigator.mediaDevices.getUserMedia({
288
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
280
289
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
281
- });
290
+ }));
282
291
  if (generation !== this.generation) {
283
292
  stream.getTracks().forEach((track) => track.stop());
284
293
  return;
@@ -373,6 +382,7 @@ var VoiceClient = class {
373
382
  });
374
383
  }
375
384
  cleanup() {
385
+ this.fullDuplex = false;
376
386
  this.generation += 1;
377
387
  if (this.processor) this.processor.onaudioprocess = null;
378
388
  this.processor?.disconnect();
@@ -509,6 +519,8 @@ var VoiceClient = class {
509
519
  }
510
520
  switch (msg.type) {
511
521
  case "session_config": {
522
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
523
+ if (this.fullDuplex) player.setContinuousMode?.(true);
512
524
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
513
525
  this.update({ interruptionMode: msg.interruptionMode });
514
526
  }
@@ -524,6 +536,32 @@ var VoiceClient = class {
524
536
  case "artifact":
525
537
  this.applyArtifactMessage(msg);
526
538
  break;
539
+ case "transcript_update": {
540
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
541
+ break;
542
+ if (msg.role === "assistant" && this.awaitingClear) break;
543
+ const entries = [...this.snapshot.transcript];
544
+ const index = entries.findIndex(
545
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
546
+ );
547
+ const entry = {
548
+ role: msg.role,
549
+ content: msg.text,
550
+ turnId: msg.turnId,
551
+ isFinal: msg.final === true,
552
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
553
+ };
554
+ if (index >= 0) entries[index] = entry;
555
+ else entries.push(entry);
556
+ this.update({ transcript: entries, interimTranscript: null });
557
+ break;
558
+ }
559
+ case "delegation_started":
560
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
561
+ break;
562
+ case "delegation_completed":
563
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
564
+ break;
527
565
  case "transcript_interim":
528
566
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
529
567
  break;
@@ -633,14 +671,14 @@ var VoiceClient = class {
633
671
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
634
672
  this.processor = processor;
635
673
  processor.onaudioprocess = (event) => {
636
- if (generation !== this.generation || this.snapshot.isMuted) return;
674
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
637
675
  const input = event.inputBuffer.getChannelData(0);
638
676
  let sum = 0;
639
677
  for (const sample of input) sum += sample * sample;
640
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
678
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
641
679
  if (socket.readyState !== WebSocket.OPEN) return;
642
680
  const pcm = new Int16Array(input.length);
643
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
681
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
644
682
  socket.send(pcm.buffer);
645
683
  return;
646
684
  }
package/dist/persona.cjs CHANGED
@@ -36,13 +36,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
36
36
  this.readOffset = 0
37
37
  this.buffered = 0
38
38
  this.waiting = true
39
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
39
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
40
40
  this.eosSeen = false
41
+ this.continuous = false
41
42
  this.revision = 0
42
43
  this.port.onmessage = (e) => {
43
44
  const msg = e.data
44
45
  this.revision = msg.revision
45
- if (msg.type === 'push') {
46
+ if (msg.type === 'continuous') {
47
+ this.continuous = msg.enabled
48
+ } else if (msg.type === 'push') {
46
49
  this.eosSeen = false
47
50
  this.chunks.push(msg.samples)
48
51
  this.buffered += msg.samples.length
@@ -82,7 +85,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
82
85
  }
83
86
  if (this.buffered === 0) {
84
87
  this.waiting = true // mid-reply underrun: re-buffer silently
85
- if (this.eosSeen) {
88
+ if (this.eosSeen || this.continuous) {
86
89
  this.eosSeen = false
87
90
  this.port.postMessage({ type: 'drained', revision: this.revision })
88
91
  }
@@ -126,6 +129,9 @@ async function createPcmPlayer(onDrained) {
126
129
  if (samples.length === 0) return;
127
130
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
128
131
  },
132
+ setContinuousMode(enabled) {
133
+ node.port.postMessage({ type: "continuous", enabled, revision });
134
+ },
129
135
  endOfStream() {
130
136
  node.port.postMessage({ type: "eos", revision });
131
137
  },
@@ -252,6 +258,7 @@ var VoiceClient = class {
252
258
  committedAssistantTurnId = null;
253
259
  activeTurnId = null;
254
260
  suppressedArtifactTurnId = null;
261
+ fullDuplex = false;
255
262
  getSnapshot = () => this.snapshot;
256
263
  subscribe = (listener) => {
257
264
  this.listeners.add(listener);
@@ -302,10 +309,12 @@ var VoiceClient = class {
302
309
  if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
303
310
  throw new Error("Invalid voice visitor credential.");
304
311
  }
312
+ if (this.options.fullDuplex !== false)
313
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
305
314
  url.hash = "";
306
- const stream = await navigator.mediaDevices.getUserMedia({
315
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
307
316
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
308
- });
317
+ }));
309
318
  if (generation !== this.generation) {
310
319
  stream.getTracks().forEach((track) => track.stop());
311
320
  return;
@@ -400,6 +409,7 @@ var VoiceClient = class {
400
409
  });
401
410
  }
402
411
  cleanup() {
412
+ this.fullDuplex = false;
403
413
  this.generation += 1;
404
414
  if (this.processor) this.processor.onaudioprocess = null;
405
415
  this.processor?.disconnect();
@@ -536,6 +546,8 @@ var VoiceClient = class {
536
546
  }
537
547
  switch (msg.type) {
538
548
  case "session_config": {
549
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
550
+ if (this.fullDuplex) player.setContinuousMode?.(true);
539
551
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
540
552
  this.update({ interruptionMode: msg.interruptionMode });
541
553
  }
@@ -551,6 +563,32 @@ var VoiceClient = class {
551
563
  case "artifact":
552
564
  this.applyArtifactMessage(msg);
553
565
  break;
566
+ case "transcript_update": {
567
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
568
+ break;
569
+ if (msg.role === "assistant" && this.awaitingClear) break;
570
+ const entries = [...this.snapshot.transcript];
571
+ const index = entries.findIndex(
572
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
573
+ );
574
+ const entry = {
575
+ role: msg.role,
576
+ content: msg.text,
577
+ turnId: msg.turnId,
578
+ isFinal: msg.final === true,
579
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
580
+ };
581
+ if (index >= 0) entries[index] = entry;
582
+ else entries.push(entry);
583
+ this.update({ transcript: entries, interimTranscript: null });
584
+ break;
585
+ }
586
+ case "delegation_started":
587
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
588
+ break;
589
+ case "delegation_completed":
590
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
591
+ break;
554
592
  case "transcript_interim":
555
593
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
556
594
  break;
@@ -660,14 +698,14 @@ var VoiceClient = class {
660
698
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
661
699
  this.processor = processor;
662
700
  processor.onaudioprocess = (event) => {
663
- if (generation !== this.generation || this.snapshot.isMuted) return;
701
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
664
702
  const input = event.inputBuffer.getChannelData(0);
665
703
  let sum = 0;
666
704
  for (const sample of input) sum += sample * sample;
667
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
705
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
668
706
  if (socket.readyState !== WebSocket.OPEN) return;
669
707
  const pcm = new Int16Array(input.length);
670
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
708
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
671
709
  socket.send(pcm.buffer);
672
710
  return;
673
711
  }
@@ -684,7 +722,7 @@ var VoiceClient = class {
684
722
 
685
723
  // src/persona.ts
686
724
  function createPersonaVoiceProvider(options) {
687
- const client = new VoiceClient(options);
725
+ const client = new VoiceClient({ ...options, fullDuplex: false });
688
726
  const statusCallbacks = /* @__PURE__ */ new Set();
689
727
  const errorCallbacks = /* @__PURE__ */ new Set();
690
728
  const transcriptCallbacks = /* @__PURE__ */ new Set();
@@ -720,7 +758,7 @@ function createPersonaVoiceProvider(options) {
720
758
  });
721
759
  snapshot.transcript.forEach((entry, index) => {
722
760
  if (entry === before.transcript[index]) return;
723
- const isFinal = entry.partial !== true;
761
+ const isFinal = entry.partial !== true && entry.isFinal !== false;
724
762
  for (const callback of transcriptCallbacks) {
725
763
  if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
726
764
  else callback(entry.role, entry.content, isFinal);
@@ -1,5 +1,5 @@
1
1
  import { AgentWidgetInitHandle, VoiceProvider } from '@runtypelabs/persona';
2
- import { b as VoiceArtifact, V as VoiceClientOptions } from './types-DaXyPeiV.cjs';
2
+ import { b as VoiceArtifact, V as VoiceClientOptions } from './types-Dkh4LxEA.cjs';
3
3
 
4
4
  type ArtifactCallback = (artifact: VoiceArtifact) => void;
5
5
  /**
package/dist/persona.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { AgentWidgetInitHandle, VoiceProvider } from '@runtypelabs/persona';
2
- import { b as VoiceArtifact, V as VoiceClientOptions } from './types-DaXyPeiV.js';
2
+ import { b as VoiceArtifact, V as VoiceClientOptions } from './types-Dkh4LxEA.js';
3
3
 
4
4
  type ArtifactCallback = (artifact: VoiceArtifact) => void;
5
5
  /**
package/dist/persona.js CHANGED
@@ -9,13 +9,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
9
9
  this.readOffset = 0
10
10
  this.buffered = 0
11
11
  this.waiting = true
12
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
12
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
13
13
  this.eosSeen = false
14
+ this.continuous = false
14
15
  this.revision = 0
15
16
  this.port.onmessage = (e) => {
16
17
  const msg = e.data
17
18
  this.revision = msg.revision
18
- if (msg.type === 'push') {
19
+ if (msg.type === 'continuous') {
20
+ this.continuous = msg.enabled
21
+ } else if (msg.type === 'push') {
19
22
  this.eosSeen = false
20
23
  this.chunks.push(msg.samples)
21
24
  this.buffered += msg.samples.length
@@ -55,7 +58,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
55
58
  }
56
59
  if (this.buffered === 0) {
57
60
  this.waiting = true // mid-reply underrun: re-buffer silently
58
- if (this.eosSeen) {
61
+ if (this.eosSeen || this.continuous) {
59
62
  this.eosSeen = false
60
63
  this.port.postMessage({ type: 'drained', revision: this.revision })
61
64
  }
@@ -99,6 +102,9 @@ async function createPcmPlayer(onDrained) {
99
102
  if (samples.length === 0) return;
100
103
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
101
104
  },
105
+ setContinuousMode(enabled) {
106
+ node.port.postMessage({ type: "continuous", enabled, revision });
107
+ },
102
108
  endOfStream() {
103
109
  node.port.postMessage({ type: "eos", revision });
104
110
  },
@@ -225,6 +231,7 @@ var VoiceClient = class {
225
231
  committedAssistantTurnId = null;
226
232
  activeTurnId = null;
227
233
  suppressedArtifactTurnId = null;
234
+ fullDuplex = false;
228
235
  getSnapshot = () => this.snapshot;
229
236
  subscribe = (listener) => {
230
237
  this.listeners.add(listener);
@@ -275,10 +282,12 @@ var VoiceClient = class {
275
282
  if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
276
283
  throw new Error("Invalid voice visitor credential.");
277
284
  }
285
+ if (this.options.fullDuplex !== false)
286
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
278
287
  url.hash = "";
279
- const stream = await navigator.mediaDevices.getUserMedia({
288
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
280
289
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
281
- });
290
+ }));
282
291
  if (generation !== this.generation) {
283
292
  stream.getTracks().forEach((track) => track.stop());
284
293
  return;
@@ -373,6 +382,7 @@ var VoiceClient = class {
373
382
  });
374
383
  }
375
384
  cleanup() {
385
+ this.fullDuplex = false;
376
386
  this.generation += 1;
377
387
  if (this.processor) this.processor.onaudioprocess = null;
378
388
  this.processor?.disconnect();
@@ -509,6 +519,8 @@ var VoiceClient = class {
509
519
  }
510
520
  switch (msg.type) {
511
521
  case "session_config": {
522
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
523
+ if (this.fullDuplex) player.setContinuousMode?.(true);
512
524
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
513
525
  this.update({ interruptionMode: msg.interruptionMode });
514
526
  }
@@ -524,6 +536,32 @@ var VoiceClient = class {
524
536
  case "artifact":
525
537
  this.applyArtifactMessage(msg);
526
538
  break;
539
+ case "transcript_update": {
540
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
541
+ break;
542
+ if (msg.role === "assistant" && this.awaitingClear) break;
543
+ const entries = [...this.snapshot.transcript];
544
+ const index = entries.findIndex(
545
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
546
+ );
547
+ const entry = {
548
+ role: msg.role,
549
+ content: msg.text,
550
+ turnId: msg.turnId,
551
+ isFinal: msg.final === true,
552
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
553
+ };
554
+ if (index >= 0) entries[index] = entry;
555
+ else entries.push(entry);
556
+ this.update({ transcript: entries, interimTranscript: null });
557
+ break;
558
+ }
559
+ case "delegation_started":
560
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
561
+ break;
562
+ case "delegation_completed":
563
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
564
+ break;
527
565
  case "transcript_interim":
528
566
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
529
567
  break;
@@ -633,14 +671,14 @@ var VoiceClient = class {
633
671
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
634
672
  this.processor = processor;
635
673
  processor.onaudioprocess = (event) => {
636
- if (generation !== this.generation || this.snapshot.isMuted) return;
674
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
637
675
  const input = event.inputBuffer.getChannelData(0);
638
676
  let sum = 0;
639
677
  for (const sample of input) sum += sample * sample;
640
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
678
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
641
679
  if (socket.readyState !== WebSocket.OPEN) return;
642
680
  const pcm = new Int16Array(input.length);
643
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
681
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
644
682
  socket.send(pcm.buffer);
645
683
  return;
646
684
  }
@@ -657,7 +695,7 @@ var VoiceClient = class {
657
695
 
658
696
  // src/persona.ts
659
697
  function createPersonaVoiceProvider(options) {
660
- const client = new VoiceClient(options);
698
+ const client = new VoiceClient({ ...options, fullDuplex: false });
661
699
  const statusCallbacks = /* @__PURE__ */ new Set();
662
700
  const errorCallbacks = /* @__PURE__ */ new Set();
663
701
  const transcriptCallbacks = /* @__PURE__ */ new Set();
@@ -693,7 +731,7 @@ function createPersonaVoiceProvider(options) {
693
731
  });
694
732
  snapshot.transcript.forEach((entry, index) => {
695
733
  if (entry === before.transcript[index]) return;
696
- const isFinal = entry.partial !== true;
734
+ const isFinal = entry.partial !== true && entry.isFinal !== false;
697
735
  for (const callback of transcriptCallbacks) {
698
736
  if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
699
737
  else callback(entry.role, entry.content, isFinal);
package/dist/react.cjs CHANGED
@@ -37,13 +37,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
37
37
  this.readOffset = 0
38
38
  this.buffered = 0
39
39
  this.waiting = true
40
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
40
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
41
41
  this.eosSeen = false
42
+ this.continuous = false
42
43
  this.revision = 0
43
44
  this.port.onmessage = (e) => {
44
45
  const msg = e.data
45
46
  this.revision = msg.revision
46
- if (msg.type === 'push') {
47
+ if (msg.type === 'continuous') {
48
+ this.continuous = msg.enabled
49
+ } else if (msg.type === 'push') {
47
50
  this.eosSeen = false
48
51
  this.chunks.push(msg.samples)
49
52
  this.buffered += msg.samples.length
@@ -83,7 +86,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
83
86
  }
84
87
  if (this.buffered === 0) {
85
88
  this.waiting = true // mid-reply underrun: re-buffer silently
86
- if (this.eosSeen) {
89
+ if (this.eosSeen || this.continuous) {
87
90
  this.eosSeen = false
88
91
  this.port.postMessage({ type: 'drained', revision: this.revision })
89
92
  }
@@ -127,6 +130,9 @@ async function createPcmPlayer(onDrained) {
127
130
  if (samples.length === 0) return;
128
131
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
129
132
  },
133
+ setContinuousMode(enabled) {
134
+ node.port.postMessage({ type: "continuous", enabled, revision });
135
+ },
130
136
  endOfStream() {
131
137
  node.port.postMessage({ type: "eos", revision });
132
138
  },
@@ -253,6 +259,7 @@ var VoiceClient = class {
253
259
  committedAssistantTurnId = null;
254
260
  activeTurnId = null;
255
261
  suppressedArtifactTurnId = null;
262
+ fullDuplex = false;
256
263
  getSnapshot = () => this.snapshot;
257
264
  subscribe = (listener) => {
258
265
  this.listeners.add(listener);
@@ -303,10 +310,12 @@ var VoiceClient = class {
303
310
  if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
304
311
  throw new Error("Invalid voice visitor credential.");
305
312
  }
313
+ if (this.options.fullDuplex !== false)
314
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
306
315
  url.hash = "";
307
- const stream = await navigator.mediaDevices.getUserMedia({
316
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
308
317
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
309
- });
318
+ }));
310
319
  if (generation !== this.generation) {
311
320
  stream.getTracks().forEach((track) => track.stop());
312
321
  return;
@@ -401,6 +410,7 @@ var VoiceClient = class {
401
410
  });
402
411
  }
403
412
  cleanup() {
413
+ this.fullDuplex = false;
404
414
  this.generation += 1;
405
415
  if (this.processor) this.processor.onaudioprocess = null;
406
416
  this.processor?.disconnect();
@@ -537,6 +547,8 @@ var VoiceClient = class {
537
547
  }
538
548
  switch (msg.type) {
539
549
  case "session_config": {
550
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
551
+ if (this.fullDuplex) player.setContinuousMode?.(true);
540
552
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
541
553
  this.update({ interruptionMode: msg.interruptionMode });
542
554
  }
@@ -552,6 +564,32 @@ var VoiceClient = class {
552
564
  case "artifact":
553
565
  this.applyArtifactMessage(msg);
554
566
  break;
567
+ case "transcript_update": {
568
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
569
+ break;
570
+ if (msg.role === "assistant" && this.awaitingClear) break;
571
+ const entries = [...this.snapshot.transcript];
572
+ const index = entries.findIndex(
573
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
574
+ );
575
+ const entry = {
576
+ role: msg.role,
577
+ content: msg.text,
578
+ turnId: msg.turnId,
579
+ isFinal: msg.final === true,
580
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
581
+ };
582
+ if (index >= 0) entries[index] = entry;
583
+ else entries.push(entry);
584
+ this.update({ transcript: entries, interimTranscript: null });
585
+ break;
586
+ }
587
+ case "delegation_started":
588
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
589
+ break;
590
+ case "delegation_completed":
591
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
592
+ break;
555
593
  case "transcript_interim":
556
594
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
557
595
  break;
@@ -661,14 +699,14 @@ var VoiceClient = class {
661
699
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
662
700
  this.processor = processor;
663
701
  processor.onaudioprocess = (event) => {
664
- if (generation !== this.generation || this.snapshot.isMuted) return;
702
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
665
703
  const input = event.inputBuffer.getChannelData(0);
666
704
  let sum = 0;
667
705
  for (const sample of input) sum += sample * sample;
668
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
706
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
669
707
  if (socket.readyState !== WebSocket.OPEN) return;
670
708
  const pcm = new Int16Array(input.length);
671
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
709
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
672
710
  socket.send(pcm.buffer);
673
711
  return;
674
712
  }
@@ -691,7 +729,9 @@ function useVoiceClient({
691
729
  artifacts,
692
730
  sessionId,
693
731
  visitorToken,
694
- onSession
732
+ onSession,
733
+ fullDuplex,
734
+ audioSource
695
735
  }) {
696
736
  const sessionRef = (0, import_react.useRef)(sessionId);
697
737
  (0, import_react.useEffect)(() => {
@@ -700,10 +740,15 @@ function useVoiceClient({
700
740
  const tokenRef = (0, import_react.useRef)(clientToken);
701
741
  const visitorRef = (0, import_react.useRef)(visitorToken);
702
742
  const sessionCallbackRef = (0, import_react.useRef)(onSession);
743
+ const audioSourceRef = (0, import_react.useRef)(audioSource);
744
+ const hasAudioSource = audioSource !== void 0;
703
745
  (0, import_react.useEffect)(() => {
704
746
  visitorRef.current = visitorToken;
705
747
  sessionCallbackRef.current = onSession;
706
748
  }, [visitorToken, onSession]);
749
+ (0, import_react.useEffect)(() => {
750
+ audioSourceRef.current = audioSource;
751
+ }, [audioSource]);
707
752
  (0, import_react.useEffect)(() => {
708
753
  tokenRef.current = clientToken;
709
754
  }, [clientToken]);
@@ -711,6 +756,8 @@ function useVoiceClient({
711
756
  agentId,
712
757
  apiUrl,
713
758
  artifacts,
759
+ fullDuplex,
760
+ ...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
714
761
  get sessionId() {
715
762
  return sessionRef.current;
716
763
  },
@@ -724,16 +771,20 @@ function useVoiceClient({
724
771
  agentId,
725
772
  apiUrl,
726
773
  artifacts,
774
+ fullDuplex,
775
+ hasAudioSource,
727
776
  sessionId,
728
777
  client: createClient()
729
778
  }));
730
- const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts;
779
+ const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
731
780
  if (configurationChanged || binding.sessionId !== sessionId) {
732
781
  const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
733
782
  setBinding({
734
783
  agentId,
735
784
  apiUrl,
736
785
  artifacts,
786
+ fullDuplex,
787
+ hasAudioSource,
737
788
  sessionId,
738
789
  client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
739
790
  });
package/dist/react.d.cts CHANGED
@@ -1,8 +1,8 @@
1
- import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-DaXyPeiV.cjs';
2
- export { a as VoiceSnapshot } from './types-DaXyPeiV.cjs';
1
+ import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-Dkh4LxEA.cjs';
2
+ export { a as VoiceSnapshot } from './types-Dkh4LxEA.cjs';
3
3
 
4
4
  /** Subscribe to a voice client and end its call when the component unmounts or changes agent. */
5
- declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, }: VoiceClientOptions): {
5
+ declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, fullDuplex, audioSource, }: VoiceClientOptions): {
6
6
  startCall: (tokenOverride?: string) => Promise<void>;
7
7
  endCall: () => void;
8
8
  resetSession: () => void;
package/dist/react.d.ts CHANGED
@@ -1,8 +1,8 @@
1
- import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-DaXyPeiV.js';
2
- export { a as VoiceSnapshot } from './types-DaXyPeiV.js';
1
+ import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-Dkh4LxEA.js';
2
+ export { a as VoiceSnapshot } from './types-Dkh4LxEA.js';
3
3
 
4
4
  /** Subscribe to a voice client and end its call when the component unmounts or changes agent. */
5
- declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, }: VoiceClientOptions): {
5
+ declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, fullDuplex, audioSource, }: VoiceClientOptions): {
6
6
  startCall: (tokenOverride?: string) => Promise<void>;
7
7
  endCall: () => void;
8
8
  resetSession: () => void;
package/dist/react.js CHANGED
@@ -14,13 +14,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
14
14
  this.readOffset = 0
15
15
  this.buffered = 0
16
16
  this.waiting = true
17
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
17
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
18
18
  this.eosSeen = false
19
+ this.continuous = false
19
20
  this.revision = 0
20
21
  this.port.onmessage = (e) => {
21
22
  const msg = e.data
22
23
  this.revision = msg.revision
23
- if (msg.type === 'push') {
24
+ if (msg.type === 'continuous') {
25
+ this.continuous = msg.enabled
26
+ } else if (msg.type === 'push') {
24
27
  this.eosSeen = false
25
28
  this.chunks.push(msg.samples)
26
29
  this.buffered += msg.samples.length
@@ -60,7 +63,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
60
63
  }
61
64
  if (this.buffered === 0) {
62
65
  this.waiting = true // mid-reply underrun: re-buffer silently
63
- if (this.eosSeen) {
66
+ if (this.eosSeen || this.continuous) {
64
67
  this.eosSeen = false
65
68
  this.port.postMessage({ type: 'drained', revision: this.revision })
66
69
  }
@@ -104,6 +107,9 @@ async function createPcmPlayer(onDrained) {
104
107
  if (samples.length === 0) return;
105
108
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
106
109
  },
110
+ setContinuousMode(enabled) {
111
+ node.port.postMessage({ type: "continuous", enabled, revision });
112
+ },
107
113
  endOfStream() {
108
114
  node.port.postMessage({ type: "eos", revision });
109
115
  },
@@ -230,6 +236,7 @@ var VoiceClient = class {
230
236
  committedAssistantTurnId = null;
231
237
  activeTurnId = null;
232
238
  suppressedArtifactTurnId = null;
239
+ fullDuplex = false;
233
240
  getSnapshot = () => this.snapshot;
234
241
  subscribe = (listener) => {
235
242
  this.listeners.add(listener);
@@ -280,10 +287,12 @@ var VoiceClient = class {
280
287
  if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
281
288
  throw new Error("Invalid voice visitor credential.");
282
289
  }
290
+ if (this.options.fullDuplex !== false)
291
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
283
292
  url.hash = "";
284
- const stream = await navigator.mediaDevices.getUserMedia({
293
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
285
294
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
286
- });
295
+ }));
287
296
  if (generation !== this.generation) {
288
297
  stream.getTracks().forEach((track) => track.stop());
289
298
  return;
@@ -378,6 +387,7 @@ var VoiceClient = class {
378
387
  });
379
388
  }
380
389
  cleanup() {
390
+ this.fullDuplex = false;
381
391
  this.generation += 1;
382
392
  if (this.processor) this.processor.onaudioprocess = null;
383
393
  this.processor?.disconnect();
@@ -514,6 +524,8 @@ var VoiceClient = class {
514
524
  }
515
525
  switch (msg.type) {
516
526
  case "session_config": {
527
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
528
+ if (this.fullDuplex) player.setContinuousMode?.(true);
517
529
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
518
530
  this.update({ interruptionMode: msg.interruptionMode });
519
531
  }
@@ -529,6 +541,32 @@ var VoiceClient = class {
529
541
  case "artifact":
530
542
  this.applyArtifactMessage(msg);
531
543
  break;
544
+ case "transcript_update": {
545
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
546
+ break;
547
+ if (msg.role === "assistant" && this.awaitingClear) break;
548
+ const entries = [...this.snapshot.transcript];
549
+ const index = entries.findIndex(
550
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
551
+ );
552
+ const entry = {
553
+ role: msg.role,
554
+ content: msg.text,
555
+ turnId: msg.turnId,
556
+ isFinal: msg.final === true,
557
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
558
+ };
559
+ if (index >= 0) entries[index] = entry;
560
+ else entries.push(entry);
561
+ this.update({ transcript: entries, interimTranscript: null });
562
+ break;
563
+ }
564
+ case "delegation_started":
565
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
566
+ break;
567
+ case "delegation_completed":
568
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
569
+ break;
532
570
  case "transcript_interim":
533
571
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
534
572
  break;
@@ -638,14 +676,14 @@ var VoiceClient = class {
638
676
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
639
677
  this.processor = processor;
640
678
  processor.onaudioprocess = (event) => {
641
- if (generation !== this.generation || this.snapshot.isMuted) return;
679
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
642
680
  const input = event.inputBuffer.getChannelData(0);
643
681
  let sum = 0;
644
682
  for (const sample of input) sum += sample * sample;
645
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
683
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
646
684
  if (socket.readyState !== WebSocket.OPEN) return;
647
685
  const pcm = new Int16Array(input.length);
648
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
686
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
649
687
  socket.send(pcm.buffer);
650
688
  return;
651
689
  }
@@ -668,7 +706,9 @@ function useVoiceClient({
668
706
  artifacts,
669
707
  sessionId,
670
708
  visitorToken,
671
- onSession
709
+ onSession,
710
+ fullDuplex,
711
+ audioSource
672
712
  }) {
673
713
  const sessionRef = useRef(sessionId);
674
714
  useEffect(() => {
@@ -677,10 +717,15 @@ function useVoiceClient({
677
717
  const tokenRef = useRef(clientToken);
678
718
  const visitorRef = useRef(visitorToken);
679
719
  const sessionCallbackRef = useRef(onSession);
720
+ const audioSourceRef = useRef(audioSource);
721
+ const hasAudioSource = audioSource !== void 0;
680
722
  useEffect(() => {
681
723
  visitorRef.current = visitorToken;
682
724
  sessionCallbackRef.current = onSession;
683
725
  }, [visitorToken, onSession]);
726
+ useEffect(() => {
727
+ audioSourceRef.current = audioSource;
728
+ }, [audioSource]);
684
729
  useEffect(() => {
685
730
  tokenRef.current = clientToken;
686
731
  }, [clientToken]);
@@ -688,6 +733,8 @@ function useVoiceClient({
688
733
  agentId,
689
734
  apiUrl,
690
735
  artifacts,
736
+ fullDuplex,
737
+ ...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
691
738
  get sessionId() {
692
739
  return sessionRef.current;
693
740
  },
@@ -701,16 +748,20 @@ function useVoiceClient({
701
748
  agentId,
702
749
  apiUrl,
703
750
  artifacts,
751
+ fullDuplex,
752
+ hasAudioSource,
704
753
  sessionId,
705
754
  client: createClient()
706
755
  }));
707
- const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts;
756
+ const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
708
757
  if (configurationChanged || binding.sessionId !== sessionId) {
709
758
  const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
710
759
  setBinding({
711
760
  agentId,
712
761
  apiUrl,
713
762
  artifacts,
763
+ fullDuplex,
764
+ hasAudioSource,
714
765
  sessionId,
715
766
  client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
716
767
  });
@@ -4,6 +4,7 @@ interface TranscriptEntry {
4
4
  readonly role: 'user' | 'assistant';
5
5
  readonly content: string;
6
6
  readonly timestamp: number;
7
+ readonly isFinal?: boolean;
7
8
  readonly turnId?: string;
8
9
  /**
9
10
  * The reply is still being spoken, so `content` holds the clauses committed so
@@ -74,6 +75,10 @@ interface VoiceSnapshot {
74
75
  }
75
76
  interface VoiceClientOptions {
76
77
  agentId: string;
78
+ /** Disable when the transcript consumer only supports sequential pipeline turns. */
79
+ fullDuplex?: boolean;
80
+ /** Optional custom audio source; the client owns and stops the returned tracks. */
81
+ audioSource?: () => Promise<MediaStream>;
77
82
  /** Browser client token, or a getter called once per call to obtain a fresh token. */
78
83
  clientToken: string | (() => string | Promise<string>);
79
84
  /** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
@@ -4,6 +4,7 @@ interface TranscriptEntry {
4
4
  readonly role: 'user' | 'assistant';
5
5
  readonly content: string;
6
6
  readonly timestamp: number;
7
+ readonly isFinal?: boolean;
7
8
  readonly turnId?: string;
8
9
  /**
9
10
  * The reply is still being spoken, so `content` holds the clauses committed so
@@ -74,6 +75,10 @@ interface VoiceSnapshot {
74
75
  }
75
76
  interface VoiceClientOptions {
76
77
  agentId: string;
78
+ /** Disable when the transcript consumer only supports sequential pipeline turns. */
79
+ fullDuplex?: boolean;
80
+ /** Optional custom audio source; the client owns and stops the returned tracks. */
81
+ audioSource?: () => Promise<MediaStream>;
77
82
  /** Browser client token, or a getter called once per call to obtain a fresh token. */
78
83
  clientToken: string | (() => string | Promise<string>);
79
84
  /** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runtypelabs/voice",
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "description": "Runtype browser voice client with React and Persona adapters",
5
5
  "type": "module",
6
6
  "license": "MIT",