@runtypelabs/voice 0.2.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/react.js CHANGED
@@ -1,7 +1,7 @@
1
1
  "use client";
2
2
 
3
3
  // src/react.ts
4
- import { useEffect, useMemo, useRef, useSyncExternalStore } from "react";
4
+ import { useEffect, useRef, useState, useSyncExternalStore } from "react";
5
5
 
6
6
  // src/pcm-player.ts
7
7
  var PCM_SAMPLE_RATE = 24e3;
@@ -14,13 +14,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
14
14
  this.readOffset = 0
15
15
  this.buffered = 0
16
16
  this.waiting = true
17
- // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
17
+ // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
18
18
  this.eosSeen = false
19
+ this.continuous = false
19
20
  this.revision = 0
20
21
  this.port.onmessage = (e) => {
21
22
  const msg = e.data
22
23
  this.revision = msg.revision
23
- if (msg.type === 'push') {
24
+ if (msg.type === 'continuous') {
25
+ this.continuous = msg.enabled
26
+ } else if (msg.type === 'push') {
24
27
  this.eosSeen = false
25
28
  this.chunks.push(msg.samples)
26
29
  this.buffered += msg.samples.length
@@ -60,7 +63,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
60
63
  }
61
64
  if (this.buffered === 0) {
62
65
  this.waiting = true // mid-reply underrun: re-buffer silently
63
- if (this.eosSeen) {
66
+ if (this.eosSeen || this.continuous) {
64
67
  this.eosSeen = false
65
68
  this.port.postMessage({ type: 'drained', revision: this.revision })
66
69
  }
@@ -104,6 +107,9 @@ async function createPcmPlayer(onDrained) {
104
107
  if (samples.length === 0) return;
105
108
  node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
106
109
  },
110
+ setContinuousMode(enabled) {
111
+ node.port.postMessage({ type: "continuous", enabled, revision });
112
+ },
107
113
  endOfStream() {
108
114
  node.port.postMessage({ type: "eos", revision });
109
115
  },
@@ -148,9 +154,58 @@ function initialSnapshot() {
148
154
  error: null,
149
155
  errorDetails: void 0,
150
156
  interruptionMode: "none",
151
- canCancel: false
157
+ canCancel: false,
158
+ artifacts: [],
159
+ session: null
152
160
  };
153
161
  }
162
+ function readString(value) {
163
+ return typeof value === "string" && value ? value : void 0;
164
+ }
165
+ function readArtifactFile(value) {
166
+ if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
167
+ const file = value;
168
+ const path = readString(file.path);
169
+ const mimeType = readString(file.mimeType);
170
+ if (!path || !mimeType) return void 0;
171
+ const language = readString(file.language);
172
+ return { path, mimeType, ...language ? { language } : {} };
173
+ }
174
+ function applyArtifactFrame(existing, frame, identity) {
175
+ if (frame.type === "artifact_start") {
176
+ const title = readString(frame.title);
177
+ const component = readString(frame.component);
178
+ const file = readArtifactFile(frame.file);
179
+ return {
180
+ ...identity,
181
+ artifactType: frame.artifactType === "component" ? "component" : "markdown",
182
+ ...title ? { title } : {},
183
+ ...file ? { file } : {},
184
+ ...component ? { component } : {},
185
+ content: "",
186
+ status: "streaming"
187
+ };
188
+ }
189
+ if (!existing) return void 0;
190
+ if (frame.type === "artifact_delta") {
191
+ const delta = typeof frame.delta === "string" ? frame.delta : "";
192
+ return delta ? { ...existing, content: existing.content + delta } : void 0;
193
+ }
194
+ if (frame.type === "artifact_update") {
195
+ const component = readString(frame.component) ?? existing.component;
196
+ const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
197
+ return {
198
+ ...existing,
199
+ ...component ? { component } : {},
200
+ ...props ? { props } : {}
201
+ };
202
+ }
203
+ if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
204
+ return void 0;
205
+ }
206
+ function readTurnId(msg) {
207
+ return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
208
+ }
154
209
  var VoiceClient = class {
155
210
  constructor(options) {
156
211
  this.options = options;
@@ -171,6 +226,17 @@ var VoiceClient = class {
171
226
  stoppedResponse = false;
172
227
  responseEnded = true;
173
228
  hasPendingAudio = false;
229
+ /**
230
+ * The assistant entry built from clause captions for the turn being spoken. The
231
+ * authoritative transcript replaces it; a stop commits it, because it captions
232
+ * audio the caller already heard.
233
+ */
234
+ pendingAssistant = null;
235
+ /** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
236
+ committedAssistantTurnId = null;
237
+ activeTurnId = null;
238
+ suppressedArtifactTurnId = null;
239
+ fullDuplex = false;
174
240
  getSnapshot = () => this.snapshot;
175
241
  subscribe = (listener) => {
176
242
  this.listeners.add(listener);
@@ -192,7 +258,7 @@ var VoiceClient = class {
192
258
  if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
193
259
  this.cleanup();
194
260
  const generation = this.generation;
195
- this.update({ ...initialSnapshot(), status: "connecting" });
261
+ this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
196
262
  try {
197
263
  const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
198
264
  if (generation !== this.generation) return;
@@ -207,10 +273,26 @@ var VoiceClient = class {
207
273
  url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
208
274
  url.search = "";
209
275
  url.searchParams.set("voiceProtocol", "runtype-browser-v1");
276
+ const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
277
+ url.searchParams.set("clientCapabilities", capabilities);
278
+ const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
279
+ if (sessionId) url.searchParams.set("sessionId", sessionId);
280
+ const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
281
+ if (generation !== this.generation) return;
282
+ if (this.options.visitorToken !== void 0 && !visitorToken) {
283
+ throw new Error(
284
+ "Voice visitor credential unavailable. Initialize the client session first."
285
+ );
286
+ }
287
+ if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
288
+ throw new Error("Invalid voice visitor credential.");
289
+ }
290
+ if (this.options.fullDuplex !== false)
291
+ url.searchParams.set("voiceCapabilities", "full-duplex-v1");
210
292
  url.hash = "";
211
- const stream = await navigator.mediaDevices.getUserMedia({
293
+ const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
212
294
  audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
213
- });
295
+ }));
214
296
  if (generation !== this.generation) {
215
297
  stream.getTracks().forEach((track) => track.stop());
216
298
  return;
@@ -230,7 +312,11 @@ var VoiceClient = class {
230
312
  return;
231
313
  }
232
314
  this.player = player;
233
- const socket = new WebSocket(url.toString(), ["runtype.bearer", token]);
315
+ const socket = new WebSocket(url.toString(), [
316
+ "runtype.bearer",
317
+ token,
318
+ ...visitorToken ? [visitorToken] : []
319
+ ]);
234
320
  socket.binaryType = "arraybuffer";
235
321
  this.socket = socket;
236
322
  socket.onopen = () => {
@@ -261,6 +347,11 @@ var VoiceClient = class {
261
347
  this.cleanup();
262
348
  this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
263
349
  };
350
+ /** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
351
+ resetSession = () => {
352
+ if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
353
+ this.update({ session: null });
354
+ };
264
355
  toggleMute = () => {
265
356
  this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
266
357
  };
@@ -275,9 +366,11 @@ var VoiceClient = class {
275
366
  this.clearPlayback();
276
367
  if (this.snapshot.interruptionMode === "none") {
277
368
  this.stoppedResponse = !this.responseEnded;
369
+ this.commitClauseCaptions();
278
370
  this.setStatus("listening");
279
371
  return;
280
372
  }
373
+ this.suppressedArtifactTurnId = this.activeTurnId;
281
374
  this.awaitingClear = true;
282
375
  this.setStatus("listening");
283
376
  this.socket.send(JSON.stringify({ type: "cancel" }));
@@ -294,6 +387,7 @@ var VoiceClient = class {
294
387
  });
295
388
  }
296
389
  cleanup() {
390
+ this.fullDuplex = false;
297
391
  this.generation += 1;
298
392
  if (this.processor) this.processor.onaudioprocess = null;
299
393
  this.processor?.disconnect();
@@ -319,6 +413,82 @@ var VoiceClient = class {
319
413
  this.awaitingClear = false;
320
414
  this.stoppedResponse = false;
321
415
  this.responseEnded = true;
416
+ this.commitClauseCaptions();
417
+ this.committedAssistantTurnId = null;
418
+ this.activeTurnId = null;
419
+ this.suppressedArtifactTurnId = null;
420
+ const settledArtifacts = this.snapshot.artifacts.filter(
421
+ (artifact) => artifact.status === "complete"
422
+ );
423
+ if (settledArtifacts.length !== this.snapshot.artifacts.length)
424
+ this.update({ artifacts: settledArtifacts });
425
+ }
426
+ /**
427
+ * Folds an `artifact` message into the snapshot. Frames of a turn the caller
428
+ * cancelled are dropped; a playback-only stop leaves the turn running, so its
429
+ * artifacts keep arriving and are kept.
430
+ */
431
+ applyArtifactMessage(msg) {
432
+ if (this.awaitingClear) return;
433
+ const frame = msg.event;
434
+ if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
435
+ const event = frame;
436
+ const id = readString(event.id);
437
+ if (!id) return;
438
+ const turnId = readTurnId(msg);
439
+ if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
440
+ if (turnId !== null) this.activeTurnId = turnId;
441
+ const artifacts = this.snapshot.artifacts;
442
+ const index = artifacts.findIndex((artifact) => artifact.id === id);
443
+ const executionId = readString(msg.executionId);
444
+ const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
445
+ id,
446
+ turnId,
447
+ ...executionId ? { executionId } : {}
448
+ });
449
+ if (!next) return;
450
+ this.update({
451
+ artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
452
+ });
453
+ }
454
+ /** Settles the spoken clauses in place and ends the turn's claim on them. */
455
+ commitClauseCaptions() {
456
+ this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
457
+ const settled = this.pendingAssistant !== null;
458
+ this.pendingAssistant = null;
459
+ if (!settled) return;
460
+ const transcript = this.snapshot.transcript;
461
+ const last = transcript[transcript.length - 1];
462
+ if (last?.role !== "assistant" || last.partial !== true) return;
463
+ const { partial: _partial, ...committed } = last;
464
+ this.update({ transcript: [...transcript.slice(0, -1), committed] });
465
+ }
466
+ /** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
467
+ appendClauseCaption(text, turnId) {
468
+ const transcript = this.snapshot.transcript;
469
+ const last = transcript[transcript.length - 1];
470
+ if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
471
+ return {
472
+ interimTranscript: null,
473
+ transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
474
+ status: "speaking"
475
+ };
476
+ }
477
+ this.pendingAssistant = { turnId };
478
+ return {
479
+ interimTranscript: null,
480
+ transcript: [
481
+ ...transcript,
482
+ {
483
+ role: "assistant",
484
+ content: text,
485
+ timestamp: Date.now(),
486
+ partial: true,
487
+ ...turnId ? { turnId } : {}
488
+ }
489
+ ],
490
+ status: "speaking"
491
+ };
322
492
  }
323
493
  clearPlayback() {
324
494
  this.playbackRevision += 1;
@@ -353,32 +523,89 @@ var VoiceClient = class {
353
523
  return;
354
524
  }
355
525
  switch (msg.type) {
356
- case "session_config":
526
+ case "session_config": {
527
+ this.fullDuplex = msg.speechMode === "speech_to_speech";
528
+ if (this.fullDuplex) player.setContinuousMode?.(true);
357
529
  if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
358
530
  this.update({ interruptionMode: msg.interruptionMode });
359
531
  }
532
+ const sessionId = readString(msg.sessionId);
533
+ const conversationId = readString(msg.conversationId);
534
+ if (sessionId && conversationId) {
535
+ const session = { sessionId, conversationId };
536
+ this.update({ session });
537
+ this.options.onSession?.(session);
538
+ }
539
+ break;
540
+ }
541
+ case "artifact":
542
+ this.applyArtifactMessage(msg);
543
+ break;
544
+ case "transcript_update": {
545
+ if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
546
+ break;
547
+ if (msg.role === "assistant" && this.awaitingClear) break;
548
+ const entries = [...this.snapshot.transcript];
549
+ const index = entries.findIndex(
550
+ (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
551
+ );
552
+ const entry = {
553
+ role: msg.role,
554
+ content: msg.text,
555
+ turnId: msg.turnId,
556
+ isFinal: msg.final === true,
557
+ timestamp: index >= 0 ? entries[index].timestamp : Date.now()
558
+ };
559
+ if (index >= 0) entries[index] = entry;
560
+ else entries.push(entry);
561
+ this.update({ transcript: entries, interimTranscript: null });
562
+ break;
563
+ }
564
+ case "delegation_started":
565
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
566
+ break;
567
+ case "delegation_completed":
568
+ if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
360
569
  break;
361
570
  case "transcript_interim":
362
571
  this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
363
572
  break;
364
- case "transcript_final":
573
+ case "transcript_partial": {
574
+ if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
575
+ this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
576
+ if (this.awaitingClear || this.stoppedResponse) break;
577
+ this.responseEnded = false;
578
+ this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
579
+ break;
580
+ }
581
+ case "transcript_final": {
365
582
  if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
366
- if (msg.role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
583
+ const role = msg.role;
584
+ const text = msg.text;
585
+ if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
586
+ const turnId = readTurnId(msg);
587
+ if (turnId !== null) this.activeTurnId = turnId;
588
+ if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
589
+ break;
367
590
  this.responseEnded = false;
591
+ const transcript = this.snapshot.transcript;
592
+ const last = transcript[transcript.length - 1];
593
+ const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
594
+ this.pendingAssistant = null;
595
+ const entry = {
596
+ role,
597
+ content: text,
598
+ // INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
599
+ timestamp: supersedes && last ? last.timestamp : Date.now(),
600
+ ...turnId ? { turnId } : {}
601
+ };
368
602
  this.update({
369
603
  interimTranscript: null,
370
- transcript: [
371
- ...this.snapshot.transcript,
372
- {
373
- role: msg.role,
374
- content: msg.text,
375
- timestamp: Date.now(),
376
- ...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
377
- }
378
- ],
379
- status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
604
+ transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
605
+ status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
380
606
  });
381
607
  break;
608
+ }
382
609
  case "audio_end": {
383
610
  this.responseEnded = true;
384
611
  if (this.stoppedResponse) {
@@ -395,10 +622,12 @@ var VoiceClient = class {
395
622
  break;
396
623
  }
397
624
  case "audio_clear":
625
+ this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
398
626
  this.clearPlayback();
399
627
  this.awaitingClear = false;
400
628
  this.stoppedResponse = false;
401
629
  this.responseEnded = true;
630
+ this.commitClauseCaptions();
402
631
  this.setStatus("listening");
403
632
  break;
404
633
  case "metrics": {
@@ -411,8 +640,11 @@ var VoiceClient = class {
411
640
  // @snake-case-ok: Existing voice wire contract.
412
641
  firstAudioMs: number(msg.first_audio_ms),
413
642
  // @snake-case-ok: Existing voice wire contract.
414
- totalMs: number(msg.total_ms)
643
+ totalMs: number(msg.total_ms),
644
+ // @snake-case-ok: Existing voice wire contract.
645
+ firstSynthesisMs: number(msg.first_synthesis_ms),
415
646
  // @snake-case-ok: Existing voice wire contract.
647
+ incremental: msg.incremental === true
416
648
  }
417
649
  });
418
650
  break;
@@ -444,14 +676,14 @@ var VoiceClient = class {
444
676
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
445
677
  this.processor = processor;
446
678
  processor.onaudioprocess = (event) => {
447
- if (generation !== this.generation || this.snapshot.isMuted) return;
679
+ if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
448
680
  const input = event.inputBuffer.getChannelData(0);
449
681
  let sum = 0;
450
682
  for (const sample of input) sum += sample * sample;
451
- this.update({ audioLevel: Math.sqrt(sum / input.length) });
683
+ this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
452
684
  if (socket.readyState !== WebSocket.OPEN) return;
453
685
  const pcm = new Int16Array(input.length);
454
- if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
686
+ if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
455
687
  socket.send(pcm.buffer);
456
688
  return;
457
689
  }
@@ -467,25 +699,81 @@ var VoiceClient = class {
467
699
  };
468
700
 
469
701
  // src/react.ts
470
- function useVoiceClient({ agentId, apiUrl, clientToken }) {
702
+ function useVoiceClient({
703
+ agentId,
704
+ apiUrl,
705
+ clientToken,
706
+ artifacts,
707
+ sessionId,
708
+ visitorToken,
709
+ onSession,
710
+ fullDuplex,
711
+ audioSource
712
+ }) {
713
+ const sessionRef = useRef(sessionId);
714
+ useEffect(() => {
715
+ sessionRef.current = sessionId;
716
+ }, [sessionId]);
471
717
  const tokenRef = useRef(clientToken);
718
+ const visitorRef = useRef(visitorToken);
719
+ const sessionCallbackRef = useRef(onSession);
720
+ const audioSourceRef = useRef(audioSource);
721
+ const hasAudioSource = audioSource !== void 0;
722
+ useEffect(() => {
723
+ visitorRef.current = visitorToken;
724
+ sessionCallbackRef.current = onSession;
725
+ }, [visitorToken, onSession]);
726
+ useEffect(() => {
727
+ audioSourceRef.current = audioSource;
728
+ }, [audioSource]);
472
729
  useEffect(() => {
473
730
  tokenRef.current = clientToken;
474
731
  }, [clientToken]);
475
- const client = useMemo(
476
- () => new VoiceClient({
732
+ const createClient = () => new VoiceClient({
733
+ agentId,
734
+ apiUrl,
735
+ artifacts,
736
+ fullDuplex,
737
+ ...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
738
+ get sessionId() {
739
+ return sessionRef.current;
740
+ },
741
+ get visitorToken() {
742
+ return visitorRef.current;
743
+ },
744
+ onSession: (session) => sessionCallbackRef.current?.(session),
745
+ clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
746
+ });
747
+ const [binding, setBinding] = useState(() => ({
748
+ agentId,
749
+ apiUrl,
750
+ artifacts,
751
+ fullDuplex,
752
+ hasAudioSource,
753
+ sessionId,
754
+ client: createClient()
755
+ }));
756
+ const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
757
+ if (configurationChanged || binding.sessionId !== sessionId) {
758
+ const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
759
+ setBinding({
477
760
  agentId,
478
761
  apiUrl,
479
- clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
480
- }),
481
- [agentId, apiUrl]
482
- );
762
+ artifacts,
763
+ fullDuplex,
764
+ hasAudioSource,
765
+ sessionId,
766
+ client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
767
+ });
768
+ }
769
+ const client = binding.client;
483
770
  const snapshot = useSyncExternalStore(client.subscribe, client.getSnapshot, client.getSnapshot);
484
771
  useEffect(() => () => client.endCall(), [client]);
485
772
  return {
486
773
  ...snapshot,
487
774
  startCall: client.startCall,
488
775
  endCall: client.endCall,
776
+ resetSession: client.resetSession,
489
777
  toggleMute: client.toggleMute,
490
778
  cancelResponse: client.cancelResponse
491
779
  };
@@ -0,0 +1,100 @@
1
+ type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
2
+ type InterruptionMode = 'none' | 'cancel' | 'barge-in';
3
+ interface TranscriptEntry {
4
+ readonly role: 'user' | 'assistant';
5
+ readonly content: string;
6
+ readonly timestamp: number;
7
+ readonly isFinal?: boolean;
8
+ readonly turnId?: string;
9
+ /**
10
+ * The reply is still being spoken, so `content` holds the clauses committed so
11
+ * far. A later snapshot replaces this entry with the authoritative transcript,
12
+ * or clears the flag in place when the caller stops the reply.
13
+ */
14
+ readonly partial?: boolean;
15
+ }
16
+ interface VoiceMetrics {
17
+ readonly llmMs?: number;
18
+ readonly ttsMs?: number;
19
+ readonly firstAudioMs?: number;
20
+ readonly totalMs?: number;
21
+ /**
22
+ * Turn start to the first synthesis request. Below `llmMs` when the engine spoke
23
+ * clauses while the reply was still generating, and equal to it otherwise.
24
+ */
25
+ readonly firstSynthesisMs?: number;
26
+ /** True when the reply was synthesized clause by clause, so `ttsMs` covers overlapped work. */
27
+ readonly incremental?: boolean;
28
+ }
29
+ /** File metadata for a previewable artifact, mirroring the unified `artifact_start.file` field. */
30
+ interface VoiceArtifactFile {
31
+ readonly path: string;
32
+ readonly mimeType: string;
33
+ readonly language?: string;
34
+ }
35
+ /**
36
+ * An artifact the agent produced during a call, assembled from the additive
37
+ * `artifact` wire message. Artifacts arrive beside speech and are never spoken.
38
+ */
39
+ interface VoiceArtifact {
40
+ readonly id: string;
41
+ readonly turnId: string | null;
42
+ readonly executionId?: string;
43
+ readonly artifactType: 'markdown' | 'component';
44
+ readonly title?: string;
45
+ readonly file?: VoiceArtifactFile;
46
+ readonly content: string;
47
+ readonly component?: string;
48
+ readonly props?: Record<string, unknown>;
49
+ readonly status: 'streaming' | 'complete';
50
+ }
51
+ /** The conversation this call is attached to, reported by the server once the call opens. */
52
+ interface VoiceSession {
53
+ readonly sessionId: string;
54
+ readonly conversationId: string;
55
+ }
56
+ interface VoiceSnapshot {
57
+ readonly status: VoiceStatus;
58
+ readonly transcript: readonly TranscriptEntry[];
59
+ readonly interimTranscript: string | null;
60
+ readonly metrics: VoiceMetrics | null;
61
+ readonly audioLevel: number;
62
+ readonly isMuted: boolean;
63
+ readonly error: string | null;
64
+ readonly errorDetails?: {
65
+ code: string;
66
+ serverId?: string;
67
+ serverName: string;
68
+ diagnosticId: string;
69
+ };
70
+ readonly interruptionMode: InterruptionMode;
71
+ readonly canCancel: boolean;
72
+ readonly artifacts: readonly VoiceArtifact[];
73
+ /** Survives `endCall` so a host can reopen the same conversation; null until the server reports it. */
74
+ readonly session: VoiceSession | null;
75
+ }
76
+ interface VoiceClientOptions {
77
+ agentId: string;
78
+ /** Disable when the transcript consumer only supports sequential pipeline turns. */
79
+ fullDuplex?: boolean;
80
+ /** Optional custom audio source; the client owns and stops the returned tracks. */
81
+ audioSource?: () => Promise<MediaStream>;
82
+ /** Browser client token, or a getter called once per call to obtain a fresh token. */
83
+ clientToken: string | (() => string | Promise<string>);
84
+ /** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
85
+ apiUrl?: string;
86
+ /**
87
+ * Declares the `artifacts` capability, so the server sends `artifact` messages
88
+ * and the client assembles `snapshot.artifacts`. Off by default, which keeps an
89
+ * existing embed's wire byte-identical.
90
+ */
91
+ artifacts?: boolean;
92
+ /** Reuses an existing client session's conversation. Overrides the id remembered from a previous call. */
93
+ sessionId?: string;
94
+ /** Proof minted by client/init; resolve from the host's token-scoped visitor store on every call. */
95
+ visitorToken?: string | (() => string | undefined | Promise<string | undefined>);
96
+ /** Observe the server's conversation so a host can reuse it for text. Never contains visitor credentials. */
97
+ onSession?: (session: VoiceSession) => void;
98
+ }
99
+
100
+ export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceArtifact as b, VoiceArtifactFile as c, VoiceMetrics as d, VoiceSession as e, VoiceStatus as f };