@runtypelabs/voice 0.2.4 → 0.4.0

package/dist/react.cjs

@@ -37,13 +37,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
     this.readOffset = 0
     this.buffered = 0
     this.waiting = true
-    // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
+    // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
     this.eosSeen = false
+    this.continuous = false
     this.revision = 0
     this.port.onmessage = (e) => {
       const msg = e.data
       this.revision = msg.revision
-      if (msg.type === 'push') {
+      if (msg.type === 'continuous') {
+        this.continuous = msg.enabled
+      } else if (msg.type === 'push') {
         this.eosSeen = false
         this.chunks.push(msg.samples)
         this.buffered += msg.samples.length
@@ -83,7 +86,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
     }
     if (this.buffered === 0) {
       this.waiting = true // mid-reply underrun: re-buffer silently
-      if (this.eosSeen) {
+      if (this.eosSeen || this.continuous) {
         this.eosSeen = false
         this.port.postMessage({ type: 'drained', revision: this.revision })
       }
@@ -127,6 +130,9 @@ async function createPcmPlayer(onDrained) {
       if (samples.length === 0) return;
       node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
     },
+    setContinuousMode(enabled) {
+      node.port.postMessage({ type: "continuous", enabled, revision });
+    },
     endOfStream() {
       node.port.postMessage({ type: "eos", revision });
     },
@@ -171,9 +177,58 @@ function initialSnapshot() {
     error: null,
     errorDetails: void 0,
     interruptionMode: "none",
-    canCancel: false
+    canCancel: false,
+    artifacts: [],
+    session: null
   };
 }
+function readString(value) {
+  return typeof value === "string" && value ? value : void 0;
+}
+function readArtifactFile(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
+  const file = value;
+  const path = readString(file.path);
+  const mimeType = readString(file.mimeType);
+  if (!path || !mimeType) return void 0;
+  const language = readString(file.language);
+  return { path, mimeType, ...language ? { language } : {} };
+}
+function applyArtifactFrame(existing, frame, identity) {
+  if (frame.type === "artifact_start") {
+    const title = readString(frame.title);
+    const component = readString(frame.component);
+    const file = readArtifactFile(frame.file);
+    return {
+      ...identity,
+      artifactType: frame.artifactType === "component" ? "component" : "markdown",
+      ...title ? { title } : {},
+      ...file ? { file } : {},
+      ...component ? { component } : {},
+      content: "",
+      status: "streaming"
+    };
+  }
+  if (!existing) return void 0;
+  if (frame.type === "artifact_delta") {
+    const delta = typeof frame.delta === "string" ? frame.delta : "";
+    return delta ? { ...existing, content: existing.content + delta } : void 0;
+  }
+  if (frame.type === "artifact_update") {
+    const component = readString(frame.component) ?? existing.component;
+    const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
+    return {
+      ...existing,
+      ...component ? { component } : {},
+      ...props ? { props } : {}
+    };
+  }
+  if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
+  return void 0;
+}
+function readTurnId(msg) {
+  return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
+}
 var VoiceClient = class {
   constructor(options) {
     this.options = options;
@@ -194,6 +249,17 @@ var VoiceClient = class {
   stoppedResponse = false;
   responseEnded = true;
   hasPendingAudio = false;
+  /**
+   * The assistant entry built from clause captions for the turn being spoken. The
+   * authoritative transcript replaces it; a stop commits it, because it captions
+   * audio the caller already heard.
+   */
+  pendingAssistant = null;
+  /** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
+  committedAssistantTurnId = null;
+  activeTurnId = null;
+  suppressedArtifactTurnId = null;
+  fullDuplex = false;
   getSnapshot = () => this.snapshot;
   subscribe = (listener) => {
     this.listeners.add(listener);
@@ -215,7 +281,7 @@ var VoiceClient = class {
     if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
     this.cleanup();
     const generation = this.generation;
-    this.update({ ...initialSnapshot(), status: "connecting" });
+    this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
     try {
       const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
       if (generation !== this.generation) return;
@@ -230,10 +296,26 @@ var VoiceClient = class {
       url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
       url.search = "";
       url.searchParams.set("voiceProtocol", "runtype-browser-v1");
+      const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
+      url.searchParams.set("clientCapabilities", capabilities);
+      const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
+      if (sessionId) url.searchParams.set("sessionId", sessionId);
+      const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
+      if (generation !== this.generation) return;
+      if (this.options.visitorToken !== void 0 && !visitorToken) {
+        throw new Error(
+          "Voice visitor credential unavailable. Initialize the client session first."
+        );
+      }
+      if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
+        throw new Error("Invalid voice visitor credential.");
+      }
+      if (this.options.fullDuplex !== false)
+        url.searchParams.set("voiceCapabilities", "full-duplex-v1");
       url.hash = "";
-      const stream = await navigator.mediaDevices.getUserMedia({
+      const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
         audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
-      });
+      }));
       if (generation !== this.generation) {
         stream.getTracks().forEach((track) => track.stop());
         return;
@@ -253,7 +335,11 @@ var VoiceClient = class {
         return;
       }
       this.player = player;
-      const socket = new WebSocket(url.toString(), ["runtype.bearer", token]);
+      const socket = new WebSocket(url.toString(), [
+        "runtype.bearer",
+        token,
+        ...visitorToken ? [visitorToken] : []
+      ]);
       socket.binaryType = "arraybuffer";
       this.socket = socket;
       socket.onopen = () => {
@@ -284,6 +370,11 @@ var VoiceClient = class {
     this.cleanup();
     this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
   };
+  /** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
+  resetSession = () => {
+    if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
+    this.update({ session: null });
+  };
   toggleMute = () => {
     this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
   };
@@ -298,9 +389,11 @@ var VoiceClient = class {
     this.clearPlayback();
     if (this.snapshot.interruptionMode === "none") {
       this.stoppedResponse = !this.responseEnded;
+      this.commitClauseCaptions();
       this.setStatus("listening");
       return;
     }
+    this.suppressedArtifactTurnId = this.activeTurnId;
     this.awaitingClear = true;
     this.setStatus("listening");
     this.socket.send(JSON.stringify({ type: "cancel" }));
@@ -317,6 +410,7 @@ var VoiceClient = class {
     });
   }
   cleanup() {
+    this.fullDuplex = false;
     this.generation += 1;
     if (this.processor) this.processor.onaudioprocess = null;
     this.processor?.disconnect();
@@ -342,6 +436,82 @@ var VoiceClient = class {
     this.awaitingClear = false;
     this.stoppedResponse = false;
     this.responseEnded = true;
+    this.commitClauseCaptions();
+    this.committedAssistantTurnId = null;
+    this.activeTurnId = null;
+    this.suppressedArtifactTurnId = null;
+    const settledArtifacts = this.snapshot.artifacts.filter(
+      (artifact) => artifact.status === "complete"
+    );
+    if (settledArtifacts.length !== this.snapshot.artifacts.length)
+      this.update({ artifacts: settledArtifacts });
+  }
+  /**
+   * Folds an `artifact` message into the snapshot. Frames of a turn the caller
+   * cancelled are dropped; a playback-only stop leaves the turn running, so its
+   * artifacts keep arriving and are kept.
+   */
+  applyArtifactMessage(msg) {
+    if (this.awaitingClear) return;
+    const frame = msg.event;
+    if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
+    const event = frame;
+    const id = readString(event.id);
+    if (!id) return;
+    const turnId = readTurnId(msg);
+    if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
+    if (turnId !== null) this.activeTurnId = turnId;
+    const artifacts = this.snapshot.artifacts;
+    const index = artifacts.findIndex((artifact) => artifact.id === id);
+    const executionId = readString(msg.executionId);
+    const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
+      id,
+      turnId,
+      ...executionId ? { executionId } : {}
+    });
+    if (!next) return;
+    this.update({
+      artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
+    });
+  }
+  /** Settles the spoken clauses in place and ends the turn's claim on them. */
+  commitClauseCaptions() {
+    this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
+    const settled = this.pendingAssistant !== null;
+    this.pendingAssistant = null;
+    if (!settled) return;
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (last?.role !== "assistant" || last.partial !== true) return;
+    const { partial: _partial, ...committed } = last;
+    this.update({ transcript: [...transcript.slice(0, -1), committed] });
+  }
+  /** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
+  appendClauseCaption(text, turnId) {
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
+      return {
+        interimTranscript: null,
+        transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
+        status: "speaking"
+      };
+    }
+    this.pendingAssistant = { turnId };
+    return {
+      interimTranscript: null,
+      transcript: [
+        ...transcript,
+        {
+          role: "assistant",
+          content: text,
+          timestamp: Date.now(),
+          partial: true,
+          ...turnId ? { turnId } : {}
+        }
+      ],
+      status: "speaking"
+    };
   }
   clearPlayback() {
     this.playbackRevision += 1;
@@ -376,32 +546,89 @@ var VoiceClient = class {
       return;
     }
     switch (msg.type) {
-      case "session_config":
+      case "session_config": {
+        this.fullDuplex = msg.speechMode === "speech_to_speech";
+        if (this.fullDuplex) player.setContinuousMode?.(true);
         if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
           this.update({ interruptionMode: msg.interruptionMode });
         }
+        const sessionId = readString(msg.sessionId);
+        const conversationId = readString(msg.conversationId);
+        if (sessionId && conversationId) {
+          const session = { sessionId, conversationId };
+          this.update({ session });
+          this.options.onSession?.(session);
+        }
+        break;
+      }
+      case "artifact":
+        this.applyArtifactMessage(msg);
+        break;
+      case "transcript_update": {
+        if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
+          break;
+        if (msg.role === "assistant" && this.awaitingClear) break;
+        const entries = [...this.snapshot.transcript];
+        const index = entries.findIndex(
+          (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
+        );
+        const entry = {
+          role: msg.role,
+          content: msg.text,
+          turnId: msg.turnId,
+          isFinal: msg.final === true,
+          timestamp: index >= 0 ? entries[index].timestamp : Date.now()
+        };
+        if (index >= 0) entries[index] = entry;
+        else entries.push(entry);
+        this.update({ transcript: entries, interimTranscript: null });
+        break;
+      }
+      case "delegation_started":
+        if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
+        break;
+      case "delegation_completed":
+        if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
         break;
       case "transcript_interim":
         this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
         break;
-      case "transcript_final":
+      case "transcript_partial": {
+        if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
+        this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
+        if (this.awaitingClear || this.stoppedResponse) break;
+        this.responseEnded = false;
+        this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
+        break;
+      }
+      case "transcript_final": {
         if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
-        if (msg.role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const role = msg.role;
+        const text = msg.text;
+        if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const turnId = readTurnId(msg);
+        if (turnId !== null) this.activeTurnId = turnId;
+        if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
+          break;
         this.responseEnded = false;
+        const transcript = this.snapshot.transcript;
+        const last = transcript[transcript.length - 1];
+        const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
+        this.pendingAssistant = null;
+        const entry = {
+          role,
+          content: text,
+          // INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
+          timestamp: supersedes && last ? last.timestamp : Date.now(),
+          ...turnId ? { turnId } : {}
+        };
         this.update({
           interimTranscript: null,
-          transcript: [
-            ...this.snapshot.transcript,
-            {
-              role: msg.role,
-              content: msg.text,
-              timestamp: Date.now(),
-              ...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
-            }
-          ],
-          status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
+          transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
+          status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
         });
         break;
+      }
       case "audio_end": {
         this.responseEnded = true;
         if (this.stoppedResponse) {
@@ -418,10 +645,12 @@ var VoiceClient = class {
         break;
       }
       case "audio_clear":
+        this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
         this.clearPlayback();
         this.awaitingClear = false;
         this.stoppedResponse = false;
         this.responseEnded = true;
+        this.commitClauseCaptions();
         this.setStatus("listening");
         break;
       case "metrics": {
@@ -434,8 +663,11 @@ var VoiceClient = class {
             // @snake-case-ok: Existing voice wire contract.
             firstAudioMs: number(msg.first_audio_ms),
             // @snake-case-ok: Existing voice wire contract.
-            totalMs: number(msg.total_ms)
+            totalMs: number(msg.total_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            firstSynthesisMs: number(msg.first_synthesis_ms),
             // @snake-case-ok: Existing voice wire contract.
+            incremental: msg.incremental === true
           }
         });
         break;
@@ -467,14 +699,14 @@ var VoiceClient = class {
     const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
     this.processor = processor;
     processor.onaudioprocess = (event) => {
-      if (generation !== this.generation || this.snapshot.isMuted) return;
+      if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
       const input = event.inputBuffer.getChannelData(0);
       let sum = 0;
       for (const sample of input) sum += sample * sample;
-      this.update({ audioLevel: Math.sqrt(sum / input.length) });
+      this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
       if (socket.readyState !== WebSocket.OPEN) return;
       const pcm = new Int16Array(input.length);
-      if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
+      if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
         socket.send(pcm.buffer);
         return;
       }
@@ -490,25 +722,81 @@ var VoiceClient = class {
 };
 
 // src/react.ts
-function useVoiceClient({ agentId, apiUrl, clientToken }) {
+function useVoiceClient({
+  agentId,
+  apiUrl,
+  clientToken,
+  artifacts,
+  sessionId,
+  visitorToken,
+  onSession,
+  fullDuplex,
+  audioSource
+}) {
+  const sessionRef = (0, import_react.useRef)(sessionId);
+  (0, import_react.useEffect)(() => {
+    sessionRef.current = sessionId;
+  }, [sessionId]);
   const tokenRef = (0, import_react.useRef)(clientToken);
+  const visitorRef = (0, import_react.useRef)(visitorToken);
+  const sessionCallbackRef = (0, import_react.useRef)(onSession);
+  const audioSourceRef = (0, import_react.useRef)(audioSource);
+  const hasAudioSource = audioSource !== void 0;
+  (0, import_react.useEffect)(() => {
+    visitorRef.current = visitorToken;
+    sessionCallbackRef.current = onSession;
+  }, [visitorToken, onSession]);
+  (0, import_react.useEffect)(() => {
+    audioSourceRef.current = audioSource;
+  }, [audioSource]);
   (0, import_react.useEffect)(() => {
     tokenRef.current = clientToken;
   }, [clientToken]);
-  const client = (0, import_react.useMemo)(
-    () => new VoiceClient({
+  const createClient = () => new VoiceClient({
+    agentId,
+    apiUrl,
+    artifacts,
+    fullDuplex,
+    ...hasAudioSource ? { audioSource: () => audioSourceRef.current() } : {},
+    get sessionId() {
+      return sessionRef.current;
+    },
+    get visitorToken() {
+      return visitorRef.current;
+    },
+    onSession: (session) => sessionCallbackRef.current?.(session),
+    clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
+  });
+  const [binding, setBinding] = (0, import_react.useState)(() => ({
+    agentId,
+    apiUrl,
+    artifacts,
+    fullDuplex,
+    hasAudioSource,
+    sessionId,
+    client: createClient()
+  }));
+  const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts || binding.fullDuplex !== fullDuplex || binding.hasAudioSource !== hasAudioSource;
+  if (configurationChanged || binding.sessionId !== sessionId) {
+    const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
+    setBinding({
       agentId,
       apiUrl,
-      clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
-    }),
-    [agentId, apiUrl]
-  );
+      artifacts,
+      fullDuplex,
+      hasAudioSource,
+      sessionId,
+      client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
+    });
+  }
+  const client = binding.client;
   const snapshot = (0, import_react.useSyncExternalStore)(client.subscribe, client.getSnapshot, client.getSnapshot);
   (0, import_react.useEffect)(() => () => client.endCall(), [client]);
   return {
     ...snapshot,
     startCall: client.startCall,
     endCall: client.endCall,
+    resetSession: client.resetSession,
     toggleMute: client.toggleMute,
     cancelResponse: client.cancelResponse
   };

package/dist/react.d.cts

@@ -1,10 +1,11 @@
-import { V as VoiceClientOptions, c as VoiceStatus, T as TranscriptEntry, b as VoiceMetrics, I as InterruptionMode } from './types-9SfA7oHc.cjs';
-export { a as VoiceSnapshot } from './types-9SfA7oHc.cjs';
+import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-Dkh4LxEA.cjs';
+export { a as VoiceSnapshot } from './types-Dkh4LxEA.cjs';
 
 /** Subscribe to a voice client and end its call when the component unmounts or changes agent. */
-declare function useVoiceClient({ agentId, apiUrl, clientToken }: VoiceClientOptions): {
+declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, fullDuplex, audioSource, }: VoiceClientOptions): {
     startCall: (tokenOverride?: string) => Promise<void>;
     endCall: () => void;
+    resetSession: () => void;
     toggleMute: () => void;
     cancelResponse: () => void;
     status: VoiceStatus;
@@ -22,6 +23,8 @@ declare function useVoiceClient({ agentId, apiUrl, clientToken }: VoiceClientOpt
     };
     interruptionMode: InterruptionMode;
     canCancel: boolean;
+    artifacts: readonly VoiceArtifact[];
+    session: VoiceSession | null;
 };
 
-export { VoiceClientOptions, VoiceStatus, useVoiceClient };
+export { VoiceArtifact, VoiceClientOptions, VoiceSession, VoiceStatus, useVoiceClient };

package/dist/react.d.ts

@@ -1,10 +1,11 @@
-import { V as VoiceClientOptions, c as VoiceStatus, T as TranscriptEntry, b as VoiceMetrics, I as InterruptionMode } from './types-9SfA7oHc.js';
-export { a as VoiceSnapshot } from './types-9SfA7oHc.js';
+import { V as VoiceClientOptions, f as VoiceStatus, T as TranscriptEntry, d as VoiceMetrics, I as InterruptionMode, b as VoiceArtifact, e as VoiceSession } from './types-Dkh4LxEA.js';
+export { a as VoiceSnapshot } from './types-Dkh4LxEA.js';
 
 /** Subscribe to a voice client and end its call when the component unmounts or changes agent. */
-declare function useVoiceClient({ agentId, apiUrl, clientToken }: VoiceClientOptions): {
+declare function useVoiceClient({ agentId, apiUrl, clientToken, artifacts, sessionId, visitorToken, onSession, fullDuplex, audioSource, }: VoiceClientOptions): {
     startCall: (tokenOverride?: string) => Promise<void>;
     endCall: () => void;
+    resetSession: () => void;
     toggleMute: () => void;
     cancelResponse: () => void;
     status: VoiceStatus;
@@ -22,6 +23,8 @@ declare function useVoiceClient({ agentId, apiUrl, clientToken }: VoiceClientOpt
     };
     interruptionMode: InterruptionMode;
     canCancel: boolean;
+    artifacts: readonly VoiceArtifact[];
+    session: VoiceSession | null;
 };
 
-export { VoiceClientOptions, VoiceStatus, useVoiceClient };
+export { VoiceArtifact, VoiceClientOptions, VoiceSession, VoiceStatus, useVoiceClient };