@runtypelabs/voice 0.2.4 → 0.4.0

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-9SfA7oHc.js';
-export { I as InterruptionMode, T as TranscriptEntry, b as VoiceMetrics, c as VoiceStatus } from './types-9SfA7oHc.js';
+import { V as VoiceClientOptions, a as VoiceSnapshot } from './types-Dkh4LxEA.js';
+export { I as InterruptionMode, T as TranscriptEntry, b as VoiceArtifact, c as VoiceArtifactFile, d as VoiceMetrics, e as VoiceSession, f as VoiceStatus } from './types-Dkh4LxEA.js';
 
 /** Browser microphone and playback lifecycle for Runtype's voice WebSocket endpoint. */
 declare class VoiceClient {
@@ -19,6 +19,17 @@ declare class VoiceClient {
     private stoppedResponse;
     private responseEnded;
     private hasPendingAudio;
+    /**
+     * The assistant entry built from clause captions for the turn being spoken. The
+     * authoritative transcript replaces it; a stop commits it, because it captions
+     * audio the caller already heard.
+     */
+    private pendingAssistant;
+    /** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
+    private committedAssistantTurnId;
+    private activeTurnId;
+    private suppressedArtifactTurnId;
+    private fullDuplex;
     constructor(options: VoiceClientOptions);
     getSnapshot: () => VoiceSnapshot;
     subscribe: (listener: () => void) => (() => void);
@@ -27,12 +38,24 @@ declare class VoiceClient {
     /** Acquire microphone access and open a call. Invoke from a user gesture. Failures populate snapshot.error. */
     startCall: (tokenOverride?: string) => Promise<void>;
     endCall: () => void;
+    /** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
+    resetSession: () => void;
     toggleMute: () => void;
     cancelResponse: () => void;
     /** Stop the current reply explicitly, including local playback when interruptions are disabled. */
     stopPlayback: () => void;
     private fail;
     private cleanup;
+    /**
+     * Folds an `artifact` message into the snapshot. Frames of a turn the caller
+     * cancelled are dropped; a playback-only stop leaves the turn running, so its
+     * artifacts keep arriving and are kept.
+     */
+    private applyArtifactMessage;
+    /** Settles the spoken clauses in place and ends the turn's claim on them. */
+    private commitClauseCaptions;
+    /** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
+    private appendClauseCaption;
     private clearPlayback;
     private handleMessage;
     private startCapture;

package/dist/index.js

@@ -9,13 +9,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
     this.readOffset = 0
     this.buffered = 0
     this.waiting = true
-    // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
+    // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
     this.eosSeen = false
+    this.continuous = false
     this.revision = 0
     this.port.onmessage = (e) => {
       const msg = e.data
       this.revision = msg.revision
-      if (msg.type === 'push') {
+      if (msg.type === 'continuous') {
+        this.continuous = msg.enabled
+      } else if (msg.type === 'push') {
         this.eosSeen = false
         this.chunks.push(msg.samples)
         this.buffered += msg.samples.length
@@ -55,7 +58,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
     }
     if (this.buffered === 0) {
       this.waiting = true // mid-reply underrun: re-buffer silently
-      if (this.eosSeen) {
+      if (this.eosSeen || this.continuous) {
         this.eosSeen = false
         this.port.postMessage({ type: 'drained', revision: this.revision })
       }
@@ -99,6 +102,9 @@ async function createPcmPlayer(onDrained) {
       if (samples.length === 0) return;
       node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
     },
+    setContinuousMode(enabled) {
+      node.port.postMessage({ type: "continuous", enabled, revision });
+    },
     endOfStream() {
       node.port.postMessage({ type: "eos", revision });
     },
@@ -143,9 +149,58 @@ function initialSnapshot() {
     error: null,
     errorDetails: void 0,
     interruptionMode: "none",
-    canCancel: false
+    canCancel: false,
+    artifacts: [],
+    session: null
   };
 }
+function readString(value) {
+  return typeof value === "string" && value ? value : void 0;
+}
+function readArtifactFile(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
+  const file = value;
+  const path = readString(file.path);
+  const mimeType = readString(file.mimeType);
+  if (!path || !mimeType) return void 0;
+  const language = readString(file.language);
+  return { path, mimeType, ...language ? { language } : {} };
+}
+function applyArtifactFrame(existing, frame, identity) {
+  if (frame.type === "artifact_start") {
+    const title = readString(frame.title);
+    const component = readString(frame.component);
+    const file = readArtifactFile(frame.file);
+    return {
+      ...identity,
+      artifactType: frame.artifactType === "component" ? "component" : "markdown",
+      ...title ? { title } : {},
+      ...file ? { file } : {},
+      ...component ? { component } : {},
+      content: "",
+      status: "streaming"
+    };
+  }
+  if (!existing) return void 0;
+  if (frame.type === "artifact_delta") {
+    const delta = typeof frame.delta === "string" ? frame.delta : "";
+    return delta ? { ...existing, content: existing.content + delta } : void 0;
+  }
+  if (frame.type === "artifact_update") {
+    const component = readString(frame.component) ?? existing.component;
+    const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
+    return {
+      ...existing,
+      ...component ? { component } : {},
+      ...props ? { props } : {}
+    };
+  }
+  if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
+  return void 0;
+}
+function readTurnId(msg) {
+  return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
+}
 var VoiceClient = class {
   constructor(options) {
     this.options = options;
@@ -166,6 +221,17 @@ var VoiceClient = class {
   stoppedResponse = false;
   responseEnded = true;
   hasPendingAudio = false;
+  /**
+   * The assistant entry built from clause captions for the turn being spoken. The
+   * authoritative transcript replaces it; a stop commits it, because it captions
+   * audio the caller already heard.
+   */
+  pendingAssistant = null;
+  /** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
+  committedAssistantTurnId = null;
+  activeTurnId = null;
+  suppressedArtifactTurnId = null;
+  fullDuplex = false;
   getSnapshot = () => this.snapshot;
   subscribe = (listener) => {
     this.listeners.add(listener);
@@ -187,7 +253,7 @@ var VoiceClient = class {
     if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
     this.cleanup();
     const generation = this.generation;
-    this.update({ ...initialSnapshot(), status: "connecting" });
+    this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
     try {
       const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
       if (generation !== this.generation) return;
@@ -202,10 +268,26 @@ var VoiceClient = class {
       url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
       url.search = "";
       url.searchParams.set("voiceProtocol", "runtype-browser-v1");
+      const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
+      url.searchParams.set("clientCapabilities", capabilities);
+      const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
+      if (sessionId) url.searchParams.set("sessionId", sessionId);
+      const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
+      if (generation !== this.generation) return;
+      if (this.options.visitorToken !== void 0 && !visitorToken) {
+        throw new Error(
+          "Voice visitor credential unavailable. Initialize the client session first."
+        );
+      }
+      if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
+        throw new Error("Invalid voice visitor credential.");
+      }
+      if (this.options.fullDuplex !== false)
+        url.searchParams.set("voiceCapabilities", "full-duplex-v1");
       url.hash = "";
-      const stream = await navigator.mediaDevices.getUserMedia({
+      const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
         audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
-      });
+      }));
       if (generation !== this.generation) {
         stream.getTracks().forEach((track) => track.stop());
         return;
@@ -225,7 +307,11 @@ var VoiceClient = class {
         return;
       }
       this.player = player;
-      const socket = new WebSocket(url.toString(), ["runtype.bearer", token]);
+      const socket = new WebSocket(url.toString(), [
+        "runtype.bearer",
+        token,
+        ...visitorToken ? [visitorToken] : []
+      ]);
       socket.binaryType = "arraybuffer";
       this.socket = socket;
       socket.onopen = () => {
@@ -256,6 +342,11 @@ var VoiceClient = class {
     this.cleanup();
     this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
   };
+  /** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
+  resetSession = () => {
+    if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
+    this.update({ session: null });
+  };
   toggleMute = () => {
     this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
   };
@@ -270,9 +361,11 @@ var VoiceClient = class {
     this.clearPlayback();
     if (this.snapshot.interruptionMode === "none") {
       this.stoppedResponse = !this.responseEnded;
+      this.commitClauseCaptions();
       this.setStatus("listening");
       return;
     }
+    this.suppressedArtifactTurnId = this.activeTurnId;
     this.awaitingClear = true;
     this.setStatus("listening");
     this.socket.send(JSON.stringify({ type: "cancel" }));
@@ -289,6 +382,7 @@ var VoiceClient = class {
     });
   }
   cleanup() {
+    this.fullDuplex = false;
     this.generation += 1;
     if (this.processor) this.processor.onaudioprocess = null;
     this.processor?.disconnect();
@@ -314,6 +408,82 @@ var VoiceClient = class {
     this.awaitingClear = false;
     this.stoppedResponse = false;
     this.responseEnded = true;
+    this.commitClauseCaptions();
+    this.committedAssistantTurnId = null;
+    this.activeTurnId = null;
+    this.suppressedArtifactTurnId = null;
+    const settledArtifacts = this.snapshot.artifacts.filter(
+      (artifact) => artifact.status === "complete"
+    );
+    if (settledArtifacts.length !== this.snapshot.artifacts.length)
+      this.update({ artifacts: settledArtifacts });
+  }
+  /**
+   * Folds an `artifact` message into the snapshot. Frames of a turn the caller
+   * cancelled are dropped; a playback-only stop leaves the turn running, so its
+   * artifacts keep arriving and are kept.
+   */
+  applyArtifactMessage(msg) {
+    if (this.awaitingClear) return;
+    const frame = msg.event;
+    if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
+    const event = frame;
+    const id = readString(event.id);
+    if (!id) return;
+    const turnId = readTurnId(msg);
+    if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
+    if (turnId !== null) this.activeTurnId = turnId;
+    const artifacts = this.snapshot.artifacts;
+    const index = artifacts.findIndex((artifact) => artifact.id === id);
+    const executionId = readString(msg.executionId);
+    const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
+      id,
+      turnId,
+      ...executionId ? { executionId } : {}
+    });
+    if (!next) return;
+    this.update({
+      artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
+    });
+  }
+  /** Settles the spoken clauses in place and ends the turn's claim on them. */
+  commitClauseCaptions() {
+    this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
+    const settled = this.pendingAssistant !== null;
+    this.pendingAssistant = null;
+    if (!settled) return;
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (last?.role !== "assistant" || last.partial !== true) return;
+    const { partial: _partial, ...committed } = last;
+    this.update({ transcript: [...transcript.slice(0, -1), committed] });
+  }
+  /** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
+  appendClauseCaption(text, turnId) {
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
+      return {
+        interimTranscript: null,
+        transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
+        status: "speaking"
+      };
+    }
+    this.pendingAssistant = { turnId };
+    return {
+      interimTranscript: null,
+      transcript: [
+        ...transcript,
+        {
+          role: "assistant",
+          content: text,
+          timestamp: Date.now(),
+          partial: true,
+          ...turnId ? { turnId } : {}
+        }
+      ],
+      status: "speaking"
+    };
   }
   clearPlayback() {
     this.playbackRevision += 1;
@@ -348,32 +518,89 @@ var VoiceClient = class {
       return;
     }
     switch (msg.type) {
-      case "session_config":
+      case "session_config": {
+        this.fullDuplex = msg.speechMode === "speech_to_speech";
+        if (this.fullDuplex) player.setContinuousMode?.(true);
         if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
           this.update({ interruptionMode: msg.interruptionMode });
         }
+        const sessionId = readString(msg.sessionId);
+        const conversationId = readString(msg.conversationId);
+        if (sessionId && conversationId) {
+          const session = { sessionId, conversationId };
+          this.update({ session });
+          this.options.onSession?.(session);
+        }
+        break;
+      }
+      case "artifact":
+        this.applyArtifactMessage(msg);
+        break;
+      case "transcript_update": {
+        if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
+          break;
+        if (msg.role === "assistant" && this.awaitingClear) break;
+        const entries = [...this.snapshot.transcript];
+        const index = entries.findIndex(
+          (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
+        );
+        const entry = {
+          role: msg.role,
+          content: msg.text,
+          turnId: msg.turnId,
+          isFinal: msg.final === true,
+          timestamp: index >= 0 ? entries[index].timestamp : Date.now()
+        };
+        if (index >= 0) entries[index] = entry;
+        else entries.push(entry);
+        this.update({ transcript: entries, interimTranscript: null });
+        break;
+      }
+      case "delegation_started":
+        if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
+        break;
+      case "delegation_completed":
+        if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
         break;
       case "transcript_interim":
         this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
         break;
-      case "transcript_final":
+      case "transcript_partial": {
+        if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
+        this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
+        if (this.awaitingClear || this.stoppedResponse) break;
+        this.responseEnded = false;
+        this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
+        break;
+      }
+      case "transcript_final": {
         if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
-        if (msg.role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const role = msg.role;
+        const text = msg.text;
+        if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const turnId = readTurnId(msg);
+        if (turnId !== null) this.activeTurnId = turnId;
+        if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
+          break;
         this.responseEnded = false;
+        const transcript = this.snapshot.transcript;
+        const last = transcript[transcript.length - 1];
+        const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
+        this.pendingAssistant = null;
+        const entry = {
+          role,
+          content: text,
+          // INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
+          timestamp: supersedes && last ? last.timestamp : Date.now(),
+          ...turnId ? { turnId } : {}
+        };
         this.update({
           interimTranscript: null,
-          transcript: [
-            ...this.snapshot.transcript,
-            {
-              role: msg.role,
-              content: msg.text,
-              timestamp: Date.now(),
-              ...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
-            }
-          ],
-          status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
+          transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
+          status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
         });
         break;
+      }
       case "audio_end": {
         this.responseEnded = true;
         if (this.stoppedResponse) {
@@ -390,10 +617,12 @@ var VoiceClient = class {
         break;
       }
       case "audio_clear":
+        this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
         this.clearPlayback();
         this.awaitingClear = false;
         this.stoppedResponse = false;
         this.responseEnded = true;
+        this.commitClauseCaptions();
         this.setStatus("listening");
         break;
       case "metrics": {
@@ -406,8 +635,11 @@ var VoiceClient = class {
             // @snake-case-ok: Existing voice wire contract.
             firstAudioMs: number(msg.first_audio_ms),
             // @snake-case-ok: Existing voice wire contract.
-            totalMs: number(msg.total_ms)
+            totalMs: number(msg.total_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            firstSynthesisMs: number(msg.first_synthesis_ms),
             // @snake-case-ok: Existing voice wire contract.
+            incremental: msg.incremental === true
           }
         });
         break;
@@ -439,14 +671,14 @@ var VoiceClient = class {
     const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
     this.processor = processor;
     processor.onaudioprocess = (event) => {
-      if (generation !== this.generation || this.snapshot.isMuted) return;
+      if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
       const input = event.inputBuffer.getChannelData(0);
       let sum = 0;
       for (const sample of input) sum += sample * sample;
-      this.update({ audioLevel: Math.sqrt(sum / input.length) });
+      this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
       if (socket.readyState !== WebSocket.OPEN) return;
       const pcm = new Int16Array(input.length);
-      if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
+      if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
         socket.send(pcm.buffer);
         return;
       }