@runtypelabs/voice 0.2.4 → 0.4.0

package/dist/persona.js

@@ -9,13 +9,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
     this.readOffset = 0
     this.buffered = 0
     this.waiting = true
-    // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
+    // INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
     this.eosSeen = false
+    this.continuous = false
     this.revision = 0
     this.port.onmessage = (e) => {
       const msg = e.data
       this.revision = msg.revision
-      if (msg.type === 'push') {
+      if (msg.type === 'continuous') {
+        this.continuous = msg.enabled
+      } else if (msg.type === 'push') {
         this.eosSeen = false
         this.chunks.push(msg.samples)
         this.buffered += msg.samples.length
@@ -55,7 +58,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
     }
     if (this.buffered === 0) {
       this.waiting = true // mid-reply underrun: re-buffer silently
-      if (this.eosSeen) {
+      if (this.eosSeen || this.continuous) {
         this.eosSeen = false
         this.port.postMessage({ type: 'drained', revision: this.revision })
       }
@@ -99,6 +102,9 @@ async function createPcmPlayer(onDrained) {
       if (samples.length === 0) return;
       node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
     },
+    setContinuousMode(enabled) {
+      node.port.postMessage({ type: "continuous", enabled, revision });
+    },
     endOfStream() {
       node.port.postMessage({ type: "eos", revision });
     },
@@ -143,9 +149,58 @@ function initialSnapshot() {
     error: null,
     errorDetails: void 0,
     interruptionMode: "none",
-    canCancel: false
+    canCancel: false,
+    artifacts: [],
+    session: null
   };
 }
+function readString(value) {
+  return typeof value === "string" && value ? value : void 0;
+}
+function readArtifactFile(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
+  const file = value;
+  const path = readString(file.path);
+  const mimeType = readString(file.mimeType);
+  if (!path || !mimeType) return void 0;
+  const language = readString(file.language);
+  return { path, mimeType, ...language ? { language } : {} };
+}
+function applyArtifactFrame(existing, frame, identity) {
+  if (frame.type === "artifact_start") {
+    const title = readString(frame.title);
+    const component = readString(frame.component);
+    const file = readArtifactFile(frame.file);
+    return {
+      ...identity,
+      artifactType: frame.artifactType === "component" ? "component" : "markdown",
+      ...title ? { title } : {},
+      ...file ? { file } : {},
+      ...component ? { component } : {},
+      content: "",
+      status: "streaming"
+    };
+  }
+  if (!existing) return void 0;
+  if (frame.type === "artifact_delta") {
+    const delta = typeof frame.delta === "string" ? frame.delta : "";
+    return delta ? { ...existing, content: existing.content + delta } : void 0;
+  }
+  if (frame.type === "artifact_update") {
+    const component = readString(frame.component) ?? existing.component;
+    const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
+    return {
+      ...existing,
+      ...component ? { component } : {},
+      ...props ? { props } : {}
+    };
+  }
+  if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
+  return void 0;
+}
+function readTurnId(msg) {
+  return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
+}
 var VoiceClient = class {
   constructor(options) {
     this.options = options;
@@ -166,6 +221,17 @@ var VoiceClient = class {
   stoppedResponse = false;
   responseEnded = true;
   hasPendingAudio = false;
+  /**
+   * The assistant entry built from clause captions for the turn being spoken. The
+   * authoritative transcript replaces it; a stop commits it, because it captions
+   * audio the caller already heard.
+   */
+  pendingAssistant = null;
+  /** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
+  committedAssistantTurnId = null;
+  activeTurnId = null;
+  suppressedArtifactTurnId = null;
+  fullDuplex = false;
   getSnapshot = () => this.snapshot;
   subscribe = (listener) => {
     this.listeners.add(listener);
@@ -187,7 +253,7 @@ var VoiceClient = class {
     if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
     this.cleanup();
     const generation = this.generation;
-    this.update({ ...initialSnapshot(), status: "connecting" });
+    this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
     try {
       const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
       if (generation !== this.generation) return;
@@ -202,10 +268,26 @@ var VoiceClient = class {
       url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
       url.search = "";
       url.searchParams.set("voiceProtocol", "runtype-browser-v1");
+      const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
+      url.searchParams.set("clientCapabilities", capabilities);
+      const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
+      if (sessionId) url.searchParams.set("sessionId", sessionId);
+      const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
+      if (generation !== this.generation) return;
+      if (this.options.visitorToken !== void 0 && !visitorToken) {
+        throw new Error(
+          "Voice visitor credential unavailable. Initialize the client session first."
+        );
+      }
+      if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
+        throw new Error("Invalid voice visitor credential.");
+      }
+      if (this.options.fullDuplex !== false)
+        url.searchParams.set("voiceCapabilities", "full-duplex-v1");
       url.hash = "";
-      const stream = await navigator.mediaDevices.getUserMedia({
+      const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
         audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
-      });
+      }));
       if (generation !== this.generation) {
         stream.getTracks().forEach((track) => track.stop());
         return;
@@ -225,7 +307,11 @@ var VoiceClient = class {
         return;
       }
       this.player = player;
-      const socket = new WebSocket(url.toString(), ["runtype.bearer", token]);
+      const socket = new WebSocket(url.toString(), [
+        "runtype.bearer",
+        token,
+        ...visitorToken ? [visitorToken] : []
+      ]);
       socket.binaryType = "arraybuffer";
       this.socket = socket;
       socket.onopen = () => {
@@ -256,6 +342,11 @@ var VoiceClient = class {
     this.cleanup();
     this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
   };
+  /** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
+  resetSession = () => {
+    if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
+    this.update({ session: null });
+  };
   toggleMute = () => {
     this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
   };
@@ -270,9 +361,11 @@ var VoiceClient = class {
     this.clearPlayback();
     if (this.snapshot.interruptionMode === "none") {
       this.stoppedResponse = !this.responseEnded;
+      this.commitClauseCaptions();
       this.setStatus("listening");
       return;
     }
+    this.suppressedArtifactTurnId = this.activeTurnId;
     this.awaitingClear = true;
     this.setStatus("listening");
     this.socket.send(JSON.stringify({ type: "cancel" }));
@@ -289,6 +382,7 @@ var VoiceClient = class {
     });
   }
   cleanup() {
+    this.fullDuplex = false;
     this.generation += 1;
     if (this.processor) this.processor.onaudioprocess = null;
     this.processor?.disconnect();
@@ -314,6 +408,82 @@ var VoiceClient = class {
     this.awaitingClear = false;
     this.stoppedResponse = false;
     this.responseEnded = true;
+    this.commitClauseCaptions();
+    this.committedAssistantTurnId = null;
+    this.activeTurnId = null;
+    this.suppressedArtifactTurnId = null;
+    const settledArtifacts = this.snapshot.artifacts.filter(
+      (artifact) => artifact.status === "complete"
+    );
+    if (settledArtifacts.length !== this.snapshot.artifacts.length)
+      this.update({ artifacts: settledArtifacts });
+  }
+  /**
+   * Folds an `artifact` message into the snapshot. Frames of a turn the caller
+   * cancelled are dropped; a playback-only stop leaves the turn running, so its
+   * artifacts keep arriving and are kept.
+   */
+  applyArtifactMessage(msg) {
+    if (this.awaitingClear) return;
+    const frame = msg.event;
+    if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
+    const event = frame;
+    const id = readString(event.id);
+    if (!id) return;
+    const turnId = readTurnId(msg);
+    if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
+    if (turnId !== null) this.activeTurnId = turnId;
+    const artifacts = this.snapshot.artifacts;
+    const index = artifacts.findIndex((artifact) => artifact.id === id);
+    const executionId = readString(msg.executionId);
+    const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
+      id,
+      turnId,
+      ...executionId ? { executionId } : {}
+    });
+    if (!next) return;
+    this.update({
+      artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
+    });
+  }
+  /** Settles the spoken clauses in place and ends the turn's claim on them. */
+  commitClauseCaptions() {
+    this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
+    const settled = this.pendingAssistant !== null;
+    this.pendingAssistant = null;
+    if (!settled) return;
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (last?.role !== "assistant" || last.partial !== true) return;
+    const { partial: _partial, ...committed } = last;
+    this.update({ transcript: [...transcript.slice(0, -1), committed] });
+  }
+  /** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
+  appendClauseCaption(text, turnId) {
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
+      return {
+        interimTranscript: null,
+        transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
+        status: "speaking"
+      };
+    }
+    this.pendingAssistant = { turnId };
+    return {
+      interimTranscript: null,
+      transcript: [
+        ...transcript,
+        {
+          role: "assistant",
+          content: text,
+          timestamp: Date.now(),
+          partial: true,
+          ...turnId ? { turnId } : {}
+        }
+      ],
+      status: "speaking"
+    };
   }
   clearPlayback() {
     this.playbackRevision += 1;
@@ -348,32 +518,89 @@ var VoiceClient = class {
       return;
     }
     switch (msg.type) {
-      case "session_config":
+      case "session_config": {
+        this.fullDuplex = msg.speechMode === "speech_to_speech";
+        if (this.fullDuplex) player.setContinuousMode?.(true);
         if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
           this.update({ interruptionMode: msg.interruptionMode });
         }
+        const sessionId = readString(msg.sessionId);
+        const conversationId = readString(msg.conversationId);
+        if (sessionId && conversationId) {
+          const session = { sessionId, conversationId };
+          this.update({ session });
+          this.options.onSession?.(session);
+        }
+        break;
+      }
+      case "artifact":
+        this.applyArtifactMessage(msg);
+        break;
+      case "transcript_update": {
+        if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
+          break;
+        if (msg.role === "assistant" && this.awaitingClear) break;
+        const entries = [...this.snapshot.transcript];
+        const index = entries.findIndex(
+          (entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
+        );
+        const entry = {
+          role: msg.role,
+          content: msg.text,
+          turnId: msg.turnId,
+          isFinal: msg.final === true,
+          timestamp: index >= 0 ? entries[index].timestamp : Date.now()
+        };
+        if (index >= 0) entries[index] = entry;
+        else entries.push(entry);
+        this.update({ transcript: entries, interimTranscript: null });
+        break;
+      }
+      case "delegation_started":
+        if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
+        break;
+      case "delegation_completed":
+        if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
         break;
       case "transcript_interim":
         this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
         break;
-      case "transcript_final":
+      case "transcript_partial": {
+        if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
+        this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
+        if (this.awaitingClear || this.stoppedResponse) break;
+        this.responseEnded = false;
+        this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
+        break;
+      }
+      case "transcript_final": {
         if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
-        if (msg.role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const role = msg.role;
+        const text = msg.text;
+        if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const turnId = readTurnId(msg);
+        if (turnId !== null) this.activeTurnId = turnId;
+        if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
+          break;
         this.responseEnded = false;
+        const transcript = this.snapshot.transcript;
+        const last = transcript[transcript.length - 1];
+        const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
+        this.pendingAssistant = null;
+        const entry = {
+          role,
+          content: text,
+          // INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
+          timestamp: supersedes && last ? last.timestamp : Date.now(),
+          ...turnId ? { turnId } : {}
+        };
         this.update({
           interimTranscript: null,
-          transcript: [
-            ...this.snapshot.transcript,
-            {
-              role: msg.role,
-              content: msg.text,
-              timestamp: Date.now(),
-              ...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
-            }
-          ],
-          status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
+          transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
+          status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
         });
         break;
+      }
       case "audio_end": {
         this.responseEnded = true;
         if (this.stoppedResponse) {
@@ -390,10 +617,12 @@ var VoiceClient = class {
         break;
       }
       case "audio_clear":
+        this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
         this.clearPlayback();
         this.awaitingClear = false;
         this.stoppedResponse = false;
         this.responseEnded = true;
+        this.commitClauseCaptions();
         this.setStatus("listening");
         break;
       case "metrics": {
@@ -406,8 +635,11 @@ var VoiceClient = class {
             // @snake-case-ok: Existing voice wire contract.
             firstAudioMs: number(msg.first_audio_ms),
             // @snake-case-ok: Existing voice wire contract.
-            totalMs: number(msg.total_ms)
+            totalMs: number(msg.total_ms),
             // @snake-case-ok: Existing voice wire contract.
+            firstSynthesisMs: number(msg.first_synthesis_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            incremental: msg.incremental === true
           }
         });
         break;
@@ -439,14 +671,14 @@ var VoiceClient = class {
     const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
     this.processor = processor;
     processor.onaudioprocess = (event) => {
-      if (generation !== this.generation || this.snapshot.isMuted) return;
+      if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
       const input = event.inputBuffer.getChannelData(0);
       let sum = 0;
       for (const sample of input) sum += sample * sample;
-      this.update({ audioLevel: Math.sqrt(sum / input.length) });
+      this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
       if (socket.readyState !== WebSocket.OPEN) return;
       const pcm = new Int16Array(input.length);
-      if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
+      if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
         socket.send(pcm.buffer);
         return;
       }
@@ -463,12 +695,13 @@ var VoiceClient = class {
 
 // src/persona.ts
 function createPersonaVoiceProvider(options) {
-  const client = new VoiceClient(options);
+  const client = new VoiceClient({ ...options, fullDuplex: false });
   const statusCallbacks = /* @__PURE__ */ new Set();
   const errorCallbacks = /* @__PURE__ */ new Set();
   const transcriptCallbacks = /* @__PURE__ */ new Set();
   const metricsCallbacks = /* @__PURE__ */ new Set();
   const levelCallbacks = /* @__PURE__ */ new Set();
+  const artifactCallbacks = /* @__PURE__ */ new Set();
   let previous = client.getSnapshot();
   let unsubscribe = null;
   const onSnapshot = () => {
@@ -492,12 +725,18 @@ function createPersonaVoiceProvider(options) {
       for (const callback of transcriptCallbacks)
         callback("user", snapshot.interimTranscript, false);
     }
-    for (const entry of snapshot.transcript.slice(before.transcript.length)) {
+    snapshot.artifacts.forEach((artifact, index) => {
+      if (artifact === before.artifacts[index]) return;
+      for (const callback of artifactCallbacks) callback(artifact);
+    });
+    snapshot.transcript.forEach((entry, index) => {
+      if (entry === before.transcript[index]) return;
+      const isFinal = entry.partial !== true && entry.isFinal !== false;
       for (const callback of transcriptCallbacks) {
-        if (entry.turnId) callback(entry.role, entry.content, true, { turnId: entry.turnId });
-        else callback(entry.role, entry.content, true);
+        if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
+        else callback(entry.role, entry.content, isFinal);
       }
-    }
+    });
   };
   const subscribe = () => {
     if (unsubscribe) return;
@@ -518,6 +757,7 @@ function createPersonaVoiceProvider(options) {
       transcriptCallbacks.clear();
       metricsCallbacks.clear();
       levelCallbacks.clear();
+      artifactCallbacks.clear();
     },
     startListening: async () => {
       subscribe();
@@ -544,6 +784,9 @@ function createPersonaVoiceProvider(options) {
     onLevel: (callback) => {
       levelCallbacks.add(callback);
     },
+    onArtifact: (callback) => {
+      artifactCallbacks.add(callback);
+    },
     getInterruptionMode: () => client.getSnapshot().interruptionMode,
     isBargeInActive: () => !["idle", "error"].includes(client.getSnapshot().status),
     deactivateBargeIn: async () => {
@@ -552,6 +795,52 @@ function createPersonaVoiceProvider(options) {
     stopPlayback: client.stopPlayback
   };
 }
+function toManualUpsert(artifact) {
+  const title = artifact.title ?? artifact.file?.path;
+  const transcript = artifact.status === "complete";
+  if (artifact.artifactType === "component") {
+    if (!artifact.component) return null;
+    return {
+      id: artifact.id,
+      artifactType: "component",
+      component: artifact.component,
+      ...title ? { title } : {},
+      ...artifact.props ? { props: artifact.props } : {},
+      transcript
+    };
+  }
+  return {
+    id: artifact.id,
+    artifactType: "markdown",
+    content: artifact.content,
+    ...title ? { title } : {},
+    ...artifact.file ? {
+      file: {
+        path: artifact.file.path,
+        mimeType: artifact.file.mimeType,
+        ...artifact.file.language ? { language: artifact.file.language } : {}
+      }
+    } : {},
+    transcript
+  };
+}
+function bindVoiceArtifactsToPersona(provider, widget, options) {
+  let released = false;
+  let shown = false;
+  provider.onArtifact((artifact) => {
+    if (released) return;
+    const manual = toManualUpsert(artifact);
+    if (!manual) return;
+    widget.upsertArtifact(manual);
+    if (shown || options?.showOnFirst === false) return;
+    shown = true;
+    widget.showArtifacts();
+  });
+  return () => {
+    released = true;
+  };
+}
 export {
+  bindVoiceArtifactsToPersona,
   createPersonaVoiceProvider
 };