@runtypelabs/voice 0.2.3 → 0.3.0

package/dist/persona.js

@@ -143,9 +143,58 @@ function initialSnapshot() {
     error: null,
     errorDetails: void 0,
     interruptionMode: "none",
-    canCancel: false
+    canCancel: false,
+    artifacts: [],
+    session: null
   };
 }
+function readString(value) {
+  return typeof value === "string" && value ? value : void 0;
+}
+function readArtifactFile(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
+  const file = value;
+  const path = readString(file.path);
+  const mimeType = readString(file.mimeType);
+  if (!path || !mimeType) return void 0;
+  const language = readString(file.language);
+  return { path, mimeType, ...language ? { language } : {} };
+}
+function applyArtifactFrame(existing, frame, identity) {
+  if (frame.type === "artifact_start") {
+    const title = readString(frame.title);
+    const component = readString(frame.component);
+    const file = readArtifactFile(frame.file);
+    return {
+      ...identity,
+      artifactType: frame.artifactType === "component" ? "component" : "markdown",
+      ...title ? { title } : {},
+      ...file ? { file } : {},
+      ...component ? { component } : {},
+      content: "",
+      status: "streaming"
+    };
+  }
+  if (!existing) return void 0;
+  if (frame.type === "artifact_delta") {
+    const delta = typeof frame.delta === "string" ? frame.delta : "";
+    return delta ? { ...existing, content: existing.content + delta } : void 0;
+  }
+  if (frame.type === "artifact_update") {
+    const component = readString(frame.component) ?? existing.component;
+    const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
+    return {
+      ...existing,
+      ...component ? { component } : {},
+      ...props ? { props } : {}
+    };
+  }
+  if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
+  return void 0;
+}
+function readTurnId(msg) {
+  return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
+}
 var VoiceClient = class {
   constructor(options) {
     this.options = options;
@@ -166,6 +215,16 @@ var VoiceClient = class {
   stoppedResponse = false;
   responseEnded = true;
   hasPendingAudio = false;
+  /**
+   * The assistant entry built from clause captions for the turn being spoken. The
+   * authoritative transcript replaces it; a stop commits it, because it captions
+   * audio the caller already heard.
+   */
+  pendingAssistant = null;
+  /** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
+  committedAssistantTurnId = null;
+  activeTurnId = null;
+  suppressedArtifactTurnId = null;
   getSnapshot = () => this.snapshot;
   subscribe = (listener) => {
     this.listeners.add(listener);
@@ -187,7 +246,7 @@ var VoiceClient = class {
     if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
     this.cleanup();
     const generation = this.generation;
-    this.update({ ...initialSnapshot(), status: "connecting" });
+    this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
     try {
       const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
       if (generation !== this.generation) return;
@@ -202,6 +261,20 @@ var VoiceClient = class {
       url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
       url.search = "";
       url.searchParams.set("voiceProtocol", "runtype-browser-v1");
+      const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
+      url.searchParams.set("clientCapabilities", capabilities);
+      const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
+      if (sessionId) url.searchParams.set("sessionId", sessionId);
+      const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
+      if (generation !== this.generation) return;
+      if (this.options.visitorToken !== void 0 && !visitorToken) {
+        throw new Error(
+          "Voice visitor credential unavailable. Initialize the client session first."
+        );
+      }
+      if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
+        throw new Error("Invalid voice visitor credential.");
+      }
       url.hash = "";
       const stream = await navigator.mediaDevices.getUserMedia({
         audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
@@ -225,7 +298,11 @@ var VoiceClient = class {
         return;
       }
       this.player = player;
-      const socket = new WebSocket(url.toString(), ["runtype.bearer", token]);
+      const socket = new WebSocket(url.toString(), [
+        "runtype.bearer",
+        token,
+        ...visitorToken ? [visitorToken] : []
+      ]);
       socket.binaryType = "arraybuffer";
       this.socket = socket;
       socket.onopen = () => {
@@ -256,6 +333,11 @@ var VoiceClient = class {
     this.cleanup();
     this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
   };
+  /** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
+  resetSession = () => {
+    if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
+    this.update({ session: null });
+  };
   toggleMute = () => {
     this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
   };
@@ -270,9 +352,11 @@ var VoiceClient = class {
     this.clearPlayback();
     if (this.snapshot.interruptionMode === "none") {
       this.stoppedResponse = !this.responseEnded;
+      this.commitClauseCaptions();
       this.setStatus("listening");
       return;
     }
+    this.suppressedArtifactTurnId = this.activeTurnId;
     this.awaitingClear = true;
     this.setStatus("listening");
     this.socket.send(JSON.stringify({ type: "cancel" }));
@@ -314,6 +398,82 @@ var VoiceClient = class {
     this.awaitingClear = false;
     this.stoppedResponse = false;
     this.responseEnded = true;
+    this.commitClauseCaptions();
+    this.committedAssistantTurnId = null;
+    this.activeTurnId = null;
+    this.suppressedArtifactTurnId = null;
+    const settledArtifacts = this.snapshot.artifacts.filter(
+      (artifact) => artifact.status === "complete"
+    );
+    if (settledArtifacts.length !== this.snapshot.artifacts.length)
+      this.update({ artifacts: settledArtifacts });
+  }
+  /**
+   * Folds an `artifact` message into the snapshot. Frames of a turn the caller
+   * cancelled are dropped; a playback-only stop leaves the turn running, so its
+   * artifacts keep arriving and are kept.
+   */
+  applyArtifactMessage(msg) {
+    if (this.awaitingClear) return;
+    const frame = msg.event;
+    if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
+    const event = frame;
+    const id = readString(event.id);
+    if (!id) return;
+    const turnId = readTurnId(msg);
+    if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
+    if (turnId !== null) this.activeTurnId = turnId;
+    const artifacts = this.snapshot.artifacts;
+    const index = artifacts.findIndex((artifact) => artifact.id === id);
+    const executionId = readString(msg.executionId);
+    const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
+      id,
+      turnId,
+      ...executionId ? { executionId } : {}
+    });
+    if (!next) return;
+    this.update({
+      artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
+    });
+  }
+  /** Settles the spoken clauses in place and ends the turn's claim on them. */
+  commitClauseCaptions() {
+    this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
+    const settled = this.pendingAssistant !== null;
+    this.pendingAssistant = null;
+    if (!settled) return;
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (last?.role !== "assistant" || last.partial !== true) return;
+    const { partial: _partial, ...committed } = last;
+    this.update({ transcript: [...transcript.slice(0, -1), committed] });
+  }
+  /** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
+  appendClauseCaption(text, turnId) {
+    const transcript = this.snapshot.transcript;
+    const last = transcript[transcript.length - 1];
+    if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
+      return {
+        interimTranscript: null,
+        transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
+        status: "speaking"
+      };
+    }
+    this.pendingAssistant = { turnId };
+    return {
+      interimTranscript: null,
+      transcript: [
+        ...transcript,
+        {
+          role: "assistant",
+          content: text,
+          timestamp: Date.now(),
+          partial: true,
+          ...turnId ? { turnId } : {}
+        }
+      ],
+      status: "speaking"
+    };
   }
   clearPlayback() {
     this.playbackRevision += 1;
@@ -348,32 +508,61 @@ var VoiceClient = class {
       return;
     }
     switch (msg.type) {
-      case "session_config":
+      case "session_config": {
         if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
           this.update({ interruptionMode: msg.interruptionMode });
         }
+        const sessionId = readString(msg.sessionId);
+        const conversationId = readString(msg.conversationId);
+        if (sessionId && conversationId) {
+          const session = { sessionId, conversationId };
+          this.update({ session });
+          this.options.onSession?.(session);
+        }
+        break;
+      }
+      case "artifact":
+        this.applyArtifactMessage(msg);
         break;
       case "transcript_interim":
         this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
         break;
-      case "transcript_final":
+      case "transcript_partial": {
+        if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
+        this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
+        if (this.awaitingClear || this.stoppedResponse) break;
+        this.responseEnded = false;
+        this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
+        break;
+      }
+      case "transcript_final": {
         if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
-        if (msg.role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const role = msg.role;
+        const text = msg.text;
+        if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        const turnId = readTurnId(msg);
+        if (turnId !== null) this.activeTurnId = turnId;
+        if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
+          break;
         this.responseEnded = false;
+        const transcript = this.snapshot.transcript;
+        const last = transcript[transcript.length - 1];
+        const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
+        this.pendingAssistant = null;
+        const entry = {
+          role,
+          content: text,
+          // INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
+          timestamp: supersedes && last ? last.timestamp : Date.now(),
+          ...turnId ? { turnId } : {}
+        };
         this.update({
           interimTranscript: null,
-          transcript: [
-            ...this.snapshot.transcript,
-            {
-              role: msg.role,
-              content: msg.text,
-              timestamp: Date.now(),
-              ...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
-            }
-          ],
-          status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
+          transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
+          status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
         });
         break;
+      }
       case "audio_end": {
         this.responseEnded = true;
         if (this.stoppedResponse) {
@@ -390,10 +579,12 @@ var VoiceClient = class {
         break;
       }
       case "audio_clear":
+        this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
         this.clearPlayback();
         this.awaitingClear = false;
         this.stoppedResponse = false;
         this.responseEnded = true;
+        this.commitClauseCaptions();
         this.setStatus("listening");
         break;
       case "metrics": {
@@ -406,8 +597,11 @@ var VoiceClient = class {
             // @snake-case-ok: Existing voice wire contract.
             firstAudioMs: number(msg.first_audio_ms),
             // @snake-case-ok: Existing voice wire contract.
-            totalMs: number(msg.total_ms)
+            totalMs: number(msg.total_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            firstSynthesisMs: number(msg.first_synthesis_ms),
             // @snake-case-ok: Existing voice wire contract.
+            incremental: msg.incremental === true
           }
         });
         break;
@@ -469,6 +663,7 @@ function createPersonaVoiceProvider(options) {
   const transcriptCallbacks = /* @__PURE__ */ new Set();
   const metricsCallbacks = /* @__PURE__ */ new Set();
   const levelCallbacks = /* @__PURE__ */ new Set();
+  const artifactCallbacks = /* @__PURE__ */ new Set();
   let previous = client.getSnapshot();
   let unsubscribe = null;
   const onSnapshot = () => {
@@ -492,12 +687,18 @@ function createPersonaVoiceProvider(options) {
       for (const callback of transcriptCallbacks)
         callback("user", snapshot.interimTranscript, false);
     }
-    for (const entry of snapshot.transcript.slice(before.transcript.length)) {
+    snapshot.artifacts.forEach((artifact, index) => {
+      if (artifact === before.artifacts[index]) return;
+      for (const callback of artifactCallbacks) callback(artifact);
+    });
+    snapshot.transcript.forEach((entry, index) => {
+      if (entry === before.transcript[index]) return;
+      const isFinal = entry.partial !== true;
       for (const callback of transcriptCallbacks) {
-        if (entry.turnId) callback(entry.role, entry.content, true, { turnId: entry.turnId });
-        else callback(entry.role, entry.content, true);
+        if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
+        else callback(entry.role, entry.content, isFinal);
       }
-    }
+    });
   };
   const subscribe = () => {
     if (unsubscribe) return;
@@ -518,6 +719,7 @@ function createPersonaVoiceProvider(options) {
       transcriptCallbacks.clear();
       metricsCallbacks.clear();
       levelCallbacks.clear();
+      artifactCallbacks.clear();
     },
     startListening: async () => {
       subscribe();
@@ -544,6 +746,9 @@ function createPersonaVoiceProvider(options) {
     onLevel: (callback) => {
       levelCallbacks.add(callback);
     },
+    onArtifact: (callback) => {
+      artifactCallbacks.add(callback);
+    },
     getInterruptionMode: () => client.getSnapshot().interruptionMode,
     isBargeInActive: () => !["idle", "error"].includes(client.getSnapshot().status),
     deactivateBargeIn: async () => {
@@ -552,6 +757,52 @@ function createPersonaVoiceProvider(options) {
     stopPlayback: client.stopPlayback
   };
 }
+function toManualUpsert(artifact) {
+  const title = artifact.title ?? artifact.file?.path;
+  const transcript = artifact.status === "complete";
+  if (artifact.artifactType === "component") {
+    if (!artifact.component) return null;
+    return {
+      id: artifact.id,
+      artifactType: "component",
+      component: artifact.component,
+      ...title ? { title } : {},
+      ...artifact.props ? { props: artifact.props } : {},
+      transcript
+    };
+  }
+  return {
+    id: artifact.id,
+    artifactType: "markdown",
+    content: artifact.content,
+    ...title ? { title } : {},
+    ...artifact.file ? {
+      file: {
+        path: artifact.file.path,
+        mimeType: artifact.file.mimeType,
+        ...artifact.file.language ? { language: artifact.file.language } : {}
+      }
+    } : {},
+    transcript
+  };
+}
+function bindVoiceArtifactsToPersona(provider, widget, options) {
+  let released = false;
+  let shown = false;
+  provider.onArtifact((artifact) => {
+    if (released) return;
+    const manual = toManualUpsert(artifact);
+    if (!manual) return;
+    widget.upsertArtifact(manual);
+    if (shown || options?.showOnFirst === false) return;
+    shown = true;
+    widget.showArtifacts();
+  });
+  return () => {
+    released = true;
+  };
+}
 export {
+  bindVoiceArtifactsToPersona,
   createPersonaVoiceProvider
 };