@craftedxp/voice-js 0.3.2 → 0.4.0

package/dist/browser.mjs

@@ -354,7 +354,8 @@ function handleServerMessage(raw, state, cb) {
       state.agentBubbleId = id;
       state.transcript = [...state.transcript, { id, role: "agent", text: "" }];
       cb.onTranscript(state.transcript);
-      cb.onAgentTurnStart();
+      const seq = typeof msg.seq === "number" ? msg.seq : void 0;
+      cb.onAgentTurnStart(seq);
       setState(state, "agent_speaking", cb);
       return;
     }
@@ -368,10 +369,13 @@ function handleServerMessage(raw, state, cb) {
       cb.onTranscript(state.transcript);
       return;
     }
-    case "agent_turn_end":
+    case "agent_turn_end": {
       state.agentBubbleId = null;
+      const seq = typeof msg.seq === "number" ? msg.seq : void 0;
+      cb.onAgentTurnEnd(seq);
       setState(state, "listening", cb);
       return;
+    }
     case "interrupt":
       cb.onInterrupt();
       return;
@@ -558,6 +562,76 @@ var dispatchClientToolCall = (send, tools, frame) => {
   })();
 };
 
+// src/ClientMarksBuffer.ts
+var createClientMarksBuffer = (args) => {
+  const now = args.now ?? (() => performance.now());
+  let pendingFirstOutboundAt = null;
+  const inFlight = /* @__PURE__ */ new Map();
+  const tryEmit = (seq) => {
+    const slot = inFlight.get(seq);
+    if (!slot) return;
+    if (!slot.ended) return;
+    const marks = {};
+    if (slot.firstOutboundAt !== null && slot.firstAudibleAt !== null) {
+      marks.client_mic_to_first_audible_ms = slot.firstAudibleAt - slot.firstOutboundAt;
+    }
+    args.send({
+      type: "client_marks",
+      seq,
+      marks,
+      clientNow: Date.now()
+    });
+    inFlight.delete(seq);
+  };
+  const markFirstOutboundAudio = () => {
+    if (pendingFirstOutboundAt !== null) return;
+    pendingFirstOutboundAt = now();
+  };
+  const markFirstAudibleOutput = () => {
+    let target;
+    for (const slot of inFlight.values()) {
+      if (!slot.ended) {
+        target = slot;
+      }
+    }
+    if (!target) return;
+    if (target.firstAudibleAt !== null) return;
+    target.firstAudibleAt = now();
+  };
+  const onAgentTurnStart = (seq) => {
+    inFlight.set(seq, {
+      firstOutboundAt: pendingFirstOutboundAt,
+      firstAudibleAt: null,
+      ended: false
+    });
+    pendingFirstOutboundAt = null;
+  };
+  const onAgentTurnEnd = (seq) => {
+    const slot = inFlight.get(seq);
+    if (!slot) {
+      args.send({ type: "client_marks", seq, marks: {}, clientNow: Date.now() });
+      return;
+    }
+    slot.ended = true;
+    tryEmit(seq);
+  };
+  const flush = () => {
+    for (const seq of [...inFlight.keys()]) {
+      const slot = inFlight.get(seq);
+      slot.ended = true;
+      tryEmit(seq);
+    }
+    pendingFirstOutboundAt = null;
+  };
+  return {
+    markFirstOutboundAudio,
+    markFirstAudibleOutput,
+    onAgentTurnStart,
+    onAgentTurnEnd,
+    flush
+  };
+};
+
 // src/VoiceClient.ts
 var BrowserVoiceClient = class {
   constructor(args) {
@@ -621,7 +695,13 @@ var BrowserVoiceClient = class {
                 this.playback?.flush();
                 this.args.options.onInterrupt?.();
               },
-              onAgentTurnStart: () => this.args.options.onAgentTurnStart?.(),
+              onAgentTurnStart: (seq) => {
+                if (typeof seq === "number") this.marks.onAgentTurnStart(seq);
+                this.args.options.onAgentTurnStart?.();
+              },
+              onAgentTurnEnd: (seq) => {
+                if (typeof seq === "number") this.marks.onAgentTurnEnd(seq);
+              },
               onCallEnd: (reason) => this.teardown(reason),
               onConnected: () => this.sendClientToolsRegister(),
               onClientToolCall: (frame) => dispatchClientToolCall(
@@ -631,6 +711,7 @@ var BrowserVoiceClient = class {
               )
             });
           } else {
+            this.marks.markFirstAudibleOutput();
             this.playback?.enqueue(ev.data);
           }
           break;
@@ -649,6 +730,7 @@ var BrowserVoiceClient = class {
       if (this.capture?.isCapturing()) return;
       this.capture = createAudioCapture({
         onChunk: (pcm) => {
+          this.marks.markFirstOutboundAudio();
           this.rws?.send(pcm);
         },
         onVolume: (v) => {
@@ -669,6 +751,10 @@ var BrowserVoiceClient = class {
       }
     };
     this.teardown = (reason) => {
+      try {
+        this.marks.flush();
+      } catch {
+      }
       this.capture?.stop();
       this.capture = null;
       this.playback?.close();
@@ -694,6 +780,14 @@ var BrowserVoiceClient = class {
     this.args = args;
     this.proto = createProtocolState();
     validateClientToolMap(args.options.clientTools);
+    this.marks = createClientMarksBuffer({
+      send: (frame) => {
+        try {
+          this.rws?.send(JSON.stringify(frame));
+        } catch {
+        }
+      }
+    });
   }
   // ---------------------------------------------------------------
   // Call interface
@@ -742,6 +836,161 @@ var BrowserVoiceClient = class {
   }
 };
 
+// src/webrtc/createWebRtcCall.ts
+async function createWebRtcCall(opts) {
+  const proto = createProtocolState();
+  let muted = false;
+  let ended = false;
+  const fireState = (next) => {
+    if (proto.state === next) return;
+    proto.state = next;
+    opts.onStateChange?.(next);
+  };
+  const dispatch = (raw) => {
+    handleServerMessage(raw, proto, {
+      onState: fireState,
+      onTranscript: (entries) => opts.onTranscript?.(entries),
+      onError: (err) => opts.onError?.(err),
+      onInterrupt: () => opts.onInterrupt?.(),
+      onAgentTurnStart: () => opts.onAgentTurnStart?.(),
+      onAgentTurnEnd: () => {
+      },
+      onCallEnd: () => teardown(),
+      onConnected: () => {
+      },
+      onClientToolCall: () => {
+      }
+    });
+  };
+  fireState("connecting");
+  const pc = new RTCPeerConnection({
+    iceServers: [{ urls: "stun:stun.l.google.com:19302" }]
+  });
+  const audioEl = document.createElement("audio");
+  audioEl.autoplay = true;
+  audioEl.style.display = "none";
+  document.body.appendChild(audioEl);
+  pc.ontrack = (event) => {
+    audioEl.srcObject = event.streams[0] ?? new MediaStream([event.track]);
+  };
+  let mic;
+  try {
+    mic = await navigator.mediaDevices.getUserMedia({ audio: true });
+  } catch (err) {
+    const code = err instanceof DOMException && err.name === "NotAllowedError" ? "mic_denied" : "mic_start_failed";
+    opts.onError?.({
+      code,
+      message: err instanceof Error ? err.message : "getUserMedia failed"
+    });
+    fireState("error");
+    pc.close();
+    audioEl.remove();
+    throw err;
+  }
+  for (const track of mic.getAudioTracks()) pc.addTrack(track, mic);
+  const dc = pc.createDataChannel("control", { ordered: true });
+  dc.onmessage = (e) => {
+    if (typeof e.data === "string") dispatch(e.data);
+  };
+  dc.onerror = () => {
+    opts.onError?.({ code: "socket_error", message: "control channel error" });
+  };
+  const gateway = opts.webrtcGatewayBase || "";
+  const offerUrl = gateway ? `${gateway}/webrtc/offer?token=${encodeURIComponent(opts.token)}` : `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/offer?token=${encodeURIComponent(opts.token)}`;
+  const iceUrl = gateway ? `${gateway}/webrtc/ice?token=${encodeURIComponent(opts.token)}` : `${opts.apiBase}/v1/agents/${encodeURIComponent(opts.agentId)}/webrtc/ice?token=${encodeURIComponent(opts.token)}`;
+  await pc.setLocalDescription(await pc.createOffer());
+  let callId;
+  try {
+    const offerRes = await fetch(offerUrl, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ sdp: pc.localDescription.sdp, type: "offer", agentId: opts.agentId })
+    });
+    if (!offerRes.ok) {
+      const code = offerRes.status === 401 ? "unauthorized" : "server_error";
+      opts.onError?.({ code, message: `signaling failed: HTTP ${offerRes.status}` });
+      fireState("error");
+      mic.getTracks().forEach((t) => t.stop());
+      pc.close();
+      audioEl.remove();
+      throw new Error(`webrtc offer failed: ${offerRes.status}`);
+    }
+    const body = await offerRes.json();
+    callId = body.callId;
+    await pc.setRemoteDescription({ type: "answer", sdp: body.sdp });
+  } catch (err) {
+    if (!ended) {
+      opts.onError?.({
+        code: "network_unreachable",
+        message: err instanceof Error ? err.message : "signaling failed"
+      });
+      fireState("error");
+      mic.getTracks().forEach((t) => t.stop());
+      pc.close();
+      audioEl.remove();
+    }
+    throw err;
+  }
+  pc.onicecandidate = (e) => {
+    if (!e.candidate) return;
+    void fetch(iceUrl, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ callId, candidate: e.candidate })
+    }).catch(() => {
+    });
+  };
+  pc.onconnectionstatechange = () => {
+    const s = pc.connectionState;
+    if (s === "connected") fireState("listening");
+    if (s === "failed" || s === "disconnected") {
+      opts.onError?.({ code: "socket_error", message: `webrtc connection ${s}` });
+      teardown();
+    }
+    if (s === "closed" && !ended) teardown();
+  };
+  const teardown = () => {
+    if (ended) return;
+    ended = true;
+    try {
+      mic.getTracks().forEach((t) => t.stop());
+    } catch {
+    }
+    try {
+      pc.close();
+    } catch {
+    }
+    try {
+      audioEl.remove();
+    } catch {
+    }
+    fireState("ended");
+    opts.onEnd?.();
+  };
+  return {
+    get state() {
+      return proto.state;
+    },
+    get transcript() {
+      return proto.transcript.slice();
+    },
+    get isMuted() {
+      return muted;
+    },
+    end: () => teardown(),
+    mute: () => {
+      if (muted) return;
+      muted = true;
+      mic.getAudioTracks().forEach((t) => t.enabled = false);
+    },
+    unmute: () => {
+      if (!muted) return;
+      muted = false;
+      mic.getAudioTracks().forEach((t) => t.enabled = true);
+    }
+  };
+}
+
 // src/browser.ts
 var browserWsFactory = (url) => new globalThis.WebSocket(url);
 var BrowserVoiceFactory = class {
@@ -757,21 +1006,42 @@ var BrowserVoiceFactory = class {
         context,
         metadata
       };
-      let token;
+      let resolved;
       if (options.token) {
-        token = options.token;
+        resolved = { token: options.token, transport: "ws" };
       } else {
-        token = await this.config.fetchToken(fetchArgs);
-        if (!token) {
+        const r = await this.config.fetchToken(fetchArgs);
+        if (!r) {
           throw new Error("configureVoiceClient.fetchToken returned empty token");
         }
+        resolved = typeof r === "string" ? { token: r, transport: "ws" } : r;
+        if (!resolved.token) {
+          throw new Error("configureVoiceClient.fetchToken returned an object without `token`");
+        }
+      }
+      if (resolved.transport === "webrtc") {
+        return createWebRtcCall({
+          agentId: options.agentId,
+          apiBase: this.config.apiBase,
+          token: resolved.token,
+          webrtcGatewayBase: resolved.webrtcGatewayBase,
+          onStateChange: options.onStateChange,
+          onTranscript: options.onTranscript,
+          onError: options.onError,
+          // Synthesise a minimal CallEndEvent. WebRTC doesn't carry an end reason
+          // from the server yet — use 'agent_ended' as placeholder. durationMs is
+          // tracked at 0 until the followup lands (see spec Followups section).
+          onEnd: options.onEnd ? () => options.onEnd({ reason: "agent_ended", durationMs: 0 }) : void 0,
+          onInterrupt: options.onInterrupt,
+          onAgentTurnStart: options.onAgentTurnStart
+        });
       }
       const client = new BrowserVoiceClient({
         config: this.config,
         // Carry merged context/metadata through to startCall so server can
         // see what the SDK saw.
         options: { ...options, context, metadata },
-        token,
+        token: resolved.token,
         wsFactory: browserWsFactory
       });
       await client.start();