@craftedxp/voice-js 0.9.0 → 0.12.0

package/dist/assistant.mjs

@@ -8,7 +8,8 @@ import {
   handleServerMessage,
   parseIncomingCall,
   startTextSession
-} from "./chunk-ZW22Y67M.mjs";
+} from "./chunk-C2U6V5NI.mjs";
+import "./chunk-OS2GZAR7.mjs";
 export {
   buildWsUrl,
   configureVoiceClient,

package/dist/browser.d.mts

@@ -1,8 +1,8 @@
-import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-D2TbvIqT.mjs';
-export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as ChatEvent, h as ClientTool, i as ClientToolMap, F as FetchToken, j as FetchTokenArgs, k as FetchTokenResult, P as ProtocolCallbacks, l as ProtocolState, S as ServerMessage, m as StartCallOptions, n as StartTextSessionOpts, T as TextSession, o as TranscriptEntry, p as VolumeEvent, q as buildWsUrl, r as createProtocolState, s as handleServerMessage, t as startTextSession } from './config-D2TbvIqT.mjs';
+import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-DfbPB1Tq.mjs';
+export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as CallTransport, h as ChatEvent, i as ClientTool, j as ClientToolMap, F as FetchToken, k as FetchTokenArgs, l as FetchTokenResult, P as ProtocolCallbacks, m as ProtocolState, S as ServerMessage, n as StartCallOptions, o as StartTextSessionOpts, T as TextSession, p as TranscriptEntry, q as TransportChangeEvent, r as VolumeEvent, s as buildWsUrl, t as createProtocolState, u as handleServerMessage, v as startTextSession } from './config-DfbPB1Tq.mjs';
 import { JoinRoomOptions, RoomSession } from './room.mjs';
 export { AnalysisMessage, RoomEventName, RoomEventPayloads, RoomParticipantInfo, SystemMessage, TranscriptMessage, joinRoom } from './room.mjs';
-export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CfRRzj2P.mjs';
+export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-2B3XB8pM.mjs';
 import 'livekit-client';
 
 /**

package/dist/browser.d.ts

@@ -1,8 +1,8 @@
-import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-D2TbvIqT.js';
-export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as ChatEvent, h as ClientTool, i as ClientToolMap, F as FetchToken, j as FetchTokenArgs, k as FetchTokenResult, P as ProtocolCallbacks, l as ProtocolState, S as ServerMessage, m as StartCallOptions, n as StartTextSessionOpts, T as TextSession, o as TranscriptEntry, p as VolumeEvent, q as buildWsUrl, r as createProtocolState, s as handleServerMessage, t as startTextSession } from './config-D2TbvIqT.js';
+import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-DfbPB1Tq.js';
+export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as CallTransport, h as ChatEvent, i as ClientTool, j as ClientToolMap, F as FetchToken, k as FetchTokenArgs, l as FetchTokenResult, P as ProtocolCallbacks, m as ProtocolState, S as ServerMessage, n as StartCallOptions, o as StartTextSessionOpts, T as TextSession, p as TranscriptEntry, q as TransportChangeEvent, r as VolumeEvent, s as buildWsUrl, t as createProtocolState, u as handleServerMessage, v as startTextSession } from './config-DfbPB1Tq.js';
 import { JoinRoomOptions, RoomSession } from './room.js';
 export { AnalysisMessage, RoomEventName, RoomEventPayloads, RoomParticipantInfo, SystemMessage, TranscriptMessage, joinRoom } from './room.js';
-export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CfRRzj2P.js';
+export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CYmeQkIw.js';
 import 'livekit-client';
 
 /**

package/dist/browser.js

@@ -523,7 +523,8 @@ function buildWsUrl(args) {
   const base = new URL(args.apiBase);
   const proto = base.protocol === "https:" ? "wss:" : "ws:";
   const bargeQS = args.bargeIn === false ? "&barge=off" : "";
-  return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}`;
+  const mediaQS = args.media === "livekit" ? "&media=livekit" : "";
+  return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}${mediaQS}`;
 }
 
 // src/clientTools.ts
@@ -677,7 +678,75 @@ var createClientMarksBuffer = (args) => {
   };
 };
 
+// src/livekit/createLiveKitMedia.ts
+var toError = (e) => e instanceof Error ? e : new Error(String(e));
+function createLiveKitMedia(lk, opts = {}) {
+  let room = null;
+  let audioEl = null;
+  let muted = false;
+  const attach = (track) => {
+    if (track.kind !== lk.Track.Kind.Audio) return;
+    if (typeof document === "undefined") return;
+    if (!audioEl) {
+      audioEl = document.createElement("audio");
+      audioEl.autoplay = true;
+      audioEl.style.display = "none";
+      document.body.appendChild(audioEl);
+    }
+    track.attach(audioEl);
+  };
+  return {
+    connect: async (url, token) => {
+      const r = new lk.Room({ adaptiveStream: false, dynacast: false });
+      room = r;
+      r.on(lk.RoomEvent.TrackSubscribed, (track) => attach(track));
+      await r.connect(url, token);
+      if (room !== r) {
+        return;
+      }
+      await r.localParticipant.setMicrophoneEnabled(!muted);
+      if (room !== r) {
+        return;
+      }
+      try {
+        await r.startAudio();
+      } catch {
+      }
+    },
+    disconnect: () => {
+      const r = room;
+      room = null;
+      if (r) void r.disconnect().catch(() => void 0);
+      audioEl?.remove();
+      audioEl = null;
+    },
+    setMuted: (m) => {
+      muted = m;
+      const r = room;
+      if (!r) return;
+      void r.localParticipant.setMicrophoneEnabled(!m).catch((e) => opts.onError?.(toError(e)));
+    }
+  };
+}
+
 // src/VoiceClient.ts
+var parseMediaFrame = (raw) => {
+  if (!raw.includes('"livekit_')) return null;
+  try {
+    const m = JSON.parse(raw);
+    if (m.type === "livekit_join" && typeof m.url === "string" && typeof m.token === "string") {
+      return { type: "livekit_join", url: m.url, token: m.token };
+    }
+    if (m.type === "livekit_fallback") {
+      return {
+        type: "livekit_fallback",
+        reason: typeof m.reason === "string" ? m.reason : void 0
+      };
+    }
+  } catch {
+  }
+  return null;
+};
 var BrowserVoiceClient = class {
   constructor(args) {
     this.rws = null;
@@ -689,6 +758,9 @@ var BrowserVoiceClient = class {
     this.startedAt = null;
     this.endedFired = false;
     this.lastError = null;
+    this.mediaMode = "ws";
+    this.lkMedia = null;
+    this.lkJoinSeen = false;
     this.end = () => {
       this.teardown("user_hangup");
     };
@@ -696,11 +768,73 @@ var BrowserVoiceClient = class {
       if (this.muted) return;
       this.muted = true;
       this.capture?.mute(true);
+      this.lkMedia?.setMuted(true);
     };
     this.unmute = () => {
       if (!this.muted) return;
       this.muted = false;
       this.capture?.mute(false);
+      this.lkMedia?.setMuted(false);
+    };
+    this.ensureWsPlayback = async () => {
+      if (this.playback) return;
+      const createPlayback = this.args.audio?.createPlayback ?? createAudioPlayback;
+      this.playback = createPlayback({
+        onVolume: (v) => {
+          this.outputVolume = v;
+          this.args.options.onVolume?.({ input: this.inputVolume, output: v });
+        }
+      });
+      try {
+        await this.playback.resume();
+      } catch {
+      }
+    };
+    // ---------------------------------------------------------------
+    // LiveKit media lifecycle (phase 35)
+    // ---------------------------------------------------------------
+    this.joinLiveKit = async (url, token) => {
+      this.lkJoinSeen = true;
+      const lk = this.args.media?.lk;
+      if (!lk) {
+        this.fallbackToWs("livekit-client unavailable");
+        return;
+      }
+      const media = createLiveKitMedia(lk, {
+        onError: (e) => this.emitError({ code: "mic_start_failed", message: e.message })
+      });
+      this.lkMedia = media;
+      if (this.muted) media.setMuted(true);
+      try {
+        await media.connect(url, token);
+        if (this.lkMedia !== media) return;
+        this.mediaMode = "livekit";
+        this.rws?.send(JSON.stringify({ type: "livekit_ready" }));
+      } catch (err) {
+        const reason = err instanceof Error ? err.message : String(err);
+        media.disconnect();
+        if (this.lkMedia === media) this.lkMedia = null;
+        this.rws?.send(JSON.stringify({ type: "livekit_failed", reason }));
+        this.fallbackToWs(reason);
+      }
+    };
+    // Idempotent: whichever side gives up first (client join error, server
+    // livekit_fallback, `connected` without a join) lands here once.
+    this.fallbackToWs = (reason) => {
+      if (this.mediaMode === "ws") return;
+      this.mediaMode = "ws";
+      this.lkMedia?.disconnect();
+      this.lkMedia = null;
+      void this.ensureWsPlayback();
+      void this.startCapture();
+      this.args.options.onTransportChange?.({ requested: "livekit", actual: "ws", reason });
+    };
+    this.resetLiveKitForReconnect = () => {
+      if (this.args.media?.kind !== "livekit") return;
+      this.lkMedia?.disconnect();
+      this.lkMedia = null;
+      this.lkJoinSeen = false;
+      this.mediaMode = "livekit-pending";
     };
     // ---------------------------------------------------------------
     // Internal
@@ -721,17 +855,27 @@ var BrowserVoiceClient = class {
     this.handleSocketEvent = (ev) => {
       switch (ev.type) {
         case "open":
-          void this.startCapture();
+          if (this.mediaMode === "ws") void this.startCapture();
           break;
         case "reconnected":
           this.proto.transcript = [];
           this.proto.agentBubbleId = null;
           this.args.options.onTranscript?.(this.proto.transcript);
-          void this.startCapture();
+          this.resetLiveKitForReconnect();
+          if (this.mediaMode === "ws") void this.startCapture();
           this.setState("listening");
           break;
         case "message":
           if (typeof ev.data === "string") {
+            const media = parseMediaFrame(ev.data);
+            if (media?.type === "livekit_join") {
+              if (this.mediaMode === "livekit-pending") void this.joinLiveKit(media.url, media.token);
+              break;
+            }
+            if (media?.type === "livekit_fallback") {
+              this.fallbackToWs(media.reason ?? "server_fallback");
+              break;
+            }
             handleServerMessage(ev.data, this.proto, {
               onState: this.setState,
               onTranscript: (entries) => this.args.options.onTranscript?.(entries),
@@ -748,7 +892,12 @@ var BrowserVoiceClient = class {
                 if (typeof seq === "number") this.marks.onAgentTurnEnd(seq);
               },
               onCallEnd: (reason) => this.teardown(reason),
-              onConnected: () => this.sendClientToolsRegister(),
+              onConnected: () => {
+                if (this.mediaMode === "livekit-pending" && !this.lkJoinSeen) {
+                  this.fallbackToWs("server_declined");
+                }
+                this.sendClientToolsRegister();
+              },
               onClientToolCall: (frame) => dispatchClientToolCall(
                 (f) => this.rws?.send(JSON.stringify(f)),
                 this.args.options.clientTools ?? {},
@@ -756,6 +905,8 @@ var BrowserVoiceClient = class {
               )
             });
           } else {
+            if (this.mediaMode === "livekit") break;
+            if (!this.playback) void this.ensureWsPlayback();
             this.marks.markFirstAudibleOutput();
             this.playback?.enqueue(ev.data);
           }
@@ -773,7 +924,8 @@ var BrowserVoiceClient = class {
     };
     this.startCapture = async () => {
       if (this.capture?.isCapturing()) return;
-      this.capture = createAudioCapture({
+      const createCapture = this.args.audio?.createCapture ?? createAudioCapture;
+      this.capture = createCapture({
         onChunk: (pcm) => {
           this.marks.markFirstOutboundAudio();
           this.rws?.send(pcm);
@@ -802,6 +954,8 @@ var BrowserVoiceClient = class {
       }
       this.capture?.stop();
       this.capture = null;
+      this.lkMedia?.disconnect();
+      this.lkMedia = null;
       this.playback?.close();
       this.playback = null;
       try {
@@ -854,28 +1008,17 @@ var BrowserVoiceClient = class {
   async start() {
     this.setState("connecting");
     this.startedAt = Date.now();
+    this.mediaMode = this.args.media?.kind === "livekit" ? "livekit-pending" : "ws";
     const url = buildWsUrl({
       apiBase: this.args.config.apiBase,
       agentId: this.args.options.agentId,
       token: this.args.token,
-      bargeIn: this.args.options.bargeIn
-    });
-    this.playback = createAudioPlayback({
-      onVolume: (v) => {
-        this.outputVolume = v;
-        this.args.options.onVolume?.({ input: this.inputVolume, output: v });
-      }
+      bargeIn: this.args.options.bargeIn,
+      media: this.mediaMode === "ws" ? void 0 : "livekit"
     });
-    try {
-      await this.playback.resume();
-    } catch {
-    }
+    if (this.mediaMode === "ws") await this.ensureWsPlayback();
     this.rws = createReconnectingWebSocket(
-      {
-        url,
-        wsFactory: this.args.wsFactory,
-        maxRetries: 3
-      },
+      { url, wsFactory: this.args.wsFactory, maxRetries: 3 },
       (ev) => this.handleSocketEvent(ev)
     );
   }
@@ -1060,6 +1203,20 @@ async function createWebRtcCall(opts) {
   };
 }
 
+// src/livekit/loadLiveKit.ts
+var lkPromise;
+var loadLiveKit = (requiredBy = "this feature") => {
+  lkPromise ?? (lkPromise = import("livekit-client").catch((e) => {
+    lkPromise = void 0;
+    const wrapped = new Error(
+      `${requiredBy} requires the optional peer dependency 'livekit-client' \u2014 npm install livekit-client`
+    );
+    wrapped.cause = e;
+    throw wrapped;
+  }));
+  return lkPromise;
+};
+
 // src/textSession.ts
 async function startTextSession(opts) {
   const f = opts.fetch ?? fetch;
@@ -1156,7 +1313,7 @@ var parseIncomingCall = (raw) => {
   if (typeof p.agentId !== "string" || p.agentId.length === 0) {
     throw new Error("parseIncomingCall: missing `agentId`");
   }
-  const transport = p.transport === "webrtc" ? "webrtc" : "ws";
+  const transport = p.transport === "webrtc" ? "webrtc" : p.transport === "livekit" ? "livekit" : "ws";
   const out = { token: p.token, agentId: p.agentId, transport };
   if (transport === "webrtc" && typeof p.webrtcGatewayBase === "string") {
     out.webrtcGatewayBase = p.webrtcGatewayBase;
@@ -1213,6 +1370,27 @@ var AssistantVoiceFactory = class {
           clientTools: options.clientTools
         });
       }
+      if (resolved.transport === "livekit") {
+        let lk = null;
+        let loadError = "";
+        try {
+          lk = await loadLiveKit("transport 'livekit'");
+        } catch (e) {
+          loadError = e instanceof Error ? e.message : String(e);
+        }
+        if (!lk) {
+          options.onTransportChange?.({ requested: "livekit", actual: "ws", reason: loadError });
+        }
+        const client2 = new BrowserVoiceClient({
+          config: this.config,
+          options: { ...options, context, metadata },
+          token: resolved.token,
+          wsFactory: browserWsFactory,
+          media: lk ? { kind: "livekit", lk } : void 0
+        });
+        await client2.start();
+        return client2;
+      }
       const client = new BrowserVoiceClient({
         config: this.config,
         // Carry merged context/metadata through to startCall so server can
@@ -1273,17 +1451,6 @@ var decodeAnalysis = (bytes) => {
 };
 
 // src/room.ts
-var lkPromise;
-var loadLiveKit = () => {
-  lkPromise ?? (lkPromise = import("livekit-client").catch((e) => {
-    lkPromise = void 0;
-    const msg = "joinRoom requires the optional peer dependency 'livekit-client' \u2014 npm install livekit-client";
-    const wrapped = new Error(msg);
-    wrapped.cause = e;
-    throw wrapped;
-  }));
-  return lkPromise;
-};
 var identityToPid = (identity) => identity.startsWith("guest:") ? identity.slice("guest:".length) : identity;
 var joinRoom = async (opts) => {
   const exchangeUrl = `${opts.apiBase.replace(/\/+$/, "")}/v1/rooms/${encodeURIComponent(
@@ -1308,7 +1475,7 @@ var joinRoom = async (opts) => {
       }
     });
   };
-  const lk = await loadLiveKit();
+  const lk = await loadLiveKit("joinRoom");
   const trackKind = (t) => t.kind === lk.Track.Kind.Video ? "video" : "audio";
   const trackSource = (s) => {
     switch (s) {