@craftedxp/voice-js 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +21 -19
  2. package/dist/assistant.d.mts +3 -3
  3. package/dist/assistant.d.ts +3 -3
  4. package/dist/assistant.js +210 -22
  5. package/dist/assistant.js.map +1 -1
  6. package/dist/assistant.mjs +2 -1
  7. package/dist/browser.d.mts +3 -3
  8. package/dist/browser.d.ts +3 -3
  9. package/dist/browser.js +201 -34
  10. package/dist/browser.js.map +1 -1
  11. package/dist/browser.mjs +3 -2
  12. package/dist/browser.mjs.map +1 -1
  13. package/dist/{chunk-3BYNC25N.mjs → chunk-C2U6V5NI.mjs} +191 -23
  14. package/dist/chunk-C2U6V5NI.mjs.map +1 -0
  15. package/dist/chunk-OS2GZAR7.mjs +18 -0
  16. package/dist/chunk-OS2GZAR7.mjs.map +1 -0
  17. package/dist/{chunk-LV7JGPYW.mjs → chunk-VSY43NGD.mjs} +6 -13
  18. package/dist/chunk-VSY43NGD.mjs.map +1 -0
  19. package/dist/{config-D2TbvIqT.d.mts → config-DfbPB1Tq.d.mts} +22 -3
  20. package/dist/{config-D2TbvIqT.d.ts → config-DfbPB1Tq.d.ts} +22 -3
  21. package/dist/embed.iife.js +42 -5
  22. package/dist/{incomingCall-CfRRzj2P.d.ts → incomingCall-2B3XB8pM.d.mts} +3 -1
  23. package/dist/{incomingCall-CfRRzj2P.d.mts → incomingCall-CYmeQkIw.d.ts} +3 -1
  24. package/dist/node.d.mts +26 -3
  25. package/dist/node.d.ts +26 -3
  26. package/dist/node.js +11 -2
  27. package/dist/node.js.map +1 -1
  28. package/dist/node.mjs +11 -2
  29. package/dist/node.mjs.map +1 -1
  30. package/dist/room.js +15 -12
  31. package/dist/room.js.map +1 -1
  32. package/dist/room.mjs +2 -1
  33. package/dist/transcribe.d.mts +2 -2
  34. package/dist/transcribe.d.ts +2 -2
  35. package/dist/transcribe.js +209 -21
  36. package/dist/transcribe.js.map +1 -1
  37. package/dist/transcribe.mjs +2 -1
  38. package/dist/transcribe.mjs.map +1 -1
  39. package/package.json +1 -1
  40. package/dist/chunk-3BYNC25N.mjs.map +0 -1
  41. package/dist/chunk-LV7JGPYW.mjs.map +0 -1
package/README.md CHANGED
@@ -12,14 +12,14 @@ Companion to [`@craftedxp/voice-rn`](https://www.npmjs.com/package/@craftedxp/vo
12
12
  npm install @craftedxp/voice-js
13
13
  # Node consumers also need:
14
14
  npm install ws
15
- # Multi-party rooms only (joinRoom) also need:
15
+ # Multi-party rooms (joinRoom) and 1:1 calls on the LiveKit media edge also need:
16
16
  npm install livekit-client
17
17
  ```
18
18
 
19
19
  `ws` and `livekit-client` are both declared as OPTIONAL peers:
20
20
 
21
21
  - `ws` — only needed in Node / Electron-main. Browsers use the native `WebSocket` and skip it.
22
- - `livekit-client` — only needed if you import from `@craftedxp/voice-js/room`. Since 0.7.0 it is no longer a direct dependency, so assistant / transcribe consumers never download it (the embed widget dropped from ~854 KB to ~100 KB). `joinRoom` lazy-`import()`s it and throws an actionable error if it's missing.
22
+ - `livekit-client` — needed for `@craftedxp/voice-js/room` (multi-party rooms) **or** for 1:1 calls when the server's media edge is LiveKit (`transport: 'livekit'`). `joinRoom` and the LiveKit transport both lazy-`import()` it; `joinRoom` throws an actionable error if it's missing, the 1:1 path falls back to WebSocket audio.
23
23
 
24
24
  ## Entry points (since 0.7.0)
25
25
 
@@ -168,22 +168,23 @@ Returns a `VoiceClientFactory` with one method:
168
168
 
169
169
  ### `factory.startCall(options)`
170
170
 
171
- | Field | Type | Notes |
172
- | ------------------ | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
173
- | `agentId` | `string` | Required. |
174
- | `userId` | `string?` | Round-tripped to fetchToken as `userId`; server uses it for contact memory. |
175
- | `context` | `Record<string, unknown>?` | Per-call structured context. Merged on top of `defaultContext`. Lowered into the agent's system prompt server-side. |
176
- | `metadata` | `Record<string, string>?` | Per-call key/value. Merged on top of `defaultMetadata`. Round-tripped on `call.ended` webhook. NOT lowered into the prompt. |
177
- | `bargeIn` | `boolean?` | Default `true`. Set `false` for alarm-style flows where the user shouldn't accidentally interrupt the script. |
178
- | `clientTools` | `ClientToolMap?` | Per-call client tools the agent's LLM can invoke. See [Client tools](#client-tools) section below. Validated synchronously at `startCall` — bad input throws. |
179
- | `token` | `string?` | **Test-only escape hatch** — pre-minted `ct_`, bypasses `fetchToken`. Don't use in production. |
180
- | `onStateChange` | `(state) => void` | Fires on every state machine transition. |
181
- | `onTranscript` | `(entries) => void` | Fires on every transcript update. |
182
- | `onInterrupt` | `() => void` | Server signaled barge-in. Browser bundle auto-flushes built-in playback before this fires. Node consumers should drain their custom playback queue here. |
183
- | `onAgentTurnStart` | `() => void` | New agent turn began. Use when you want a precise turn-start anchor without diffing `onStateChange`. |
184
- | `onVolume` | `({ input, output }) => void` | 0-1 RMS. ~10 Hz cadence. Browser bundle only. |
185
- | `onError` | `(err) => void` | Stable `code` from `CallErrorCode`; matches `voice-rn` codes where overlap. |
186
- | `onEnd` | `({ reason, errorCode?, durationMs }) => void` | Fires once when the call ends. |
171
+ | Field | Type | Notes |
172
+ | ------------------- | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
173
+ | `agentId` | `string` | Required. |
174
+ | `userId` | `string?` | Round-tripped to fetchToken as `userId`; server uses it for contact memory. |
175
+ | `context` | `Record<string, unknown>?` | Per-call structured context. Merged on top of `defaultContext`. Lowered into the agent's system prompt server-side. |
176
+ | `metadata` | `Record<string, string>?` | Per-call key/value. Merged on top of `defaultMetadata`. Round-tripped on `call.ended` webhook. NOT lowered into the prompt. |
177
+ | `bargeIn` | `boolean?` | Default `true`. Set `false` for alarm-style flows where the user shouldn't accidentally interrupt the script. |
178
+ | `clientTools` | `ClientToolMap?` | Per-call client tools the agent's LLM can invoke. See [Client tools](#client-tools) section below. Validated synchronously at `startCall` — bad input throws. |
179
+ | `token` | `string?` | **Test-only escape hatch** — pre-minted `ct_`, bypasses `fetchToken`. Don't use in production. |
180
+ | `onStateChange` | `(state) => void` | Fires on every state machine transition. |
181
+ | `onTranscript` | `(entries) => void` | Fires on every transcript update. |
182
+ | `onInterrupt` | `() => void` | Server signaled barge-in. Browser bundle auto-flushes built-in playback before this fires. Node consumers should drain their custom playback queue here. |
183
+ | `onAgentTurnStart` | `() => void` | New agent turn began. Use when you want a precise turn-start anchor without diffing `onStateChange`. |
184
+ | `onVolume` | `({ input, output }) => void` | 0-1 RMS. ~10 Hz cadence. Browser bundle only. |
185
+ | `onError` | `(err) => void` | Stable `code` from `CallErrorCode`; matches `voice-rn` codes where overlap. |
186
+ | `onEnd` | `({ reason, errorCode?, durationMs }) => void` | Fires once when the call ends. |
187
+ | `onTransportChange` | `({ requested, actual, reason? }) => void` | Fires when the transport actually used for the call differs from the one requested by the mint response — e.g. `requested: 'livekit'`, `actual: 'ws'` when `livekit-client` isn't installed, the room join fails, or the server sends `livekit_fallback`. `reason` is free text; match on `actual !== requested` rather than parsing it. |
187
188
 
188
189
  Resolves to a `Call` handle:
189
190
 
@@ -422,7 +423,8 @@ Renders a floating call button with a Shadow-DOM transcript panel. Pre-mint the
422
423
 
423
424
  ## Status
424
425
 
425
- - **0.10.0** (current) — `sendImage(data, mimeType)` on `NodeCall`: image input for `channel:'multimodal'` sessions (`user_image` frame). `data` can be a base64 string or buffer; `mimeType` must be one of `image/png`, `image/jpeg`, `image/webp`, `image/gif`. Returns `false` (and sends nothing) if the WS isn't open, data is empty, mimeType is unsupported, or base64-encoded size exceeds 5 MB; never throws. Also updates `sendText` doc to note it works on both `channel:'text'` and `channel:'multimodal'` sessions. Additive — drop-in for 0.9.0 consumers.
426
+ - **0.12.0** (current) — `livekit` transport (phase 35). When the mint response says `transport: 'livekit'`, the browser SDK opens the same call WebSocket with `?media=livekit`, joins the LiveKit room announced by the server's `livekit_join` frame via the optional peer `livekit-client`, publishes the mic and plays the agent's track; every control frame (transcripts, client tools, `interrupt`, …) stays on the WS. **Fallback is automatic:** if `livekit-client` isn't installed, the room join fails, or the server sends `livekit_fallback`, the call continues with WS audio on the same connection and the new `onTransportChange({ requested, actual, reason? })` callback fires (`reason` is free text, match on `actual !== requested`). Node entry, `NodeCall`, and `parseIncomingCall` treat `livekit` as `ws` unless noted. Additive — drop-in for 0.11.0 consumers.
427
+ - 0.10.0 — `sendImage(data, mimeType)` on `NodeCall`: image input for `channel:'multimodal'` sessions (`user_image` frame). `data` can be a base64 string or buffer; `mimeType` must be one of `image/png`, `image/jpeg`, `image/webp`, `image/gif`. Returns `false` (and sends nothing) if the WS isn't open, data is empty, mimeType is unsupported, or base64-encoded size exceeds 5 MB; never throws. Also updates `sendText` doc to note it works on both `channel:'text'` and `channel:'multimodal'` sessions. Additive — drop-in for 0.9.0 consumers.
426
428
  - 0.9.0 — `sendText(text)` on `NodeCall`: typed user turns for `channel:'text'` WS sessions (`user_text` frame). Server accepts `user_text` only on text-channel sessions and rejects it on voice-channel sessions (anti-injection). Text is trimmed; returns `false` (and sends nothing) if the WS isn't open or text is empty/whitespace, never throws. Additive — drop-in for 0.8.0 consumers.
427
429
  - 0.8.0 — `sendClientEvent(text)` on `NodeCall`: advisory context lines for live calls (`client_event` frame). Server buffers these and prepends them to the next turn's context; never triggers a response by itself. Text is trimmed and truncated to 400 chars; returns `false` if the WS isn't open yet or text is empty. Additive — drop-in for 0.7.0 consumers.
428
430
  - 0.7.0 — Per-type entry points + LiveKit goes optional. New subpaths `@craftedxp/voice-js/assistant`, `/room`, `/transcribe` (plus the existing `/node`); import the one matching your [agent `type`](https://www.npmjs.com/package/@craftedxp/sdk-node). `livekit-client` moved from `dependencies` to an **optional peerDependency** — assistant / transcribe consumers no longer download it, and the embed widget dropped from ~854 KB to ~100 KB. `joinRoom` lazy-`import()`s LiveKit and throws an actionable error if it isn't installed, so the deprecated barrel keeps working for non-room consumers. **Breaking for room consumers:** add `npm install livekit-client`. Non-room consumers are drop-in (prefer migrating barrel imports to `/assistant`).
@@ -1,6 +1,6 @@
1
- import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-D2TbvIqT.mjs';
2
- export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as ChatEvent, h as ClientTool, i as ClientToolMap, F as FetchToken, j as FetchTokenArgs, k as FetchTokenResult, P as ProtocolCallbacks, l as ProtocolState, S as ServerMessage, m as StartCallOptions, n as StartTextSessionOpts, T as TextSession, o as TranscriptEntry, p as VolumeEvent, q as buildWsUrl, r as createProtocolState, s as handleServerMessage, t as startTextSession } from './config-D2TbvIqT.mjs';
3
- export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CfRRzj2P.mjs';
1
+ import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-DfbPB1Tq.mjs';
2
+ export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as CallTransport, h as ChatEvent, i as ClientTool, j as ClientToolMap, F as FetchToken, k as FetchTokenArgs, l as FetchTokenResult, P as ProtocolCallbacks, m as ProtocolState, S as ServerMessage, n as StartCallOptions, o as StartTextSessionOpts, T as TextSession, p as TranscriptEntry, q as TransportChangeEvent, r as VolumeEvent, s as buildWsUrl, t as createProtocolState, u as handleServerMessage, v as startTextSession } from './config-DfbPB1Tq.mjs';
3
+ export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-2B3XB8pM.mjs';
4
4
 
5
5
  /**
6
6
  * One-time SDK setup. Returns a factory you call `startCall` on for
@@ -1,6 +1,6 @@
1
- import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-D2TbvIqT.js';
2
- export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as ChatEvent, h as ClientTool, i as ClientToolMap, F as FetchToken, j as FetchTokenArgs, k as FetchTokenResult, P as ProtocolCallbacks, l as ProtocolState, S as ServerMessage, m as StartCallOptions, n as StartTextSessionOpts, T as TextSession, o as TranscriptEntry, p as VolumeEvent, q as buildWsUrl, r as createProtocolState, s as handleServerMessage, t as startTextSession } from './config-D2TbvIqT.js';
3
- export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CfRRzj2P.js';
1
+ import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-DfbPB1Tq.js';
2
+ export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as CallTransport, h as ChatEvent, i as ClientTool, j as ClientToolMap, F as FetchToken, k as FetchTokenArgs, l as FetchTokenResult, P as ProtocolCallbacks, m as ProtocolState, S as ServerMessage, n as StartCallOptions, o as StartTextSessionOpts, T as TextSession, p as TranscriptEntry, q as TransportChangeEvent, r as VolumeEvent, s as buildWsUrl, t as createProtocolState, u as handleServerMessage, v as startTextSession } from './config-DfbPB1Tq.js';
3
+ export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CYmeQkIw.js';
4
4
 
5
5
  /**
6
6
  * One-time SDK setup. Returns a factory you call `startCall` on for
package/dist/assistant.js CHANGED
@@ -1,7 +1,9 @@
1
1
  "use strict";
2
+ var __create = Object.create;
2
3
  var __defProp = Object.defineProperty;
3
4
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
5
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
5
7
  var __hasOwnProp = Object.prototype.hasOwnProperty;
6
8
  var __export = (target, all) => {
7
9
  for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
15
17
  }
16
18
  return to;
17
19
  };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
18
28
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
29
 
20
30
  // src/assistant.ts
@@ -512,7 +522,8 @@ function buildWsUrl(args) {
512
522
  const base = new URL(args.apiBase);
513
523
  const proto = base.protocol === "https:" ? "wss:" : "ws:";
514
524
  const bargeQS = args.bargeIn === false ? "&barge=off" : "";
515
- return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}`;
525
+ const mediaQS = args.media === "livekit" ? "&media=livekit" : "";
526
+ return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}${mediaQS}`;
516
527
  }
517
528
 
518
529
  // src/clientTools.ts
@@ -666,7 +677,75 @@ var createClientMarksBuffer = (args) => {
666
677
  };
667
678
  };
668
679
 
680
+ // src/livekit/createLiveKitMedia.ts
681
+ var toError = (e) => e instanceof Error ? e : new Error(String(e));
682
+ function createLiveKitMedia(lk, opts = {}) {
683
+ let room = null;
684
+ let audioEl = null;
685
+ let muted = false;
686
+ const attach = (track) => {
687
+ if (track.kind !== lk.Track.Kind.Audio) return;
688
+ if (typeof document === "undefined") return;
689
+ if (!audioEl) {
690
+ audioEl = document.createElement("audio");
691
+ audioEl.autoplay = true;
692
+ audioEl.style.display = "none";
693
+ document.body.appendChild(audioEl);
694
+ }
695
+ track.attach(audioEl);
696
+ };
697
+ return {
698
+ connect: async (url, token) => {
699
+ const r = new lk.Room({ adaptiveStream: false, dynacast: false });
700
+ room = r;
701
+ r.on(lk.RoomEvent.TrackSubscribed, (track) => attach(track));
702
+ await r.connect(url, token);
703
+ if (room !== r) {
704
+ return;
705
+ }
706
+ await r.localParticipant.setMicrophoneEnabled(!muted);
707
+ if (room !== r) {
708
+ return;
709
+ }
710
+ try {
711
+ await r.startAudio();
712
+ } catch {
713
+ }
714
+ },
715
+ disconnect: () => {
716
+ const r = room;
717
+ room = null;
718
+ if (r) void r.disconnect().catch(() => void 0);
719
+ audioEl?.remove();
720
+ audioEl = null;
721
+ },
722
+ setMuted: (m) => {
723
+ muted = m;
724
+ const r = room;
725
+ if (!r) return;
726
+ void r.localParticipant.setMicrophoneEnabled(!m).catch((e) => opts.onError?.(toError(e)));
727
+ }
728
+ };
729
+ }
730
+
669
731
  // src/VoiceClient.ts
732
+ var parseMediaFrame = (raw) => {
733
+ if (!raw.includes('"livekit_')) return null;
734
+ try {
735
+ const m = JSON.parse(raw);
736
+ if (m.type === "livekit_join" && typeof m.url === "string" && typeof m.token === "string") {
737
+ return { type: "livekit_join", url: m.url, token: m.token };
738
+ }
739
+ if (m.type === "livekit_fallback") {
740
+ return {
741
+ type: "livekit_fallback",
742
+ reason: typeof m.reason === "string" ? m.reason : void 0
743
+ };
744
+ }
745
+ } catch {
746
+ }
747
+ return null;
748
+ };
670
749
  var BrowserVoiceClient = class {
671
750
  constructor(args) {
672
751
  this.rws = null;
@@ -678,6 +757,9 @@ var BrowserVoiceClient = class {
678
757
  this.startedAt = null;
679
758
  this.endedFired = false;
680
759
  this.lastError = null;
760
+ this.mediaMode = "ws";
761
+ this.lkMedia = null;
762
+ this.lkJoinSeen = false;
681
763
  this.end = () => {
682
764
  this.teardown("user_hangup");
683
765
  };
@@ -685,11 +767,73 @@ var BrowserVoiceClient = class {
685
767
  if (this.muted) return;
686
768
  this.muted = true;
687
769
  this.capture?.mute(true);
770
+ this.lkMedia?.setMuted(true);
688
771
  };
689
772
  this.unmute = () => {
690
773
  if (!this.muted) return;
691
774
  this.muted = false;
692
775
  this.capture?.mute(false);
776
+ this.lkMedia?.setMuted(false);
777
+ };
778
+ this.ensureWsPlayback = async () => {
779
+ if (this.playback) return;
780
+ const createPlayback = this.args.audio?.createPlayback ?? createAudioPlayback;
781
+ this.playback = createPlayback({
782
+ onVolume: (v) => {
783
+ this.outputVolume = v;
784
+ this.args.options.onVolume?.({ input: this.inputVolume, output: v });
785
+ }
786
+ });
787
+ try {
788
+ await this.playback.resume();
789
+ } catch {
790
+ }
791
+ };
792
+ // ---------------------------------------------------------------
793
+ // LiveKit media lifecycle (phase 35)
794
+ // ---------------------------------------------------------------
795
+ this.joinLiveKit = async (url, token) => {
796
+ this.lkJoinSeen = true;
797
+ const lk = this.args.media?.lk;
798
+ if (!lk) {
799
+ this.fallbackToWs("livekit-client unavailable");
800
+ return;
801
+ }
802
+ const media = createLiveKitMedia(lk, {
803
+ onError: (e) => this.emitError({ code: "mic_start_failed", message: e.message })
804
+ });
805
+ this.lkMedia = media;
806
+ if (this.muted) media.setMuted(true);
807
+ try {
808
+ await media.connect(url, token);
809
+ if (this.lkMedia !== media) return;
810
+ this.mediaMode = "livekit";
811
+ this.rws?.send(JSON.stringify({ type: "livekit_ready" }));
812
+ } catch (err) {
813
+ const reason = err instanceof Error ? err.message : String(err);
814
+ media.disconnect();
815
+ if (this.lkMedia === media) this.lkMedia = null;
816
+ this.rws?.send(JSON.stringify({ type: "livekit_failed", reason }));
817
+ this.fallbackToWs(reason);
818
+ }
819
+ };
820
+ // Idempotent: whichever side gives up first (client join error, server
821
+ // livekit_fallback, `connected` without a join) lands here once.
822
+ this.fallbackToWs = (reason) => {
823
+ if (this.mediaMode === "ws") return;
824
+ this.mediaMode = "ws";
825
+ this.lkMedia?.disconnect();
826
+ this.lkMedia = null;
827
+ void this.ensureWsPlayback();
828
+ void this.startCapture();
829
+ this.args.options.onTransportChange?.({ requested: "livekit", actual: "ws", reason });
830
+ };
831
+ this.resetLiveKitForReconnect = () => {
832
+ if (this.args.media?.kind !== "livekit") return;
833
+ this.lkMedia?.disconnect();
834
+ this.lkMedia = null;
835
+ this.lkJoinSeen = false;
836
+ this.mediaMode = "livekit-pending";
693
837
  };
694
838
  // ---------------------------------------------------------------
695
839
  // Internal
@@ -710,17 +854,27 @@ var BrowserVoiceClient = class {
710
854
  this.handleSocketEvent = (ev) => {
711
855
  switch (ev.type) {
712
856
  case "open":
713
- void this.startCapture();
857
+ if (this.mediaMode === "ws") void this.startCapture();
714
858
  break;
715
859
  case "reconnected":
716
860
  this.proto.transcript = [];
717
861
  this.proto.agentBubbleId = null;
718
862
  this.args.options.onTranscript?.(this.proto.transcript);
719
- void this.startCapture();
863
+ this.resetLiveKitForReconnect();
864
+ if (this.mediaMode === "ws") void this.startCapture();
720
865
  this.setState("listening");
721
866
  break;
722
867
  case "message":
723
868
  if (typeof ev.data === "string") {
869
+ const media = parseMediaFrame(ev.data);
870
+ if (media?.type === "livekit_join") {
871
+ if (this.mediaMode === "livekit-pending") void this.joinLiveKit(media.url, media.token);
872
+ break;
873
+ }
874
+ if (media?.type === "livekit_fallback") {
875
+ this.fallbackToWs(media.reason ?? "server_fallback");
876
+ break;
877
+ }
724
878
  handleServerMessage(ev.data, this.proto, {
725
879
  onState: this.setState,
726
880
  onTranscript: (entries) => this.args.options.onTranscript?.(entries),
@@ -737,7 +891,12 @@ var BrowserVoiceClient = class {
737
891
  if (typeof seq === "number") this.marks.onAgentTurnEnd(seq);
738
892
  },
739
893
  onCallEnd: (reason) => this.teardown(reason),
740
- onConnected: () => this.sendClientToolsRegister(),
894
+ onConnected: () => {
895
+ if (this.mediaMode === "livekit-pending" && !this.lkJoinSeen) {
896
+ this.fallbackToWs("server_declined");
897
+ }
898
+ this.sendClientToolsRegister();
899
+ },
741
900
  onClientToolCall: (frame) => dispatchClientToolCall(
742
901
  (f) => this.rws?.send(JSON.stringify(f)),
743
902
  this.args.options.clientTools ?? {},
@@ -745,6 +904,8 @@ var BrowserVoiceClient = class {
745
904
  )
746
905
  });
747
906
  } else {
907
+ if (this.mediaMode === "livekit") break;
908
+ if (!this.playback) void this.ensureWsPlayback();
748
909
  this.marks.markFirstAudibleOutput();
749
910
  this.playback?.enqueue(ev.data);
750
911
  }
@@ -762,7 +923,8 @@ var BrowserVoiceClient = class {
762
923
  };
763
924
  this.startCapture = async () => {
764
925
  if (this.capture?.isCapturing()) return;
765
- this.capture = createAudioCapture({
926
+ const createCapture = this.args.audio?.createCapture ?? createAudioCapture;
927
+ this.capture = createCapture({
766
928
  onChunk: (pcm) => {
767
929
  this.marks.markFirstOutboundAudio();
768
930
  this.rws?.send(pcm);
@@ -791,6 +953,8 @@ var BrowserVoiceClient = class {
791
953
  }
792
954
  this.capture?.stop();
793
955
  this.capture = null;
956
+ this.lkMedia?.disconnect();
957
+ this.lkMedia = null;
794
958
  this.playback?.close();
795
959
  this.playback = null;
796
960
  try {
@@ -843,28 +1007,17 @@ var BrowserVoiceClient = class {
843
1007
  async start() {
844
1008
  this.setState("connecting");
845
1009
  this.startedAt = Date.now();
1010
+ this.mediaMode = this.args.media?.kind === "livekit" ? "livekit-pending" : "ws";
846
1011
  const url = buildWsUrl({
847
1012
  apiBase: this.args.config.apiBase,
848
1013
  agentId: this.args.options.agentId,
849
1014
  token: this.args.token,
850
- bargeIn: this.args.options.bargeIn
851
- });
852
- this.playback = createAudioPlayback({
853
- onVolume: (v) => {
854
- this.outputVolume = v;
855
- this.args.options.onVolume?.({ input: this.inputVolume, output: v });
856
- }
1015
+ bargeIn: this.args.options.bargeIn,
1016
+ media: this.mediaMode === "ws" ? void 0 : "livekit"
857
1017
  });
858
- try {
859
- await this.playback.resume();
860
- } catch {
861
- }
1018
+ if (this.mediaMode === "ws") await this.ensureWsPlayback();
862
1019
  this.rws = createReconnectingWebSocket(
863
- {
864
- url,
865
- wsFactory: this.args.wsFactory,
866
- maxRetries: 3
867
- },
1020
+ { url, wsFactory: this.args.wsFactory, maxRetries: 3 },
868
1021
  (ev) => this.handleSocketEvent(ev)
869
1022
  );
870
1023
  }
@@ -1049,6 +1202,20 @@ async function createWebRtcCall(opts) {
1049
1202
  };
1050
1203
  }
1051
1204
 
1205
+ // src/livekit/loadLiveKit.ts
1206
+ var lkPromise;
1207
+ var loadLiveKit = (requiredBy = "this feature") => {
1208
+ lkPromise ?? (lkPromise = import("livekit-client").catch((e) => {
1209
+ lkPromise = void 0;
1210
+ const wrapped = new Error(
1211
+ `${requiredBy} requires the optional peer dependency 'livekit-client' \u2014 npm install livekit-client`
1212
+ );
1213
+ wrapped.cause = e;
1214
+ throw wrapped;
1215
+ }));
1216
+ return lkPromise;
1217
+ };
1218
+
1052
1219
  // src/textSession.ts
1053
1220
  async function startTextSession(opts) {
1054
1221
  const f = opts.fetch ?? fetch;
@@ -1145,7 +1312,7 @@ var parseIncomingCall = (raw) => {
1145
1312
  if (typeof p.agentId !== "string" || p.agentId.length === 0) {
1146
1313
  throw new Error("parseIncomingCall: missing `agentId`");
1147
1314
  }
1148
- const transport = p.transport === "webrtc" ? "webrtc" : "ws";
1315
+ const transport = p.transport === "webrtc" ? "webrtc" : p.transport === "livekit" ? "livekit" : "ws";
1149
1316
  const out = { token: p.token, agentId: p.agentId, transport };
1150
1317
  if (transport === "webrtc" && typeof p.webrtcGatewayBase === "string") {
1151
1318
  out.webrtcGatewayBase = p.webrtcGatewayBase;
@@ -1202,6 +1369,27 @@ var AssistantVoiceFactory = class {
1202
1369
  clientTools: options.clientTools
1203
1370
  });
1204
1371
  }
1372
+ if (resolved.transport === "livekit") {
1373
+ let lk = null;
1374
+ let loadError = "";
1375
+ try {
1376
+ lk = await loadLiveKit("transport 'livekit'");
1377
+ } catch (e) {
1378
+ loadError = e instanceof Error ? e.message : String(e);
1379
+ }
1380
+ if (!lk) {
1381
+ options.onTransportChange?.({ requested: "livekit", actual: "ws", reason: loadError });
1382
+ }
1383
+ const client2 = new BrowserVoiceClient({
1384
+ config: this.config,
1385
+ options: { ...options, context, metadata },
1386
+ token: resolved.token,
1387
+ wsFactory: browserWsFactory,
1388
+ media: lk ? { kind: "livekit", lk } : void 0
1389
+ });
1390
+ await client2.start();
1391
+ return client2;
1392
+ }
1205
1393
  const client = new BrowserVoiceClient({
1206
1394
  config: this.config,
1207
1395
  // Carry merged context/metadata through to startCall so server can