@craftedxp/voice-js 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -19
- package/dist/assistant.d.mts +3 -3
- package/dist/assistant.d.ts +3 -3
- package/dist/assistant.js +210 -22
- package/dist/assistant.js.map +1 -1
- package/dist/assistant.mjs +2 -1
- package/dist/browser.d.mts +3 -3
- package/dist/browser.d.ts +3 -3
- package/dist/browser.js +201 -34
- package/dist/browser.js.map +1 -1
- package/dist/browser.mjs +3 -2
- package/dist/browser.mjs.map +1 -1
- package/dist/{chunk-3BYNC25N.mjs → chunk-C2U6V5NI.mjs} +191 -23
- package/dist/chunk-C2U6V5NI.mjs.map +1 -0
- package/dist/chunk-OS2GZAR7.mjs +18 -0
- package/dist/chunk-OS2GZAR7.mjs.map +1 -0
- package/dist/{chunk-LV7JGPYW.mjs → chunk-VSY43NGD.mjs} +6 -13
- package/dist/chunk-VSY43NGD.mjs.map +1 -0
- package/dist/{config-D2TbvIqT.d.mts → config-DfbPB1Tq.d.mts} +22 -3
- package/dist/{config-D2TbvIqT.d.ts → config-DfbPB1Tq.d.ts} +22 -3
- package/dist/embed.iife.js +42 -5
- package/dist/{incomingCall-CfRRzj2P.d.ts → incomingCall-2B3XB8pM.d.mts} +3 -1
- package/dist/{incomingCall-CfRRzj2P.d.mts → incomingCall-CYmeQkIw.d.ts} +3 -1
- package/dist/node.d.mts +26 -3
- package/dist/node.d.ts +26 -3
- package/dist/node.js +11 -2
- package/dist/node.js.map +1 -1
- package/dist/node.mjs +11 -2
- package/dist/node.mjs.map +1 -1
- package/dist/room.js +15 -12
- package/dist/room.js.map +1 -1
- package/dist/room.mjs +2 -1
- package/dist/transcribe.d.mts +2 -2
- package/dist/transcribe.d.ts +2 -2
- package/dist/transcribe.js +209 -21
- package/dist/transcribe.js.map +1 -1
- package/dist/transcribe.mjs +2 -1
- package/dist/transcribe.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-3BYNC25N.mjs.map +0 -1
- package/dist/chunk-LV7JGPYW.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -12,14 +12,14 @@ Companion to [`@craftedxp/voice-rn`](https://www.npmjs.com/package/@craftedxp/vo
|
|
|
12
12
|
npm install @craftedxp/voice-js
|
|
13
13
|
# Node consumers also need:
|
|
14
14
|
npm install ws
|
|
15
|
-
# Multi-party rooms
|
|
15
|
+
# Multi-party rooms (joinRoom) and 1:1 calls on the LiveKit media edge also need:
|
|
16
16
|
npm install livekit-client
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
`ws` and `livekit-client` are both declared as OPTIONAL peers:
|
|
20
20
|
|
|
21
21
|
- `ws` — only needed in Node / Electron-main. Browsers use the native `WebSocket` and skip it.
|
|
22
|
-
- `livekit-client` —
|
|
22
|
+
- `livekit-client` — needed for `@craftedxp/voice-js/room` (multi-party rooms) **or** for 1:1 calls when the server's media edge is LiveKit (`transport: 'livekit'`). `joinRoom` and the LiveKit transport both lazy-`import()` it; `joinRoom` throws an actionable error if it's missing, the 1:1 path falls back to WebSocket audio.
|
|
23
23
|
|
|
24
24
|
## Entry points (since 0.7.0)
|
|
25
25
|
|
|
@@ -168,22 +168,23 @@ Returns a `VoiceClientFactory` with one method:
|
|
|
168
168
|
|
|
169
169
|
### `factory.startCall(options)`
|
|
170
170
|
|
|
171
|
-
| Field
|
|
172
|
-
|
|
|
173
|
-
| `agentId`
|
|
174
|
-
| `userId`
|
|
175
|
-
| `context`
|
|
176
|
-
| `metadata`
|
|
177
|
-
| `bargeIn`
|
|
178
|
-
| `clientTools`
|
|
179
|
-
| `token`
|
|
180
|
-
| `onStateChange`
|
|
181
|
-
| `onTranscript`
|
|
182
|
-
| `onInterrupt`
|
|
183
|
-
| `onAgentTurnStart`
|
|
184
|
-
| `onVolume`
|
|
185
|
-
| `onError`
|
|
186
|
-
| `onEnd`
|
|
171
|
+
| Field | Type | Notes |
|
|
172
|
+
| ------------------- | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
173
|
+
| `agentId` | `string` | Required. |
|
|
174
|
+
| `userId` | `string?` | Round-tripped to fetchToken as `userId`; server uses it for contact memory. |
|
|
175
|
+
| `context` | `Record<string, unknown>?` | Per-call structured context. Merged on top of `defaultContext`. Lowered into the agent's system prompt server-side. |
|
|
176
|
+
| `metadata` | `Record<string, string>?` | Per-call key/value. Merged on top of `defaultMetadata`. Round-tripped on `call.ended` webhook. NOT lowered into the prompt. |
|
|
177
|
+
| `bargeIn` | `boolean?` | Default `true`. Set `false` for alarm-style flows where the user shouldn't accidentally interrupt the script. |
|
|
178
|
+
| `clientTools` | `ClientToolMap?` | Per-call client tools the agent's LLM can invoke. See [Client tools](#client-tools) section below. Validated synchronously at `startCall` — bad input throws. |
|
|
179
|
+
| `token` | `string?` | **Test-only escape hatch** — pre-minted `ct_`, bypasses `fetchToken`. Don't use in production. |
|
|
180
|
+
| `onStateChange` | `(state) => void` | Fires on every state machine transition. |
|
|
181
|
+
| `onTranscript` | `(entries) => void` | Fires on every transcript update. |
|
|
182
|
+
| `onInterrupt` | `() => void` | Server signaled barge-in. Browser bundle auto-flushes built-in playback before this fires. Node consumers should drain their custom playback queue here. |
|
|
183
|
+
| `onAgentTurnStart` | `() => void` | New agent turn began. Use when you want a precise turn-start anchor without diffing `onStateChange`. |
|
|
184
|
+
| `onVolume` | `({ input, output }) => void` | 0-1 RMS. ~10 Hz cadence. Browser bundle only. |
|
|
185
|
+
| `onError` | `(err) => void` | Stable `code` from `CallErrorCode`; matches `voice-rn` codes where overlap. |
|
|
186
|
+
| `onEnd` | `({ reason, errorCode?, durationMs }) => void` | Fires once when the call ends. |
|
|
187
|
+
| `onTransportChange` | `({ requested, actual, reason? }) => void` | Fires when the transport actually used for the call differs from the one requested by the mint response — e.g. `requested: 'livekit'`, `actual: 'ws'` when `livekit-client` isn't installed, the room join fails, or the server sends `livekit_fallback`. `reason` is free text; match on `actual !== requested` rather than parsing it. |
|
|
187
188
|
|
|
188
189
|
Resolves to a `Call` handle:
|
|
189
190
|
|
|
@@ -422,7 +423,8 @@ Renders a floating call button with a Shadow-DOM transcript panel. Pre-mint the
|
|
|
422
423
|
|
|
423
424
|
## Status
|
|
424
425
|
|
|
425
|
-
- **0.
|
|
426
|
+
- **0.12.0** (current) — `livekit` transport (phase 35). When the mint response says `transport: 'livekit'`, the browser SDK opens the same call WebSocket with `?media=livekit`, joins the LiveKit room announced by the server's `livekit_join` frame via the optional peer `livekit-client`, publishes the mic and plays the agent's track; every control frame (transcripts, client tools, `interrupt`, …) stays on the WS. **Fallback is automatic:** if `livekit-client` isn't installed, the room join fails, or the server sends `livekit_fallback`, the call continues with WS audio on the same connection and the new `onTransportChange({ requested, actual, reason? })` callback fires (`reason` is free text, match on `actual !== requested`). Node entry, `NodeCall`, and `parseIncomingCall` treat `livekit` as `ws` unless noted. Additive — drop-in for 0.11.0 consumers.
|
|
427
|
+
- 0.10.0 — `sendImage(data, mimeType)` on `NodeCall`: image input for `channel:'multimodal'` sessions (`user_image` frame). `data` can be a base64 string or buffer; `mimeType` must be one of `image/png`, `image/jpeg`, `image/webp`, `image/gif`. Returns `false` (and sends nothing) if the WS isn't open, data is empty, mimeType is unsupported, or base64-encoded size exceeds 5 MB; never throws. Also updates `sendText` doc to note it works on both `channel:'text'` and `channel:'multimodal'` sessions. Additive — drop-in for 0.9.0 consumers.
|
|
426
428
|
- 0.9.0 — `sendText(text)` on `NodeCall`: typed user turns for `channel:'text'` WS sessions (`user_text` frame). Server accepts `user_text` only on text-channel sessions and rejects it on voice-channel sessions (anti-injection). Text is trimmed; returns `false` (and sends nothing) if the WS isn't open or text is empty/whitespace, never throws. Additive — drop-in for 0.8.0 consumers.
|
|
427
429
|
- 0.8.0 — `sendClientEvent(text)` on `NodeCall`: advisory context lines for live calls (`client_event` frame). Server buffers these and prepends them to the next turn's context; never triggers a response by itself. Text is trimmed and truncated to 400 chars; returns `false` if the WS isn't open yet or text is empty. Additive — drop-in for 0.7.0 consumers.
|
|
428
430
|
- 0.7.0 — Per-type entry points + LiveKit goes optional. New subpaths `@craftedxp/voice-js/assistant`, `/room`, `/transcribe` (plus the existing `/node`); import the one matching your [agent `type`](https://www.npmjs.com/package/@craftedxp/sdk-node). `livekit-client` moved from `dependencies` to an **optional peerDependency** — assistant / transcribe consumers no longer download it, and the embed widget dropped from ~854 KB to ~100 KB. `joinRoom` lazy-`import()`s LiveKit and throws an actionable error if it isn't installed, so the deprecated barrel keeps working for non-room consumers. **Breaking for room consumers:** add `npm install livekit-client`. Non-room consumers are drop-in (prefer migrating barrel imports to `/assistant`).
|
package/dist/assistant.d.mts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-
|
|
2
|
-
export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as
|
|
3
|
-
export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-
|
|
1
|
+
import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-DfbPB1Tq.mjs';
|
|
2
|
+
export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as CallTransport, h as ChatEvent, i as ClientTool, j as ClientToolMap, F as FetchToken, k as FetchTokenArgs, l as FetchTokenResult, P as ProtocolCallbacks, m as ProtocolState, S as ServerMessage, n as StartCallOptions, o as StartTextSessionOpts, T as TextSession, p as TranscriptEntry, q as TransportChangeEvent, r as VolumeEvent, s as buildWsUrl, t as createProtocolState, u as handleServerMessage, v as startTextSession } from './config-DfbPB1Tq.mjs';
|
|
3
|
+
export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-2B3XB8pM.mjs';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* One-time SDK setup. Returns a factory you call `startCall` on for
|
package/dist/assistant.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-
|
|
2
|
-
export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as
|
|
3
|
-
export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-
|
|
1
|
+
import { V as VoiceClientConfig, a as VoiceClientFactory } from './config-DfbPB1Tq.js';
|
|
2
|
+
export { C as Call, b as CallEndEvent, c as CallEndReason, d as CallError, e as CallErrorCode, f as CallState, g as CallTransport, h as ChatEvent, i as ClientTool, j as ClientToolMap, F as FetchToken, k as FetchTokenArgs, l as FetchTokenResult, P as ProtocolCallbacks, m as ProtocolState, S as ServerMessage, n as StartCallOptions, o as StartTextSessionOpts, T as TextSession, p as TranscriptEntry, q as TransportChangeEvent, r as VolumeEvent, s as buildWsUrl, t as createProtocolState, u as handleServerMessage, v as startTextSession } from './config-DfbPB1Tq.js';
|
|
3
|
+
export { C as CaptureController, a as CaptureOptions, I as IncomingCallPayload, O as OnAgentSpeakingChange, b as OnChunk, c as OnError, d as OnVolume, P as PlaybackController, e as PlaybackOptions, R as RWSEvent, f as RWSOptions, g as ReconnectingWebSocket, W as WebSocketFactory, h as WebSocketLike, i as createAudioCapture, j as createAudioPlayback, k as createReconnectingWebSocket, p as parseIncomingCall } from './incomingCall-CYmeQkIw.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* One-time SDK setup. Returns a factory you call `startCall` on for
|
package/dist/assistant.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/assistant.ts
|
|
@@ -512,7 +522,8 @@ function buildWsUrl(args) {
|
|
|
512
522
|
const base = new URL(args.apiBase);
|
|
513
523
|
const proto = base.protocol === "https:" ? "wss:" : "ws:";
|
|
514
524
|
const bargeQS = args.bargeIn === false ? "&barge=off" : "";
|
|
515
|
-
|
|
525
|
+
const mediaQS = args.media === "livekit" ? "&media=livekit" : "";
|
|
526
|
+
return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}${mediaQS}`;
|
|
516
527
|
}
|
|
517
528
|
|
|
518
529
|
// src/clientTools.ts
|
|
@@ -666,7 +677,75 @@ var createClientMarksBuffer = (args) => {
|
|
|
666
677
|
};
|
|
667
678
|
};
|
|
668
679
|
|
|
680
|
+
// src/livekit/createLiveKitMedia.ts
|
|
681
|
+
var toError = (e) => e instanceof Error ? e : new Error(String(e));
|
|
682
|
+
function createLiveKitMedia(lk, opts = {}) {
|
|
683
|
+
let room = null;
|
|
684
|
+
let audioEl = null;
|
|
685
|
+
let muted = false;
|
|
686
|
+
const attach = (track) => {
|
|
687
|
+
if (track.kind !== lk.Track.Kind.Audio) return;
|
|
688
|
+
if (typeof document === "undefined") return;
|
|
689
|
+
if (!audioEl) {
|
|
690
|
+
audioEl = document.createElement("audio");
|
|
691
|
+
audioEl.autoplay = true;
|
|
692
|
+
audioEl.style.display = "none";
|
|
693
|
+
document.body.appendChild(audioEl);
|
|
694
|
+
}
|
|
695
|
+
track.attach(audioEl);
|
|
696
|
+
};
|
|
697
|
+
return {
|
|
698
|
+
connect: async (url, token) => {
|
|
699
|
+
const r = new lk.Room({ adaptiveStream: false, dynacast: false });
|
|
700
|
+
room = r;
|
|
701
|
+
r.on(lk.RoomEvent.TrackSubscribed, (track) => attach(track));
|
|
702
|
+
await r.connect(url, token);
|
|
703
|
+
if (room !== r) {
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
await r.localParticipant.setMicrophoneEnabled(!muted);
|
|
707
|
+
if (room !== r) {
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
try {
|
|
711
|
+
await r.startAudio();
|
|
712
|
+
} catch {
|
|
713
|
+
}
|
|
714
|
+
},
|
|
715
|
+
disconnect: () => {
|
|
716
|
+
const r = room;
|
|
717
|
+
room = null;
|
|
718
|
+
if (r) void r.disconnect().catch(() => void 0);
|
|
719
|
+
audioEl?.remove();
|
|
720
|
+
audioEl = null;
|
|
721
|
+
},
|
|
722
|
+
setMuted: (m) => {
|
|
723
|
+
muted = m;
|
|
724
|
+
const r = room;
|
|
725
|
+
if (!r) return;
|
|
726
|
+
void r.localParticipant.setMicrophoneEnabled(!m).catch((e) => opts.onError?.(toError(e)));
|
|
727
|
+
}
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
|
|
669
731
|
// src/VoiceClient.ts
|
|
732
|
+
var parseMediaFrame = (raw) => {
|
|
733
|
+
if (!raw.includes('"livekit_')) return null;
|
|
734
|
+
try {
|
|
735
|
+
const m = JSON.parse(raw);
|
|
736
|
+
if (m.type === "livekit_join" && typeof m.url === "string" && typeof m.token === "string") {
|
|
737
|
+
return { type: "livekit_join", url: m.url, token: m.token };
|
|
738
|
+
}
|
|
739
|
+
if (m.type === "livekit_fallback") {
|
|
740
|
+
return {
|
|
741
|
+
type: "livekit_fallback",
|
|
742
|
+
reason: typeof m.reason === "string" ? m.reason : void 0
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
} catch {
|
|
746
|
+
}
|
|
747
|
+
return null;
|
|
748
|
+
};
|
|
670
749
|
var BrowserVoiceClient = class {
|
|
671
750
|
constructor(args) {
|
|
672
751
|
this.rws = null;
|
|
@@ -678,6 +757,9 @@ var BrowserVoiceClient = class {
|
|
|
678
757
|
this.startedAt = null;
|
|
679
758
|
this.endedFired = false;
|
|
680
759
|
this.lastError = null;
|
|
760
|
+
this.mediaMode = "ws";
|
|
761
|
+
this.lkMedia = null;
|
|
762
|
+
this.lkJoinSeen = false;
|
|
681
763
|
this.end = () => {
|
|
682
764
|
this.teardown("user_hangup");
|
|
683
765
|
};
|
|
@@ -685,11 +767,73 @@ var BrowserVoiceClient = class {
|
|
|
685
767
|
if (this.muted) return;
|
|
686
768
|
this.muted = true;
|
|
687
769
|
this.capture?.mute(true);
|
|
770
|
+
this.lkMedia?.setMuted(true);
|
|
688
771
|
};
|
|
689
772
|
this.unmute = () => {
|
|
690
773
|
if (!this.muted) return;
|
|
691
774
|
this.muted = false;
|
|
692
775
|
this.capture?.mute(false);
|
|
776
|
+
this.lkMedia?.setMuted(false);
|
|
777
|
+
};
|
|
778
|
+
this.ensureWsPlayback = async () => {
|
|
779
|
+
if (this.playback) return;
|
|
780
|
+
const createPlayback = this.args.audio?.createPlayback ?? createAudioPlayback;
|
|
781
|
+
this.playback = createPlayback({
|
|
782
|
+
onVolume: (v) => {
|
|
783
|
+
this.outputVolume = v;
|
|
784
|
+
this.args.options.onVolume?.({ input: this.inputVolume, output: v });
|
|
785
|
+
}
|
|
786
|
+
});
|
|
787
|
+
try {
|
|
788
|
+
await this.playback.resume();
|
|
789
|
+
} catch {
|
|
790
|
+
}
|
|
791
|
+
};
|
|
792
|
+
// ---------------------------------------------------------------
|
|
793
|
+
// LiveKit media lifecycle (phase 35)
|
|
794
|
+
// ---------------------------------------------------------------
|
|
795
|
+
this.joinLiveKit = async (url, token) => {
|
|
796
|
+
this.lkJoinSeen = true;
|
|
797
|
+
const lk = this.args.media?.lk;
|
|
798
|
+
if (!lk) {
|
|
799
|
+
this.fallbackToWs("livekit-client unavailable");
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
const media = createLiveKitMedia(lk, {
|
|
803
|
+
onError: (e) => this.emitError({ code: "mic_start_failed", message: e.message })
|
|
804
|
+
});
|
|
805
|
+
this.lkMedia = media;
|
|
806
|
+
if (this.muted) media.setMuted(true);
|
|
807
|
+
try {
|
|
808
|
+
await media.connect(url, token);
|
|
809
|
+
if (this.lkMedia !== media) return;
|
|
810
|
+
this.mediaMode = "livekit";
|
|
811
|
+
this.rws?.send(JSON.stringify({ type: "livekit_ready" }));
|
|
812
|
+
} catch (err) {
|
|
813
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
814
|
+
media.disconnect();
|
|
815
|
+
if (this.lkMedia === media) this.lkMedia = null;
|
|
816
|
+
this.rws?.send(JSON.stringify({ type: "livekit_failed", reason }));
|
|
817
|
+
this.fallbackToWs(reason);
|
|
818
|
+
}
|
|
819
|
+
};
|
|
820
|
+
// Idempotent: whichever side gives up first (client join error, server
|
|
821
|
+
// livekit_fallback, `connected` without a join) lands here once.
|
|
822
|
+
this.fallbackToWs = (reason) => {
|
|
823
|
+
if (this.mediaMode === "ws") return;
|
|
824
|
+
this.mediaMode = "ws";
|
|
825
|
+
this.lkMedia?.disconnect();
|
|
826
|
+
this.lkMedia = null;
|
|
827
|
+
void this.ensureWsPlayback();
|
|
828
|
+
void this.startCapture();
|
|
829
|
+
this.args.options.onTransportChange?.({ requested: "livekit", actual: "ws", reason });
|
|
830
|
+
};
|
|
831
|
+
this.resetLiveKitForReconnect = () => {
|
|
832
|
+
if (this.args.media?.kind !== "livekit") return;
|
|
833
|
+
this.lkMedia?.disconnect();
|
|
834
|
+
this.lkMedia = null;
|
|
835
|
+
this.lkJoinSeen = false;
|
|
836
|
+
this.mediaMode = "livekit-pending";
|
|
693
837
|
};
|
|
694
838
|
// ---------------------------------------------------------------
|
|
695
839
|
// Internal
|
|
@@ -710,17 +854,27 @@ var BrowserVoiceClient = class {
|
|
|
710
854
|
this.handleSocketEvent = (ev) => {
|
|
711
855
|
switch (ev.type) {
|
|
712
856
|
case "open":
|
|
713
|
-
void this.startCapture();
|
|
857
|
+
if (this.mediaMode === "ws") void this.startCapture();
|
|
714
858
|
break;
|
|
715
859
|
case "reconnected":
|
|
716
860
|
this.proto.transcript = [];
|
|
717
861
|
this.proto.agentBubbleId = null;
|
|
718
862
|
this.args.options.onTranscript?.(this.proto.transcript);
|
|
719
|
-
|
|
863
|
+
this.resetLiveKitForReconnect();
|
|
864
|
+
if (this.mediaMode === "ws") void this.startCapture();
|
|
720
865
|
this.setState("listening");
|
|
721
866
|
break;
|
|
722
867
|
case "message":
|
|
723
868
|
if (typeof ev.data === "string") {
|
|
869
|
+
const media = parseMediaFrame(ev.data);
|
|
870
|
+
if (media?.type === "livekit_join") {
|
|
871
|
+
if (this.mediaMode === "livekit-pending") void this.joinLiveKit(media.url, media.token);
|
|
872
|
+
break;
|
|
873
|
+
}
|
|
874
|
+
if (media?.type === "livekit_fallback") {
|
|
875
|
+
this.fallbackToWs(media.reason ?? "server_fallback");
|
|
876
|
+
break;
|
|
877
|
+
}
|
|
724
878
|
handleServerMessage(ev.data, this.proto, {
|
|
725
879
|
onState: this.setState,
|
|
726
880
|
onTranscript: (entries) => this.args.options.onTranscript?.(entries),
|
|
@@ -737,7 +891,12 @@ var BrowserVoiceClient = class {
|
|
|
737
891
|
if (typeof seq === "number") this.marks.onAgentTurnEnd(seq);
|
|
738
892
|
},
|
|
739
893
|
onCallEnd: (reason) => this.teardown(reason),
|
|
740
|
-
onConnected: () =>
|
|
894
|
+
onConnected: () => {
|
|
895
|
+
if (this.mediaMode === "livekit-pending" && !this.lkJoinSeen) {
|
|
896
|
+
this.fallbackToWs("server_declined");
|
|
897
|
+
}
|
|
898
|
+
this.sendClientToolsRegister();
|
|
899
|
+
},
|
|
741
900
|
onClientToolCall: (frame) => dispatchClientToolCall(
|
|
742
901
|
(f) => this.rws?.send(JSON.stringify(f)),
|
|
743
902
|
this.args.options.clientTools ?? {},
|
|
@@ -745,6 +904,8 @@ var BrowserVoiceClient = class {
|
|
|
745
904
|
)
|
|
746
905
|
});
|
|
747
906
|
} else {
|
|
907
|
+
if (this.mediaMode === "livekit") break;
|
|
908
|
+
if (!this.playback) void this.ensureWsPlayback();
|
|
748
909
|
this.marks.markFirstAudibleOutput();
|
|
749
910
|
this.playback?.enqueue(ev.data);
|
|
750
911
|
}
|
|
@@ -762,7 +923,8 @@ var BrowserVoiceClient = class {
|
|
|
762
923
|
};
|
|
763
924
|
this.startCapture = async () => {
|
|
764
925
|
if (this.capture?.isCapturing()) return;
|
|
765
|
-
this.
|
|
926
|
+
const createCapture = this.args.audio?.createCapture ?? createAudioCapture;
|
|
927
|
+
this.capture = createCapture({
|
|
766
928
|
onChunk: (pcm) => {
|
|
767
929
|
this.marks.markFirstOutboundAudio();
|
|
768
930
|
this.rws?.send(pcm);
|
|
@@ -791,6 +953,8 @@ var BrowserVoiceClient = class {
|
|
|
791
953
|
}
|
|
792
954
|
this.capture?.stop();
|
|
793
955
|
this.capture = null;
|
|
956
|
+
this.lkMedia?.disconnect();
|
|
957
|
+
this.lkMedia = null;
|
|
794
958
|
this.playback?.close();
|
|
795
959
|
this.playback = null;
|
|
796
960
|
try {
|
|
@@ -843,28 +1007,17 @@ var BrowserVoiceClient = class {
|
|
|
843
1007
|
async start() {
|
|
844
1008
|
this.setState("connecting");
|
|
845
1009
|
this.startedAt = Date.now();
|
|
1010
|
+
this.mediaMode = this.args.media?.kind === "livekit" ? "livekit-pending" : "ws";
|
|
846
1011
|
const url = buildWsUrl({
|
|
847
1012
|
apiBase: this.args.config.apiBase,
|
|
848
1013
|
agentId: this.args.options.agentId,
|
|
849
1014
|
token: this.args.token,
|
|
850
|
-
bargeIn: this.args.options.bargeIn
|
|
851
|
-
|
|
852
|
-
this.playback = createAudioPlayback({
|
|
853
|
-
onVolume: (v) => {
|
|
854
|
-
this.outputVolume = v;
|
|
855
|
-
this.args.options.onVolume?.({ input: this.inputVolume, output: v });
|
|
856
|
-
}
|
|
1015
|
+
bargeIn: this.args.options.bargeIn,
|
|
1016
|
+
media: this.mediaMode === "ws" ? void 0 : "livekit"
|
|
857
1017
|
});
|
|
858
|
-
|
|
859
|
-
await this.playback.resume();
|
|
860
|
-
} catch {
|
|
861
|
-
}
|
|
1018
|
+
if (this.mediaMode === "ws") await this.ensureWsPlayback();
|
|
862
1019
|
this.rws = createReconnectingWebSocket(
|
|
863
|
-
{
|
|
864
|
-
url,
|
|
865
|
-
wsFactory: this.args.wsFactory,
|
|
866
|
-
maxRetries: 3
|
|
867
|
-
},
|
|
1020
|
+
{ url, wsFactory: this.args.wsFactory, maxRetries: 3 },
|
|
868
1021
|
(ev) => this.handleSocketEvent(ev)
|
|
869
1022
|
);
|
|
870
1023
|
}
|
|
@@ -1049,6 +1202,20 @@ async function createWebRtcCall(opts) {
|
|
|
1049
1202
|
};
|
|
1050
1203
|
}
|
|
1051
1204
|
|
|
1205
|
+
// src/livekit/loadLiveKit.ts
|
|
1206
|
+
var lkPromise;
|
|
1207
|
+
var loadLiveKit = (requiredBy = "this feature") => {
|
|
1208
|
+
lkPromise ?? (lkPromise = import("livekit-client").catch((e) => {
|
|
1209
|
+
lkPromise = void 0;
|
|
1210
|
+
const wrapped = new Error(
|
|
1211
|
+
`${requiredBy} requires the optional peer dependency 'livekit-client' \u2014 npm install livekit-client`
|
|
1212
|
+
);
|
|
1213
|
+
wrapped.cause = e;
|
|
1214
|
+
throw wrapped;
|
|
1215
|
+
}));
|
|
1216
|
+
return lkPromise;
|
|
1217
|
+
};
|
|
1218
|
+
|
|
1052
1219
|
// src/textSession.ts
|
|
1053
1220
|
async function startTextSession(opts) {
|
|
1054
1221
|
const f = opts.fetch ?? fetch;
|
|
@@ -1145,7 +1312,7 @@ var parseIncomingCall = (raw) => {
|
|
|
1145
1312
|
if (typeof p.agentId !== "string" || p.agentId.length === 0) {
|
|
1146
1313
|
throw new Error("parseIncomingCall: missing `agentId`");
|
|
1147
1314
|
}
|
|
1148
|
-
const transport = p.transport === "webrtc" ? "webrtc" : "ws";
|
|
1315
|
+
const transport = p.transport === "webrtc" ? "webrtc" : p.transport === "livekit" ? "livekit" : "ws";
|
|
1149
1316
|
const out = { token: p.token, agentId: p.agentId, transport };
|
|
1150
1317
|
if (transport === "webrtc" && typeof p.webrtcGatewayBase === "string") {
|
|
1151
1318
|
out.webrtcGatewayBase = p.webrtcGatewayBase;
|
|
@@ -1202,6 +1369,27 @@ var AssistantVoiceFactory = class {
|
|
|
1202
1369
|
clientTools: options.clientTools
|
|
1203
1370
|
});
|
|
1204
1371
|
}
|
|
1372
|
+
if (resolved.transport === "livekit") {
|
|
1373
|
+
let lk = null;
|
|
1374
|
+
let loadError = "";
|
|
1375
|
+
try {
|
|
1376
|
+
lk = await loadLiveKit("transport 'livekit'");
|
|
1377
|
+
} catch (e) {
|
|
1378
|
+
loadError = e instanceof Error ? e.message : String(e);
|
|
1379
|
+
}
|
|
1380
|
+
if (!lk) {
|
|
1381
|
+
options.onTransportChange?.({ requested: "livekit", actual: "ws", reason: loadError });
|
|
1382
|
+
}
|
|
1383
|
+
const client2 = new BrowserVoiceClient({
|
|
1384
|
+
config: this.config,
|
|
1385
|
+
options: { ...options, context, metadata },
|
|
1386
|
+
token: resolved.token,
|
|
1387
|
+
wsFactory: browserWsFactory,
|
|
1388
|
+
media: lk ? { kind: "livekit", lk } : void 0
|
|
1389
|
+
});
|
|
1390
|
+
await client2.start();
|
|
1391
|
+
return client2;
|
|
1392
|
+
}
|
|
1205
1393
|
const client = new BrowserVoiceClient({
|
|
1206
1394
|
config: this.config,
|
|
1207
1395
|
// Carry merged context/metadata through to startCall so server can
|