@craftedxp/voice-js 0.4.2 → 0.5.4

package/CONSUMING.md

@@ -79,14 +79,22 @@ export async function POST(req) {
 
   // Mint with the user's identity. Server attaches contactId, applies the
   // org's tier gate, etc.
-  const token = await platform.callTokens.mint({
+  const result = await platform.callTokens.mint({
     agentId,
     contactId: session.userId,
     context,
     metadata,
     ttlSeconds: 600,
   })
-  return Response.json({ token: token.token })
+  // Forward transport + webrtcGatewayBase so the browser SDK can
+  // dispatch WS vs WebRTC per the agent's configuration. New agents
+  // default to WebRTC since 2026-05-16; legacy agents keep their
+  // stored 'ws' setting.
+  return Response.json({
+    token: result.token,
+    transport: result.transport,
+    webrtcGatewayBase: result.webrtcGatewayBase,
+  })
 }
 ```
 

package/README.md

@@ -41,13 +41,22 @@ import { configureVoiceClient } from '@craftedxp/voice-js'
 const voice = configureVoiceClient({
   apiBase: 'https://api.your-server.com',
   // SDK calls this whenever it needs a fresh ct_ — initial connect
-  // and any mid-call token refresh. Your backend handles the mint.
+  // and any mid-call token refresh. Your backend handles the mint and
+  // forwards the mint response's `transport` + `webrtcGatewayBase` so
+  // the SDK can dispatch WS vs WebRTC per the agent's configuration.
+  // (New agents default to WebRTC since 2026-05-16. The bare-string
+  // form below is still accepted for back-compat — it always uses WS.)
   fetchToken: async ({ agentId }) => {
     const r = await fetch('/api/voice/mint', {
       method: 'POST',
       body: JSON.stringify({ agentId }),
     })
-    return (await r.json()).token
+    const body = await r.json()
+    return {
+      token: body.token,
+      transport: body.transport, // 'ws' | 'webrtc'
+      webrtcGatewayBase: body.webrtcGatewayBase, // present when transport=webrtc
+    }
   },
   // Optional — applied to every call. Per-call options merge on top.
   defaultMetadata: { surface: 'web', appVersion: '1.4.0' },
@@ -131,12 +140,12 @@ The Node bundle has the same `configureVoiceClient` / `startCall` shape, plus an
 
 ### `configureVoiceClient(config)`
 
-| Field             | Type                        | Notes                                                                                                                                           |
-| ----------------- | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
-| `apiBase`         | `string`                    | Full HTTPS URL of the Voissia server. WS scheme derived: `https`→`wss`. Trailing slash optional.                                                |
-| `fetchToken`      | `(args) => Promise<string>` | Called by the SDK whenever it needs a fresh `ct_`. Mirrors `@craftedxp/voice-rn`'s shape exactly — `{ agentId, userId?, context?, metadata? }`. |
-| `defaultMetadata` | `Record<string, string>?`   | Applied to every `startCall`. Per-call merges on top.                                                                                           |
-| `defaultContext`  | `Record<string, unknown>?`  | Applied to every `startCall`. Per-call merges on top.                                                                                           |
+| Field             | Type                                            | Notes                                                                                                                                                                                                                                                                                              |
+| ----------------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `apiBase`         | `string`                                        | Full HTTPS URL of the Voissia server. WS scheme derived: `https`→`wss`. Trailing slash optional.                                                                                                                                                                                                   |
+| `fetchToken`      | `(args) => Promise<string \| FetchTokenResult>` | Called by the SDK whenever it needs a fresh `ct_`. Args: `{ agentId, userId?, context?, metadata? }`. Return either a bare `ct_` string (always WS, back-compat) or the rich `{ token, transport, webrtcGatewayBase? }` object so the SDK can dispatch WS vs WebRTC per the agent's configuration. |
+| `defaultMetadata` | `Record<string, string>?`                       | Applied to every `startCall`. Per-call merges on top.                                                                                                                                                                                                                                              |
+| `defaultContext`  | `Record<string, unknown>?`                      | Applied to every `startCall`. Per-call merges on top.                                                                                                                                                                                                                                              |
 
 Returns a `VoiceClientFactory` with one method:
 
@@ -211,6 +220,50 @@ type CallErrorCode =
 type CallEndReason = 'agent_ended' | 'user_hangup' | 'timeout' | 'error'
 ```
 
+## Animating while the agent is talking
+
+Four optional callbacks on `startCall` give you everything you need to drive an "agent is talking" animation:
+
+| Callback                      | What                                                                                                          |
+| ----------------------------- | ------------------------------------------------------------------------------------------------------------- |
+| `onStateChange(state)`        | `state === 'agent_speaking'` is true while the agent is in its speaking turn.                                 |
+| `onAgentTurnStart()`          | Fires the moment the agent starts a turn — a discrete trigger if you want an "activation" cue.                |
+| `onInterrupt()`               | The user barged in. End the animation early.                                                                  |
+| `onVolume({ input, output })` | Real-time amplitude. `output` is the agent's playback level (≈ what the listener hears); `input` is your mic. |
+
+`onStateChange === 'agent_speaking'` is **protocol-driven**: it flips on as soon as the server begins the turn, _before_ the first audio sample reaches the speaker. For a visual that follows what the listener actually hears, drive it from `onVolume.output`.
+
+The usual recipe is a stable gate × live amplitude:
+
+```ts
+let isAgentTurn = false
+let amplitude = 0
+
+const call = await client.startCall({
+  agentId,
+  onStateChange: (s) => {
+    isAgentTurn = s === 'agent_speaking'
+    render()
+  },
+  onInterrupt: () => {
+    isAgentTurn = false
+    render()
+  },
+  onVolume: ({ output }) => {
+    amplitude = output
+    render()
+  },
+})
+
+function render() {
+  if (isAgentTurn)
+    showPulse({ scale: 1 + amplitude }) // your renderer
+  else hidePulse()
+}
+```
+
+> 1:1 calls only — in multi-party rooms (`joinRoom`) the in-room agent is a silent notetaker and has no speaking turn, so none of these signals fire for it.
+
 ## Client tools
 
 You can declare tools the agent's LLM can call **on the consumer's machine**. The
@@ -251,7 +304,7 @@ sent back; throws become `{ error: ... }` frames. The server enforces a default
 10s / max 30s timeout per `timeoutMs` in your declaration.
 
 For the full wire protocol, sequencing, and constraints see
-[`docs/integration-echocheck.md`](../../docs/integration-echocheck.md#client-declared-tools).
+[`docs/sdks.md` → Client tools](../../docs/sdks.md#client-tools).
 
 ## Migrating from `@voxline/web`
 
@@ -324,7 +377,7 @@ false-positive VAD on background noise). Three quick fixes, in order:
 
 For the full diagnostic walkthrough (including the rarer Gemini-Live
 stale-audio-leak case and audio-handling guidance for Node/Electron consumers),
-see [`docs/integration-echocheck.md` → Audio quality](../../docs/integration-echocheck.md#audio-quality--the-cut-off-syllable-trap).
+see [`docs/sdks.md` → Audio quality troubleshooting](../../docs/sdks.md#audio-quality-troubleshooting).
 
 ## Embed widget
 
@@ -344,7 +397,12 @@ Renders a floating call button with a Shadow-DOM transcript panel. Pre-mint the
 
 ## Status
 
-- **0.4.2** (current) — WebRTC reliability: `onicecandidate` + `onconnectionstatechange` listeners now register before `setLocalDescription` (where ICE gathering starts). Candidates emitted before `callId` is known are buffered and flushed once the answer arrives. Browsers buffer candidates internally so the prior listen-after-`setRemoteDescription` pattern worked in Chrome/Safari, but the explicit ordering is correct per spec and matches the fix shipped in `@craftedxp/voice-rn@0.4.1`. No API surface change — drop-in upgrade.
+- **0.5.4** (current) — Screen sharing. `setScreenShareEnabled(on, { audio })` publishes a `screen_share` video track (and, where the browser allows, a `screen_share_audio` track — Chrome captures tab/system audio; macOS Chrome is tab-audio only; Safari/Firefox don't capture share audio). `RoomTrackEvent` gains a `source` field (`camera` / `microphone` / `screen_share` / `screen_share_audio` / `unknown`) so a screen share can render as its own tile instead of replacing the participant's camera. Adds `isScreenShareEnabled()` and `getLocalScreenTrack()`. No 1:1 call-surface change. Drop-in for 0.5.3 consumers (the new `source` field is additive).
+- 0.5.3 — `session.participantId` (this session's own stable `p_…` id). `active.speakers` includes the local participant, so a focus-tile UI rendered you as a remote when you spoke (no remote track → blank tile with the raw id). Filter `participantId` out of `active.speakers` to focus only remote speakers (self-view when none). No 1:1 call-surface change. Drop-in upgrade for 0.5.2 consumers.
+- 0.5.2 — `getRemoteTracks(): RoomTrackEvent[]`. Returns remote tracks already subscribed at call time. A late joiner misses the live `track.subscribed` events for tracks published before it connected (LiveKit delivers them during `connect`, before consumer listeners attach), so it never rendered participants who already had their camera on (e.g. the host). Call `getRemoteTracks()` right after registering `track.subscribed` to backfill them. No 1:1 call-surface change. Drop-in upgrade for 0.5.1 consumers.
+- 0.5.1 — Room video surface. `RoomSession` gains: `track.subscribed` / `track.unsubscribed` events with payload `{ participantId: string; kind: 'audio' | 'video'; track: RemoteTrack }` (raw livekit-client track — call `track.attach(el)` / `track.detach()`); `active.speakers` event (`string[]` of participantIds currently speaking, drives active-speaker UI); `setMicEnabled(on: boolean): Promise<void>` / `setCameraEnabled(on: boolean): Promise<void>` (mid-call toggles); `isMicEnabled(): boolean` / `isCameraEnabled(): boolean` (read current state for toggle button UI); `getLocalCameraTrack(): LocalVideoTrack | null` (attach to self-view element). No API changes to the 1:1 call surface (`startCall` / `Call`). Drop-in upgrade for 0.5.0 consumers.
+- 0.5.0 — Multi-party rooms. `joinRoom({ roomId, joinCode, name })` returns a typed `RoomSession` with `participant.{joined,left}`, `transcript.partial`, `system.message`, `room.ended` events. Adds `livekit-client` as a direct dependency (~250 KB gzipped, tree-shakes if unused). `publishMic()` / `publishCamera()` / `leave()` methods. See [`docs/sdks.md` → Multi-party rooms](../../docs/sdks.md#multi-party-rooms).
+- 0.4.2 — WebRTC reliability: `onicecandidate` + `onconnectionstatechange` listeners now register before `setLocalDescription` (where ICE gathering starts). Candidates emitted before `callId` is known are buffered and flushed once the answer arrives. Browsers buffer candidates internally so the prior listen-after-`setRemoteDescription` pattern worked in Chrome/Safari, but the explicit ordering is correct per spec and matches the fix shipped in `@craftedxp/voice-rn@0.4.1`. No API surface change — drop-in upgrade.
 - 0.4.1 — `client_tools` over the WebRTC DataChannel — tool register on `connected`, dispatch on `client_tool_call` frames, parity with the WS transport.
 - 0.4.0 — WebRTC transport support. `fetchToken` may now return `{ token, transport, webrtcGatewayBase? }`; the SDK dispatches WS vs WebRTC accordingly. Backwards-compatible — bare-string returns still always use WS.
 - 0.3.2 — bug fix: `onStateChange` now fires for state transitions driven by server frames (`connected → listening`, `agent_turn_start → agent_speaking`, etc.). Latent regression since 0.2.0; `onTranscript`-only consumers were unaffected, but anyone deriving UI from `onStateChange` should upgrade. No API changes — drop-in.

package/dist/browser.d.mts

@@ -1,3 +1,5 @@
+import { RemoteTrack, LocalVideoTrack } from 'livekit-client';
+
 interface ClientTool {
     description: string;
     parameters: Record<string, unknown>;
@@ -79,6 +81,146 @@ interface BuildWsUrlArgs {
 }
 declare function buildWsUrl(args: BuildWsUrlArgs): string;
 
+type SystemMessage = {
+    kind: 'room.starting';
+    at: string;
+} | {
+    kind: 'room.ending.soon';
+    minutesRemaining: 5 | 1;
+} | {
+    kind: 'room.ended';
+    reason: 'duration_reached' | 'manual' | 'empty';
+} | {
+    kind: 'role.promoted';
+    participantId: string;
+    name: string;
+} | {
+    kind: 'role.demoted';
+    participantId: string;
+    name: string;
+} | {
+    kind: 'participant.removed';
+    participantId: string;
+    name: string;
+    byHost?: string;
+} | {
+    kind: 'notetaker.connected';
+} | {
+    kind: 'notetaker.disconnected';
+} | {
+    kind: 'notetaker.partial_degraded';
+    participantId: string;
+};
+type TranscriptMessage = {
+    kind: 'partial';
+    participantId: string;
+    speakerName: string;
+    text: string;
+    startedAt: string;
+};
+
+interface JoinRoomOptions {
+    /** Full HTTPS URL of the Voissia server. Same shape as VoiceClientConfig.apiBase. */
+    apiBase: string;
+    /** Server-generated room id (`rm_…`). */
+    roomId: string;
+    /** Shared room join token from the invite link. Mints a fresh participant each call. */
+    joinCode: string;
+    /** Display name the joiner registers under for this participant. */
+    name: string;
+}
+interface RoomParticipantInfo {
+    /** Stable participant id (`p_…`); strips any `guest:` LiveKit identity prefix. */
+    participantId: string;
+    /** Display name as the worker registered it; may be empty. */
+    name: string;
+}
+type RoomTrackKind = 'audio' | 'video';
+/** What a track is — lets consumers tell a camera apart from a screen share
+ *  (a participant can publish both at once). Mirrors livekit `Track.Source`. */
+type RoomTrackSource = 'camera' | 'microphone' | 'screen_share' | 'screen_share_audio' | 'unknown';
+interface RoomTrackEvent {
+    /** Stable participant id (`p_…`); `guest:` prefix stripped. */
+    participantId: string;
+    kind: RoomTrackKind;
+    /** Distinguishes camera vs screen_share so each can render as its own tile. */
+    source: RoomTrackSource;
+    /** livekit-client track — call `.attach(el)` / `.detach()` to render. */
+    track: RemoteTrack;
+}
+type RoomEventName = 'participant.joined' | 'participant.left' | 'transcript.partial' | 'transcript.final' | 'system.message' | 'room.ended' | 'track.subscribed' | 'track.unsubscribed' | 'active.speakers';
+interface RoomEventPayloads {
+    'participant.joined': RoomParticipantInfo;
+    'participant.left': RoomParticipantInfo;
+    'transcript.partial': TranscriptMessage;
+    /**
+     * Reserved — the worker currently emits only partials over the transcript
+     * topic. Final-utterance events will land in Phase 8 once the worker
+     * publishes a `final` kind; the SDK keeps the slot reserved so consumers
+     * can register handlers today.
+     */
+    'transcript.final': {
+        participantId: string;
+        speakerName: string;
+        text: string;
+        startedAt: string;
+    };
+    'system.message': SystemMessage;
+    'room.ended': undefined;
+    'track.subscribed': RoomTrackEvent;
+    'track.unsubscribed': RoomTrackEvent;
+    /** participantIds currently speaking (drives an active-speaker UI). */
+    'active.speakers': string[];
+}
+type Handler<E extends RoomEventName> = (payload: RoomEventPayloads[E]) => void;
+interface RoomSession {
+    /** This session's own stable participant id (`p_…`). Useful to filter
+     *  yourself out of `active.speakers`, which includes the local participant. */
+    readonly participantId: string;
+    /** Snapshot of the remote participants currently connected. */
+    readonly participants: RoomParticipantInfo[];
+    /** Subscribe to a typed event. No unsubscribe surface yet (mirrors `Call.onX`). */
+    on<E extends RoomEventName>(event: E, handler: Handler<E>): void;
+    /** Publish the local mic track. Resolves once the track is live on LiveKit. */
+    publishMic(): Promise<void>;
+    /** Publish the local camera track. */
+    publishCamera(): Promise<void>;
+    /** Mid-call mute/unmute of the local mic. */
+    setMicEnabled(on: boolean): Promise<void>;
+    /** Mid-call camera on/off. */
+    setCameraEnabled(on: boolean): Promise<void>;
+    /** Current local mic state (for toggle UI). */
+    isMicEnabled(): boolean;
+    /** Current local camera state (for toggle UI). */
+    isCameraEnabled(): boolean;
+    /** The local camera track for self-view, or null before publishCamera resolves. */
+    getLocalCameraTrack(): LocalVideoTrack | null;
+    /**
+     * Remote tracks already subscribed at this moment. A late joiner misses the
+     * live `track.subscribed` events for tracks published before it connected
+     * (LiveKit delivers them during `connect`, before consumer listeners attach).
+     * Call this right after registering `track.subscribed` to backfill them.
+     */
+    getRemoteTracks(): RoomTrackEvent[];
+    /**
+     * Start/stop sharing the screen (via `getDisplayMedia`). Pass `{ audio: true }`
+     * to also capture shared/system audio where the browser allows it (Chrome:
+     * tab or system audio; macOS Chrome is tab-audio only; Safari/Firefox don't
+     * capture share audio). Publishes a `screen_share` video track (+ optional
+     * `screen_share_audio`); remote peers receive them via `track.subscribed`.
+     */
+    setScreenShareEnabled(on: boolean, opts?: {
+        audio?: boolean;
+    }): Promise<void>;
+    /** Current local screen-share state (for toggle UI). */
+    isScreenShareEnabled(): boolean;
+    /** The local screen-share video track for self-preview, or null when off. */
+    getLocalScreenTrack(): LocalVideoTrack | null;
+    /** Disconnect from LiveKit. Idempotent. Triggers `room.ended` via Disconnected. */
+    leave(): Promise<void>;
+}
+declare const joinRoom: (opts: JoinRoomOptions) => Promise<RoomSession>;
+
 interface FetchTokenArgs {
     /** The agent the SDK is about to call. */
     agentId: string;
@@ -228,6 +370,16 @@ interface VoiceClientFactory {
      * don't reject this promise.
      */
     startCall: (options: StartCallOptions) => Promise<Call>;
+    /**
+     * Phase 7 (multi-party rooms). Browser only. Exchange a single-use
+     * joinCode for a LiveKit JWT and connect to the room. The returned
+     * `RoomSession` exposes a typed event surface
+     * (participant.joined / participant.left / transcript.partial /
+     *  transcript.final / system.message / room.ended) plus
+     * publishMic / publishCamera / leave. The Node bundle does NOT
+     * implement this — livekit-client is a browser-only WebRTC client.
+     */
+    joinRoom?: (options: Omit<JoinRoomOptions, 'apiBase'>) => Promise<RoomSession>;
 }
 
 type OnChunk = (pcm: ArrayBuffer) => void;
@@ -307,6 +459,31 @@ declare const createReconnectingWebSocket: (options: RWSOptions, onEvent: (ev: R
 };
 type ReconnectingWebSocket = ReturnType<typeof createReconnectingWebSocket>;
 
+/**
+ * Canonical payload a tenant places in their VoIP/FCM push so an
+ * agent-initiated call can connect. It is the `callTokens.mint` result
+ * (token + transport) plus two optional display fields for the native
+ * incoming-call UI. The host receives the push, runs `parseIncomingCall`,
+ * and on accept passes `token` (+ transport / webrtcGatewayBase) into the
+ * voice client. Web background-wake is best-effort (Web Push); native is
+ * the real target. See docs/sdks.md "Agent-initiated calls".
+ */
+interface IncomingCallPayload {
+    token: string;
+    agentId: string;
+    transport: 'ws' | 'webrtc';
+    webrtcGatewayBase?: string;
+    expiresAt?: number;
+    agentName?: string;
+    agentAvatarUrl?: string;
+}
+/**
+ * Validate + normalise a raw push payload into an IncomingCallPayload.
+ * Throws synchronously on malformed input. Unknown transports fall back to
+ * 'ws'; webrtcGatewayBase is ignored unless transport === 'webrtc'.
+ */
+declare const parseIncomingCall: (raw: unknown) => IncomingCallPayload;
+
 /**
  * One-time SDK setup. Returns a factory you call `startCall` on for
  * every voice call.
@@ -334,4 +511,4 @@ type ReconnectingWebSocket = ReturnType<typeof createReconnectingWebSocket>;
  */
 declare function configureVoiceClient(config: VoiceClientConfig): VoiceClientFactory;
 
-export { type Call, type CallEndEvent, type CallEndReason, type CallError, type CallErrorCode, type CallState, type CaptureController, type CaptureOptions, type ClientTool, type ClientToolMap, type FetchToken, type FetchTokenArgs, type FetchTokenResult, type OnAgentSpeakingChange, type OnChunk, type OnError, type OnVolume$1 as OnVolume, type PlaybackController, type PlaybackOptions, type ProtocolCallbacks, type ProtocolState, type RWSEvent, type RWSOptions, type ReconnectingWebSocket, type ServerMessage, type StartCallOptions, type TranscriptEntry, type VoiceClientConfig, type VoiceClientFactory, type VolumeEvent, type WebSocketFactory, type WebSocketLike, buildWsUrl, configureVoiceClient, createAudioCapture, createAudioPlayback, createProtocolState, createReconnectingWebSocket, handleServerMessage };
+export { type Call, type CallEndEvent, type CallEndReason, type CallError, type CallErrorCode, type CallState, type CaptureController, type CaptureOptions, type ClientTool, type ClientToolMap, type FetchToken, type FetchTokenArgs, type FetchTokenResult, type IncomingCallPayload, type JoinRoomOptions, type OnAgentSpeakingChange, type OnChunk, type OnError, type OnVolume$1 as OnVolume, type PlaybackController, type PlaybackOptions, type ProtocolCallbacks, type ProtocolState, type RWSEvent, type RWSOptions, type ReconnectingWebSocket, type RoomEventName, type RoomEventPayloads, type RoomParticipantInfo, type RoomSession, type ServerMessage, type StartCallOptions, type SystemMessage, type TranscriptEntry, type TranscriptMessage, type VoiceClientConfig, type VoiceClientFactory, type VolumeEvent, type WebSocketFactory, type WebSocketLike, buildWsUrl, configureVoiceClient, createAudioCapture, createAudioPlayback, createProtocolState, createReconnectingWebSocket, handleServerMessage, joinRoom, parseIncomingCall };