@craftedxp/voice-js 0.9.0 → 0.12.0

package/dist/{incomingCall-CfRRzj2P.d.ts → incomingCall-2B3XB8pM.d.mts}

@@ -1,3 +1,5 @@
+import { g as CallTransport } from './config-DfbPB1Tq.mjs';
+
 type OnChunk = (pcm: ArrayBuffer) => void;
 type OnVolume$1 = (rms01: number) => void;
 type OnError = (err: Error) => void;
@@ -87,7 +89,7 @@ type ReconnectingWebSocket = ReturnType<typeof createReconnectingWebSocket>;
 interface IncomingCallPayload {
     token: string;
     agentId: string;
-    transport: 'ws' | 'webrtc';
+    transport: CallTransport;
     webrtcGatewayBase?: string;
     expiresAt?: number;
     agentName?: string;

package/dist/{incomingCall-CfRRzj2P.d.mts → incomingCall-CYmeQkIw.d.ts}

@@ -1,3 +1,5 @@
+import { g as CallTransport } from './config-DfbPB1Tq.js';
+
 type OnChunk = (pcm: ArrayBuffer) => void;
 type OnVolume$1 = (rms01: number) => void;
 type OnError = (err: Error) => void;
@@ -87,7 +89,7 @@ type ReconnectingWebSocket = ReturnType<typeof createReconnectingWebSocket>;
 interface IncomingCallPayload {
     token: string;
     agentId: string;
-    transport: 'ws' | 'webrtc';
+    transport: CallTransport;
     webrtcGatewayBase?: string;
     expiresAt?: number;
     agentName?: string;

package/dist/node.d.mts

@@ -76,6 +76,8 @@ interface BuildWsUrlArgs {
     agentId: string;
     token: string;
     bargeIn?: boolean;
+    /** Phase 35 — request LiveKit media on the same call socket. */
+    media?: 'livekit';
 }
 declare function buildWsUrl(args: BuildWsUrlArgs): string;
 
@@ -153,6 +155,19 @@ interface FetchTokenArgs {
      */
     metadata?: Record<string, string>;
 }
+/** Media transports the server can select per call. `livekit` since 0.12.0 (phase 35). */
+type CallTransport = 'ws' | 'webrtc' | 'livekit';
+/**
+ * Fired once per call when the transport actually used differs from the one
+ * the server asked for (e.g. `livekit` requested, `livekit-client` not
+ * installed or the room join failed → `ws`). `reason` is free-form diagnostic
+ * text, never a stable code — match on `actual !== requested`.
+ */
+interface TransportChangeEvent {
+    requested: CallTransport;
+    actual: CallTransport;
+    reason?: string;
+}
 /**
  * What `fetchToken` may return. The rich object form lets the server
  * choose the transport per call. Returning a bare string is backwards-
@@ -161,8 +176,10 @@ interface FetchTokenArgs {
 interface FetchTokenResult {
     /** Raw `ct_` to feed into the WS open / WebRTC offer. */
     token: string;
-    /** Server-selected transport. Default `'ws'` if absent. */
-    transport?: 'ws' | 'webrtc';
+    /** Server-selected transport. Default `'ws'` if absent. `'livekit'` =
+     *  audio on LiveKit tracks, control on the call WS; needs the optional
+     *  peer 'livekit-client', otherwise the SDK falls back to 'ws'. */
+    transport?: CallTransport;
     /** Required when `transport === 'webrtc'` AND the server uses a
      *  separate signaling gateway. When omitted on a webrtc result, the
      *  SDK falls back to the API base's Phase-1 routes (local dev). */
@@ -257,6 +274,8 @@ interface StartCallOptions {
      * without diffing state.
      */
     onAgentTurnStart?: () => void;
+    /** See `TransportChangeEvent`. Browser only; Node always uses `ws`. */
+    onTransportChange?: (e: TransportChangeEvent) => void;
 }
 interface Call {
     /** Current state. Snapshot — subscribe via onStateChange for live updates. */
@@ -368,7 +387,7 @@ interface NodeCall extends Call {
     sendClientEvent: (text: string) => boolean;
     /**
      * Send a typed user turn (text/multimodal sessions — token minted
-     * with `channel:'text'`). Sends `{type:'user_text', text}`. The
+     * with `channel:'text'` OR `channel:'multimodal'`). Sends `{type:'user_text', text}`. The
      * server accepts `user_text` only on text-channel sessions and
      * rejects it on voice (anti-injection), so this is a no-op in
      * practice on voice calls beyond the frame being ignored
@@ -376,6 +395,20 @@ interface NodeCall extends Call {
      * if the WS isn't open or text is empty/whitespace. Never throws.
      */
     sendText: (text: string) => boolean;
+    /**
+     * Send an image for multimodal sessions (token minted with
+     * `channel:'multimodal'`). Sends `{type:'user_image', mimeType, data}`.
+     * `data` can be a base64 string (treated as-is) or a buffer
+     * (base64-encoded by the SDK). Returns `false` (and sends nothing)
+     * if: the WS isn't open, data is empty, mimeType is not
+     * png/jpeg/webp/gif, or the base64-encoded length exceeds 5 MB.
+     * Never throws.
+     */
+    sendImage: (data: ArrayBuffer | ArrayBufferView | string, mimeType: string) => boolean;
+    /** Trusted, structured operator directive (vterm workflow buttons).
+     *  Carries only an allow-listed action + a workflow name — no free text —
+     *  so the server accepts it on voice and drives an agent turn from it. */
+    sendDirective: (action: 'observe_workflow' | 'run_workflow', workflow: string) => boolean;
 }
 /**
  * Node bundle's analog of `VoiceClientFactory`. Same shape but
@@ -401,7 +434,7 @@ interface NodeVoiceClientFactory {
 interface IncomingCallPayload {
     token: string;
     agentId: string;
-    transport: 'ws' | 'webrtc';
+    transport: CallTransport;
     webrtcGatewayBase?: string;
     expiresAt?: number;
     agentName?: string;

package/dist/node.d.ts

@@ -76,6 +76,8 @@ interface BuildWsUrlArgs {
     agentId: string;
     token: string;
     bargeIn?: boolean;
+    /** Phase 35 — request LiveKit media on the same call socket. */
+    media?: 'livekit';
 }
 declare function buildWsUrl(args: BuildWsUrlArgs): string;
 
@@ -153,6 +155,19 @@ interface FetchTokenArgs {
      */
     metadata?: Record<string, string>;
 }
+/** Media transports the server can select per call. `livekit` since 0.12.0 (phase 35). */
+type CallTransport = 'ws' | 'webrtc' | 'livekit';
+/**
+ * Fired once per call when the transport actually used differs from the one
+ * the server asked for (e.g. `livekit` requested, `livekit-client` not
+ * installed or the room join failed → `ws`). `reason` is free-form diagnostic
+ * text, never a stable code — match on `actual !== requested`.
+ */
+interface TransportChangeEvent {
+    requested: CallTransport;
+    actual: CallTransport;
+    reason?: string;
+}
 /**
  * What `fetchToken` may return. The rich object form lets the server
  * choose the transport per call. Returning a bare string is backwards-
@@ -161,8 +176,10 @@ interface FetchTokenArgs {
 interface FetchTokenResult {
     /** Raw `ct_` to feed into the WS open / WebRTC offer. */
     token: string;
-    /** Server-selected transport. Default `'ws'` if absent. */
-    transport?: 'ws' | 'webrtc';
+    /** Server-selected transport. Default `'ws'` if absent. `'livekit'` =
+     *  audio on LiveKit tracks, control on the call WS; needs the optional
+     *  peer 'livekit-client', otherwise the SDK falls back to 'ws'. */
+    transport?: CallTransport;
     /** Required when `transport === 'webrtc'` AND the server uses a
      *  separate signaling gateway. When omitted on a webrtc result, the
      *  SDK falls back to the API base's Phase-1 routes (local dev). */
@@ -257,6 +274,8 @@ interface StartCallOptions {
      * without diffing state.
      */
     onAgentTurnStart?: () => void;
+    /** See `TransportChangeEvent`. Browser only; Node always uses `ws`. */
+    onTransportChange?: (e: TransportChangeEvent) => void;
 }
 interface Call {
     /** Current state. Snapshot — subscribe via onStateChange for live updates. */
@@ -368,7 +387,7 @@ interface NodeCall extends Call {
     sendClientEvent: (text: string) => boolean;
     /**
      * Send a typed user turn (text/multimodal sessions — token minted
-     * with `channel:'text'`). Sends `{type:'user_text', text}`. The
+     * with `channel:'text'` OR `channel:'multimodal'`). Sends `{type:'user_text', text}`. The
      * server accepts `user_text` only on text-channel sessions and
      * rejects it on voice (anti-injection), so this is a no-op in
      * practice on voice calls beyond the frame being ignored
@@ -376,6 +395,20 @@ interface NodeCall extends Call {
      * if the WS isn't open or text is empty/whitespace. Never throws.
      */
     sendText: (text: string) => boolean;
+    /**
+     * Send an image for multimodal sessions (token minted with
+     * `channel:'multimodal'`). Sends `{type:'user_image', mimeType, data}`.
+     * `data` can be a base64 string (treated as-is) or a buffer
+     * (base64-encoded by the SDK). Returns `false` (and sends nothing)
+     * if: the WS isn't open, data is empty, mimeType is not
+     * png/jpeg/webp/gif, or the base64-encoded length exceeds 5 MB.
+     * Never throws.
+     */
+    sendImage: (data: ArrayBuffer | ArrayBufferView | string, mimeType: string) => boolean;
+    /** Trusted, structured operator directive (vterm workflow buttons).
+     *  Carries only an allow-listed action + a workflow name — no free text —
+     *  so the server accepts it on voice and drives an agent turn from it. */
+    sendDirective: (action: 'observe_workflow' | 'run_workflow', workflow: string) => boolean;
 }
 /**
  * Node bundle's analog of `VoiceClientFactory`. Same shape but
@@ -401,7 +434,7 @@ interface NodeVoiceClientFactory {
 interface IncomingCallPayload {
     token: string;
     agentId: string;
-    transport: 'ws' | 'webrtc';
+    transport: CallTransport;
     webrtcGatewayBase?: string;
     expiresAt?: number;
     agentName?: string;

package/dist/node.js

@@ -299,7 +299,8 @@ function buildWsUrl(args) {
   const base = new URL(args.apiBase);
   const proto = base.protocol === "https:" ? "wss:" : "ws:";
   const bargeQS = args.bargeIn === false ? "&barge=off" : "";
-  return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}`;
+  const mediaQS = args.media === "livekit" ? "&media=livekit" : "";
+  return `${proto}//${base.host}/v1/agents/${encodeURIComponent(args.agentId)}/call?token=${encodeURIComponent(args.token)}${bargeQS}${mediaQS}`;
 }
 
 // src/clientTools.ts
@@ -503,6 +504,34 @@ var NodeVoiceClient = class {
       this.rws.send(JSON.stringify({ type: "user_text", text: t }));
       return true;
     };
+    this.sendDirective = (action, workflow) => {
+      if (!this.rws || this.rws.readyState() !== READYSTATE_OPEN2) return false;
+      if (action !== "observe_workflow" && action !== "run_workflow") return false;
+      const w = workflow.trim().slice(0, 64);
+      if (!w) return false;
+      this.rws.send(JSON.stringify({ type: "client_directive", action, workflow: w }));
+      return true;
+    };
+    // Send an image for multimodal sessions (channel:'multimodal').
+    // Data can be a base64 string (used as-is) or a buffer (base64-encoded).
+    // Allowed mimes: png, jpeg, webp, gif. Max 5 MB base64-encoded size.
+    this.sendImage = (data, mimeType) => {
+      if (!this.rws || this.rws.readyState() !== READYSTATE_OPEN2) return false;
+      const allowedMimes = ["image/png", "image/jpeg", "image/webp", "image/gif"];
+      if (!allowedMimes.includes(mimeType)) return false;
+      let base64Data;
+      if (typeof data === "string") {
+        base64Data = data;
+      } else {
+        const buffer = ArrayBuffer.isView(data) ? Buffer.from(data.buffer, data.byteOffset, data.byteLength) : Buffer.from(data);
+        base64Data = buffer.toString("base64");
+      }
+      if (!base64Data) return false;
+      const maxSize = 5 * 1024 * 1024;
+      if (base64Data.length > maxSize) return false;
+      this.rws.send(JSON.stringify({ type: "user_image", mimeType, data: base64Data }));
+      return true;
+    };
     // ---------------------------------------------------------------
     // Internal
     // ---------------------------------------------------------------
@@ -649,7 +678,7 @@ var parseIncomingCall = (raw) => {
   if (typeof p.agentId !== "string" || p.agentId.length === 0) {
     throw new Error("parseIncomingCall: missing `agentId`");
   }
-  const transport = p.transport === "webrtc" ? "webrtc" : "ws";
+  const transport = p.transport === "webrtc" ? "webrtc" : p.transport === "livekit" ? "livekit" : "ws";
   const out = { token: p.token, agentId: p.agentId, transport };
   if (transport === "webrtc" && typeof p.webrtcGatewayBase === "string") {
     out.webrtcGatewayBase = p.webrtcGatewayBase;