@elevenlabs/client 0.6.2 → 0.7.1

package/dist/utils/BaseConnection.d.ts

@@ -1,6 +1,8 @@
 import type { IncomingSocketEvent, OutgoingSocketEvent } from "./events";
 import type { Mode } from "../BaseConversation";
-export type Language = "en" | "ja" | "zh" | "de" | "hi" | "fr" | "ko" | "pt" | "pt-br" | "it" | "es" | "id" | "nl" | "tr" | "pl" | "sv" | "bg" | "ro" | "ar" | "cs" | "el" | "fi" | "ms" | "da" | "ta" | "uk" | "ru" | "hu" | "hr" | "sk" | "no" | "vi" | "tl";
+import type { ConversationConfigOverrideAgentLanguage as Language } from "@elevenlabs/types/generated/types/asyncapi-types";
+import type { DisconnectionDetails } from "@elevenlabs/types";
+export type { DisconnectionDetails, ConversationConfigOverrideAgentLanguage as Language, } from "@elevenlabs/types";
 export type DelayConfig = {
     default: number;
     android?: number;
@@ -11,16 +13,6 @@ export type FormatConfig = {
     sampleRate: number;
     outputDeviceId?: string;
 };
-export type DisconnectionDetails = {
-    reason: "error";
-    message: string;
-    context: Event;
-} | {
-    reason: "agent";
-    context: CloseEvent;
-} | {
-    reason: "user";
-};
 export type OnDisconnectCallback = (details: DisconnectionDetails) => void;
 export type OnMessageCallback = (event: IncomingSocketEvent) => void;
 export type BaseSessionConfig = {

package/dist/utils/{audioConcatProcessor.d.ts → audioConcatProcessor.generated.d.ts}

@@ -1 +1 @@
-export declare const loadAudioConcatProcessor: (worklet: AudioWorklet) => Promise<void>;
+export declare const loadAudioConcatProcessor: (worklet: AudioWorklet, path?: string) => Promise<void>;

package/dist/utils/createWorkletModuleLoader.d.ts

@@ -1 +1 @@
-export declare function createWorkletModuleLoader(name: string, sourceCode: string): (worklet: AudioWorklet) => Promise<void>;
+export declare function createWorkletModuleLoader(name: string, sourceCode: string): (worklet: AudioWorklet, path?: string) => Promise<void>;

package/dist/utils/events.d.ts

@@ -1,161 +1,29 @@
-import type { Language } from "./connection";
-import type { CONVERSATION_INITIATION_CLIENT_DATA_TYPE } from "./overrides";
-export type UserTranscriptionEvent = {
-    type: "user_transcript";
-    user_transcription_event: {
-        user_transcript: string;
-    };
-};
-export type AgentResponseEvent = {
-    type: "agent_response";
-    agent_response_event: {
-        agent_response: string;
-    };
-};
-export type AgentAudioEvent = {
-    type: "audio";
-    audio_event: {
-        audio_base_64: string;
-        event_id: number;
-    };
-};
-export type InterruptionEvent = {
-    type: "interruption";
-    interruption_event: {
-        event_id: number;
-    };
-};
-export type InternalTentativeAgentResponseEvent = {
-    type: "internal_tentative_agent_response";
-    tentative_agent_response_internal_event: {
-        tentative_agent_response: string;
-    };
-};
-export type ConfigEvent = {
-    type: "conversation_initiation_metadata";
-    conversation_initiation_metadata_event: {
-        conversation_id: string;
-        agent_output_audio_format: string;
-        user_input_audio_format?: string;
-    };
-};
-export type PingEvent = {
-    type: "ping";
-    ping_event: {
-        event_id: number;
-        ping_ms?: number;
-    };
-};
-export type ClientToolCallEvent = {
-    type: "client_tool_call";
-    client_tool_call: {
-        tool_name: string;
-        tool_call_id: string;
-        parameters: any;
-        expects_response: boolean;
-    };
-};
-export type VadScoreEvent = {
-    type: "vad_score";
-    vad_score_event: {
-        vad_score: number;
-    };
-};
-interface BaseMCPToolCallClientEventData {
-    service_id: string;
-    tool_call_id: string;
-    tool_name: string;
-    tool_description?: string;
-    parameters: Record<string, any>;
-    timestamp: string;
-}
-interface MCPToolCallClientEventLoadingData extends BaseMCPToolCallClientEventData {
-    state: "loading";
-}
-interface MCPToolCallClientEventAwaitingApprovalData extends BaseMCPToolCallClientEventData {
-    state: "awaiting_approval";
-    approval_timeout_secs: number;
-}
-interface MCPToolCallClientEventSuccessData extends BaseMCPToolCallClientEventData {
-    state: "success";
-    result: any[];
-}
-interface MCPToolCallClientEventFailureData extends BaseMCPToolCallClientEventData {
-    state: "failure";
-    error_message: string;
-}
-type MCPToolCallClientEventData = MCPToolCallClientEventLoadingData | MCPToolCallClientEventAwaitingApprovalData | MCPToolCallClientEventSuccessData | MCPToolCallClientEventFailureData;
-export type MCPToolCallClientEvent = {
-    type: "mcp_tool_call";
-    mcp_tool_call: MCPToolCallClientEventData;
-};
-export type AgentResponseCorrectionEvent = {
-    type: "agent_response_correction";
-    agent_response_correction_event: {
-        original_agent_response: string;
-        corrected_agent_response: string;
-    };
-};
-export type IncomingSocketEvent = UserTranscriptionEvent | AgentResponseEvent | AgentResponseCorrectionEvent | AgentAudioEvent | InterruptionEvent | InternalTentativeAgentResponseEvent | ConfigEvent | PingEvent | ClientToolCallEvent | VadScoreEvent | MCPToolCallClientEvent;
-export type PongEvent = {
-    type: "pong";
-    event_id: number;
-};
-export type UserAudioEvent = {
-    user_audio_chunk: string;
-};
-export type UserFeedbackEvent = {
-    type: "feedback";
-    score: "like" | "dislike";
-    event_id: number;
-};
-export type ClientToolResultEvent = {
-    type: "client_tool_result";
-    tool_call_id: string;
-    result: any;
-    is_error: boolean;
-};
-export type InitiationClientDataEvent = {
-    type: typeof CONVERSATION_INITIATION_CLIENT_DATA_TYPE;
-    conversation_config_override?: {
-        agent?: {
-            prompt?: {
-                prompt?: string;
-            };
-            first_message?: string;
-            language?: Language;
-        };
-        tts?: {
-            voice_id?: string;
-        };
-        conversation?: {
-            text_only?: boolean;
-        };
-    };
-    custom_llm_extra_body?: any;
-    dynamic_variables?: Record<string, string | number | boolean>;
-    user_id?: string;
-    source_info?: {
-        source?: string;
-        version?: string;
-    };
-};
-export type ContextualUpdateEvent = {
-    type: "contextual_update";
-    text: string;
-};
-export type UserMessageEvent = {
-    type: "user_message";
-    text: string;
-};
-export type UserActivityEvent = {
-    type: "user_activity";
-};
-export type MCPToolApprovalResultEvent = {
-    type: "mcp_tool_approval_result";
-    tool_call_id: string;
-    is_approved: boolean;
-};
+import { Outgoing } from "@elevenlabs/types";
+import { AgentResponse, AgentResponseCorrection, AgentToolResponseClientEvent, AsrInitiationMetadataEvent as AsrMetadataEvent, Audio, ClientToolCallMessage, ConversationMetadata, Interruption, McpConnectionStatusClientEvent, McpToolCall, Ping, InternalTentativeAgentResponse as TentativeAgentResponseInternal, UserTranscript, VadScore } from "@elevenlabs/types/generated/types/asyncapi-types";
+export type UserTranscriptionEvent = UserTranscript;
+export type AgentResponseEvent = AgentResponse;
+export type AgentAudioEvent = Audio;
+export type InterruptionEvent = Interruption;
+export type InternalTentativeAgentResponseEvent = TentativeAgentResponseInternal;
+export type ConfigEvent = ConversationMetadata;
+export type PingEvent = Ping;
+export type ClientToolCallEvent = ClientToolCallMessage;
+export type VadScoreEvent = VadScore;
+export type MCPToolCallClientEvent = McpToolCall;
+export type AgentResponseCorrectionEvent = AgentResponseCorrection;
+export type AgentToolResponseEvent = AgentToolResponseClientEvent;
+export type ConversationMetadataEvent = ConversationMetadata;
+export type AsrInitiationMetadataEvent = AsrMetadataEvent;
+export type MCPConnectionStatusEvent = McpConnectionStatusClientEvent;
+export type IncomingSocketEvent = UserTranscriptionEvent | AgentResponseEvent | AgentResponseCorrectionEvent | AgentAudioEvent | InterruptionEvent | InternalTentativeAgentResponseEvent | ConfigEvent | PingEvent | ClientToolCallEvent | VadScoreEvent | MCPToolCallClientEvent | AgentToolResponseEvent | ConversationMetadataEvent | AsrInitiationMetadataEvent | MCPConnectionStatusEvent;
+export type PongEvent = Outgoing.PongClientToOrchestratorEvent;
+export type UserAudioEvent = Outgoing.UserAudio;
+export type UserFeedbackEvent = Outgoing.UserFeedbackClientToOrchestratorEvent;
+export type ClientToolResultEvent = Outgoing.ClientToolResultClientToOrchestratorEvent;
+export type InitiationClientDataEvent = Outgoing.ConversationInitiationClientToOrchestratorEvent;
+export type ContextualUpdateEvent = Outgoing.ContextualUpdateClientToOrchestratorEvent;
+export type UserMessageEvent = Outgoing.UserMessageClientToOrchestratorEvent;
+export type UserActivityEvent = Outgoing.UserActivityClientToOrchestratorEvent;
+export type MCPToolApprovalResultEvent = Outgoing.McpToolApprovalResultClientToOrchestratorEvent;
 export type OutgoingSocketEvent = PongEvent | UserAudioEvent | InitiationClientDataEvent | UserFeedbackEvent | ClientToolResultEvent | ContextualUpdateEvent | UserMessageEvent | UserActivityEvent | MCPToolApprovalResultEvent;
 export declare function isValidSocketEvent(event: any): event is IncomingSocketEvent;
-export {};

package/dist/utils/input.d.ts

@@ -1,4 +1,5 @@
 import type { FormatConfig } from "./connection";
+import type { AudioWorkletConfig } from "../BaseConversation";
 export type InputConfig = {
     preferHeadphonesForIosDevices?: boolean;
     inputDeviceId?: string;
@@ -9,9 +10,9 @@ export declare class Input {
     readonly worklet: AudioWorkletNode;
     inputStream: MediaStream;
     private mediaStreamSource;
-    static create({ sampleRate, format, preferHeadphonesForIosDevices, inputDeviceId, }: FormatConfig & InputConfig): Promise<Input>;
+    static create({ sampleRate, format, preferHeadphonesForIosDevices, inputDeviceId, workletPaths, libsampleratePath, }: FormatConfig & InputConfig & AudioWorkletConfig): Promise<Input>;
     private constructor();
     close(): Promise<void>;
     setMuted(isMuted: boolean): void;
-    setInputDevice(inputDeviceId: string): Promise<void>;
+    setInputDevice(inputDeviceId?: string): Promise<void>;
 }

package/dist/utils/output.d.ts

@@ -1,12 +1,16 @@
 import type { FormatConfig } from "./connection";
+import type { AudioWorkletConfig } from "../BaseConversation";
+export type OutputConfig = {
+    outputDeviceId?: string;
+};
 export declare class Output {
     readonly context: AudioContext;
     readonly analyser: AnalyserNode;
     readonly gain: GainNode;
     readonly worklet: AudioWorkletNode;
     readonly audioElement: HTMLAudioElement;
-    static create({ sampleRate, format, outputDeviceId, }: FormatConfig): Promise<Output>;
+    static create({ sampleRate, format, outputDeviceId, workletPaths, }: FormatConfig & OutputConfig & AudioWorkletConfig): Promise<Output>;
     private constructor();
-    setOutputDevice(deviceId: string): Promise<void>;
+    setOutputDevice(deviceId?: string): Promise<void>;
     close(): Promise<void>;
 }

package/dist/utils/{rawAudioProcessor.d.ts → rawAudioProcessor.generated.d.ts}

@@ -1 +1 @@
-export declare const loadRawAudioProcessor: (worklet: AudioWorklet) => Promise<void>;
+export declare const loadRawAudioProcessor: (worklet: AudioWorklet, path?: string) => Promise<void>;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const PACKAGE_VERSION = "0.6.2";
+export declare const PACKAGE_VERSION = "0.7.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elevenlabs/client",
-  "version": "0.6.2",
+  "version": "0.7.1",
   "description": "ElevenLabs JavaScript Client Library",
   "main": "./dist/lib.umd.js",
   "module": "./dist/lib.module.js",
@@ -34,15 +34,20 @@
     "url": "git+https://github.com/elevenlabs/packages.git",
     "directory": "packages/client"
   },
+  "publishConfig": {
+    "access": "public"
+  },
   "dependencies": {
-    "livekit-client": "^2.11.4"
+    "livekit-client": "^2.11.4",
+    "@elevenlabs/types": "0.0.1"
   },
   "scripts": {
     "generate-version": "printf \"// This file is auto-generated during build\\nexport const PACKAGE_VERSION = \\\"%s\\\";\\n\" \"$npm_package_version\" > src/version.ts",
-    "prebuild": "pnpm run generate-version",
+    "generate-worklets": "node scripts/generateWorklets.js",
+    "prebuild": "pnpm run generate-version && pnpm run generate-worklets",
     "build": "BROWSERSLIST_ENV=modern microbundle --jsx React.createElement --jsxFragment React.Fragment --jsxImportSource react src/index.ts",
     "clean": "rm -rf ./dist",
-    "dev": "pnpm run clean && pnpm run generate-version && BROWSERSLIST_ENV=development microbundle --jsx React.createElement --jsxFragment React.Fragment --jsxImportSource react src/index.ts -w -f modern",
+    "dev": "pnpm run clean && pnpm run generate-version && pnpm run generate-worklets && BROWSERSLIST_ENV=development microbundle --jsx React.createElement --jsxFragment React.Fragment --jsxImportSource react src/index.ts -w -f modern",
     "lint": "pnpm run lint:ts && pnpm run lint:es && pnpm run lint:prettier",
     "lint:ts": "tsc --noEmit --skipLibCheck",
     "lint:es": "pnpm exec eslint .",

package/scripts/generateWorklets.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const workletDir = path.join(__dirname, '../worklets');
+const outputDir = path.join(__dirname, '../src/utils');
+
+/** Creates ts versions of needed js scripts to code so that they can be exported and used during runtime. Automatically run during build */
+
+const worklets = [
+  {
+    jsFile: 'rawAudioProcessor.js',
+    tsFile: 'rawAudioProcessor.generated.ts',
+    exportName: 'loadRawAudioProcessor',
+    processorName: 'rawAudioProcessor'
+  },
+  {
+    jsFile: 'audioConcatProcessor.js', 
+    tsFile: 'audioConcatProcessor.generated.ts',
+    exportName: 'loadAudioConcatProcessor',
+    processorName: 'audioConcatProcessor'
+  }
+];
+
+console.log('Generating TypeScript worklet files...');
+
+worklets.forEach(({ jsFile, tsFile, exportName, processorName }) => {
+  const jsPath = path.join(workletDir, jsFile);
+  const tsPath = path.join(outputDir, tsFile);
+  
+  const jsContent = fs.readFileSync(jsPath, 'utf8');
+  
+  const tsContent = `// AUTO-GENERATED BY packages/client/scripts/generateWorklets.js
+import { createWorkletModuleLoader } from "./createWorkletModuleLoader";
+
+export const ${exportName} = createWorkletModuleLoader(
+  "${processorName}",
+  // language=JavaScript
+  \`${jsContent}\`
+);
+`;
+
+  // Write the TS file
+  fs.writeFileSync(tsPath, tsContent);
+  console.log(`Generated ${tsFile} from ${jsFile}`);
+});
+
+console.log('Worklet generation complete!');

package/worklets/audioConcatProcessor.js

@@ -0,0 +1,92 @@
+/*
+ * ulaw decoding logic taken from the wavefile library
+ * https://github.com/rochars/wavefile/blob/master/lib/codecs/mulaw.js
+ * USED BY @elevenlabs/client
+ */
+
+const decodeTable = [0,132,396,924,1980,4092,8316,16764];
+
+function decodeSample(muLawSample) {
+  let sign;
+  let exponent;
+  let mantissa;
+  let sample;
+  muLawSample = ~muLawSample;
+  sign = (muLawSample & 0x80);
+  exponent = (muLawSample >> 4) & 0x07;
+  mantissa = muLawSample & 0x0F;
+  sample = decodeTable[exponent] + (mantissa << (exponent+3));
+  if (sign !== 0) sample = -sample;
+
+  return sample;
+}
+
+class AudioConcatProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.buffers = []; // Initialize an empty buffer
+    this.cursor = 0;
+    this.currentBuffer = null;
+    this.wasInterrupted = false;
+    this.finished = false;
+    
+    this.port.onmessage = ({ data }) => {
+      switch (data.type) {
+        case "setFormat":
+          this.format = data.format;
+          break;
+        case "buffer":
+          this.wasInterrupted = false;
+          this.buffers.push(
+            this.format === "ulaw"
+              ? new Uint8Array(data.buffer)
+              : new Int16Array(data.buffer)
+          );
+          break;
+        case "interrupt":
+          this.wasInterrupted = true;
+          break;
+        case "clearInterrupted":
+          if (this.wasInterrupted) {
+            this.wasInterrupted = false;
+            this.buffers = [];
+            this.currentBuffer = null;
+          }
+      }
+    };
+  }
+  process(_, outputs) {
+    let finished = false;
+    const output = outputs[0][0];
+    for (let i = 0; i < output.length; i++) {
+      if (!this.currentBuffer) {
+        if (this.buffers.length === 0) {
+          finished = true;
+          break;
+        }
+        this.currentBuffer = this.buffers.shift();
+        this.cursor = 0;
+      }
+
+      let value = this.currentBuffer[this.cursor];
+      if (this.format === "ulaw") {
+        value = decodeSample(value);
+      }
+      output[i] = value / 32768;
+      this.cursor++;
+
+      if (this.cursor >= this.currentBuffer.length) {
+        this.currentBuffer = null;
+      }
+    }
+
+    if (this.finished !== finished) {
+      this.finished = finished;
+      this.port.postMessage({ type: "process", finished });
+    }
+
+    return true; // Continue processing
+  }
+}
+
+registerProcessor("audioConcatProcessor", AudioConcatProcessor);

package/worklets/rawAudioProcessor.js

@@ -0,0 +1,124 @@
+/*
+ * ulaw encoding logic taken from the wavefile library
+ * https://github.com/rochars/wavefile/blob/master/lib/codecs/mulaw.js
+ * USED BY @elevenlabs/client
+ */
+
+const BIAS = 0x84;
+const CLIP = 32635;
+const encodeTable = [
+  0,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,
+  4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
+  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
+  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7
+];
+
+function encodeSample(sample) {
+  let sign;
+  let exponent;
+  let mantissa;
+  let muLawSample;
+  sign = (sample >> 8) & 0x80;
+  if (sign !== 0) sample = -sample;
+  sample = sample + BIAS;
+  if (sample > CLIP) sample = CLIP;
+  exponent = encodeTable[(sample>>7) & 0xFF];
+  mantissa = (sample >> (exponent+3)) & 0x0F;
+  muLawSample = ~(sign | (exponent << 4) | mantissa);
+  
+  return muLawSample;
+}
+
+class RawAudioProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+              
+    this.port.onmessage = ({ data }) => {
+      switch (data.type) {
+        case "setFormat":
+          this.isMuted = false;
+          this.buffer = []; // Initialize an empty buffer
+          this.bufferSize = data.sampleRate / 4;
+          this.format = data.format;
+
+          if (globalThis.LibSampleRate && sampleRate !== data.sampleRate) {
+            globalThis.LibSampleRate.create(1, sampleRate, data.sampleRate).then(resampler => {
+              this.resampler = resampler;
+            });
+          }
+          break;
+        case "setMuted":
+          this.isMuted = data.isMuted;
+          break;
+      }
+    };
+  }
+  process(inputs) {
+    if (!this.buffer) {
+      return true;
+    }
+    
+    const input = inputs[0]; // Get the first input node
+    if (input.length > 0) {
+      let channelData = input[0]; // Get the first channel's data
+
+      // Resample the audio if necessary
+      if (this.resampler) {
+        channelData = this.resampler.full(channelData);
+      }
+
+      // Add channel data to the buffer
+      this.buffer.push(...channelData);
+      // Get max volume 
+      let sum = 0.0;
+      for (let i = 0; i < channelData.length; i++) {
+        sum += channelData[i] * channelData[i];
+      }
+      const maxVolume = Math.sqrt(sum / channelData.length);
+      // Check if buffer size has reached or exceeded the threshold
+      if (this.buffer.length >= this.bufferSize) {
+        const float32Array = this.isMuted 
+          ? new Float32Array(this.buffer.length)
+          : new Float32Array(this.buffer);
+
+        let encodedArray = this.format === "ulaw"
+          ? new Uint8Array(float32Array.length)
+          : new Int16Array(float32Array.length);
+
+        // Iterate through the Float32Array and convert each sample to PCM16
+        for (let i = 0; i < float32Array.length; i++) {
+          // Clamp the value to the range [-1, 1]
+          let sample = Math.max(-1, Math.min(1, float32Array[i]));
+
+          // Scale the sample to the range [-32768, 32767]
+          let value = sample < 0 ? sample * 32768 : sample * 32767;
+          if (this.format === "ulaw") {
+            value = encodeSample(Math.round(value));
+          }
+
+          encodedArray[i] = value;
+        }
+
+        // Send the buffered data to the main script
+        this.port.postMessage([encodedArray, maxVolume]);
+
+        // Clear the buffer after sending
+        this.buffer = [];
+      }
+    }
+    return true; // Continue processing
+  }
+}
+registerProcessor("rawAudioProcessor", RawAudioProcessor);