@tanstack/ai-elevenlabs 0.1.3 → 0.1.5

package/dist/esm/realtime/adapter.js

@@ -27,18 +27,14 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
   const elevenLabsClientTools = {};
   if (clientToolDefs) {
     for (const tool of clientToolDefs) {
-      elevenLabsClientTools[tool.name] = {
-        handler: async (params) => {
-          if (tool.execute) {
-            const result = await tool.execute(params);
-            return typeof result === "string" ? result : JSON.stringify(result);
-          }
-          return JSON.stringify({
-            error: `No execute function for tool ${tool.name}`
-          });
-        },
-        description: tool.description,
-        parameters: tool.inputSchema ? tool.inputSchema : { type: "object", properties: {} }
+      elevenLabsClientTools[tool.name] = async (params) => {
+        if (tool.execute) {
+          const result = await tool.execute(params);
+          return typeof result === "string" ? result : JSON.stringify(result);
+        }
+        return JSON.stringify({
+          error: `No execute function for tool ${tool.name}`
+        });
       };
     }
   }
@@ -58,18 +54,21 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
     },
     onMessage: ({ message, source }) => {
       const role = source === "user" ? "user" : "assistant";
-      emit("transcript", {
-        role,
-        transcript: message,
-        isFinal: true
-      });
-      const realtimeMessage = {
-        id: generateMessageId(),
-        role,
-        timestamp: Date.now(),
-        parts: [{ type: "audio", transcript: message }]
-      };
-      emit("message_complete", { message: realtimeMessage });
+      if (role === "user") {
+        emit("transcript", {
+          role,
+          transcript: message,
+          isFinal: true
+        });
+      } else {
+        const realtimeMessage = {
+          id: generateMessageId(),
+          role,
+          timestamp: Date.now(),
+          parts: [{ type: "audio", transcript: message }]
+        };
+        emit("message_complete", { message: realtimeMessage });
+      }
     },
     onError: (error) => {
       emit("error", {

package/dist/esm/realtime/adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@11labs/client'\nimport type {\n  AnyClientTool,\n  AudioVisualization,\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeMessage,\n  RealtimeMode,\n  RealtimeSessionConfig,\n  RealtimeStatus,\n  RealtimeToken,\n} from '@tanstack/ai'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @11labs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n *   getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n *   adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n  options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n  return {\n    provider: 'elevenlabs',\n\n    async connect(\n      token: RealtimeToken,\n      clientToolDefs?: ReadonlyArray<AnyClientTool>,\n    ): Promise<RealtimeConnection> {\n      return createElevenLabsConnection(token, options, clientToolDefs)\n    },\n  }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n  token: RealtimeToken,\n  _options: ElevenLabsRealtimeOptions,\n  clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n  const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n  let conversation: Awaited<\n    ReturnType<typeof Conversation.startSession>\n  > | null = null\n  let messageIdCounter = 0\n\n  // Empty arrays for when visualization isn't available\n  const emptyFrequencyData = new Uint8Array(128)\n  const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n  // Helper to emit events\n  function emit<TEvent extends RealtimeEvent>(\n    event: TEvent,\n    payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n  ) {\n    const handlers = eventHandlers.get(event)\n    if (handlers) {\n      for (const handler of handlers) {\n        handler(payload)\n      }\n    }\n  }\n\n  function generateMessageId(): string {\n    return `el-msg-${Date.now()}-${++messageIdCounter}`\n  }\n\n  // Convert TanStack tool definitions to ElevenLabs clientTools format\n  const elevenLabsClientTools: Record<\n    string,\n    {\n      handler: (params: unknown) => Promise<string>\n      description: string\n      parameters: Record<string, unknown>\n    }\n  > = {}\n\n  if (clientToolDefs) {\n    for (const tool of clientToolDefs) {\n      elevenLabsClientTools[tool.name] = {\n        handler: async (params: unknown) => {\n          if (tool.execute) {\n            const result = await tool.execute(params)\n            return typeof result === 'string' ? result : JSON.stringify(result)\n          }\n          return JSON.stringify({\n            error: `No execute function for tool ${tool.name}`,\n          })\n        },\n        description: tool.description,\n        parameters: tool.inputSchema\n          ? (tool.inputSchema as Record<string, unknown>)\n          : { type: 'object', properties: {} },\n      }\n    }\n  }\n\n  // Build session options\n  const sessionOptions: Record<string, unknown> = {\n    signedUrl: token.token,\n\n    onConnect: () => {\n      emit('status_change', { status: 'connected' as RealtimeStatus })\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    onDisconnect: () => {\n      emit('status_change', { status: 'idle' as RealtimeStatus })\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    onModeChange: ({ mode }: { mode: string }) => {\n      const mappedMode: RealtimeMode =\n        mode === 'speaking' ? 'speaking' : 'listening'\n      emit('mode_change', { mode: mappedMode })\n    },\n\n    onMessage: ({ message, source }: { message: string; source: string }) => {\n      const role = source === 'user' ? 'user' : 'assistant'\n\n      // Emit transcript update\n      emit('transcript', {\n        role,\n        transcript: message,\n        isFinal: true,\n      })\n\n      // Create and emit message\n      const realtimeMessage: RealtimeMessage = {\n        id: generateMessageId(),\n        role,\n        timestamp: Date.now(),\n        parts: [{ type: 'audio', transcript: message }],\n      }\n      emit('message_complete', { message: realtimeMessage })\n    },\n\n    onError: (error: string | Error) => {\n      emit('error', {\n        error: new Error(\n          typeof error === 'string' ? error : error.message || 'Unknown error',\n        ),\n      })\n    },\n  }\n\n  // Only add clientTools if we have any\n  if (Object.keys(elevenLabsClientTools).length > 0) {\n    sessionOptions.clientTools = elevenLabsClientTools\n  }\n\n  // Start the conversation session\n  conversation = await Conversation.startSession(\n    sessionOptions as Parameters<typeof Conversation.startSession>[0],\n  )\n\n  // Connection implementation\n  const connection: RealtimeConnection = {\n    async disconnect() {\n      if (conversation) {\n        await conversation.endSession()\n        conversation = null\n      }\n      emit('status_change', { status: 'idle' as RealtimeStatus })\n    },\n\n    async startAudioCapture() {\n      // ElevenLabs SDK handles audio capture automatically\n      // This is called when the session starts\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    stopAudioCapture() {\n      // ElevenLabs SDK handles this\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    sendText(text: string) {\n      if (!conversation) return\n      conversation.sendUserMessage(text)\n    },\n\n    sendImage(_imageData: string, _mimeType: string) {\n      // ElevenLabs does not support direct image input in the conversation API\n      console.warn(\n        'ElevenLabs realtime does not support sending images directly.',\n      )\n    },\n\n    sendToolResult(_callId: string, _result: string) {\n      // ElevenLabs client tools are handled via the clientTools handlers\n      // registered at session start — results are returned automatically\n    },\n\n    updateSession(_config: Partial<RealtimeSessionConfig>) {\n      // ElevenLabs session config is set at creation time\n      console.warn(\n        'ElevenLabs does not support runtime session updates. Configure at connection time.',\n      )\n    },\n\n    interrupt() {\n      // ElevenLabs handles interruption automatically via barge-in\n      // No explicit API to call\n      emit('mode_change', { mode: 'listening' })\n      emit('interrupted', {})\n    },\n\n    on<TEvent extends RealtimeEvent>(\n      event: TEvent,\n      handler: RealtimeEventHandler<TEvent>,\n    ): () => void {\n      if (!eventHandlers.has(event)) {\n        eventHandlers.set(event, new Set())\n      }\n      eventHandlers.get(event)!.add(handler)\n\n      return () => {\n        eventHandlers.get(event)?.delete(handler)\n      }\n    },\n\n    getAudioVisualization(): AudioVisualization {\n      return {\n        get inputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getInputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        get outputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getOutputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        getInputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getInputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getOutputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getOutputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getInputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        getOutputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        get inputSampleRate() {\n          return 16000\n        },\n\n        get outputSampleRate() {\n          return 16000\n        },\n      }\n    },\n  }\n\n  return connection\n}\n"],"names":[],"mappings":";AAkCO,SAAS,mBACd,UAAqC,IACpB;AACjB,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,2BAA2B,OAAO,SAAS,cAAc;AAAA,IAClE;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAGA,QAAM,wBAOF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI;AAAA,QACjC,SAAS,OAAO,WAAoB;AAClC,cAAI,KAAK,SAAS;AAChB,kBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,mBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,UACpE;AACA,iBAAO,KAAK,UAAU;AAAA,YACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,UAAA,CACjD;AAAA,QACH;AAAA,QACA,aAAa,KAAK;AAAA,QAClB,YAAY,KAAK,cACZ,KAAK,cACN,EAAE,MAAM,UAAU,YAAY,CAAA,EAAC;AAAA,MAAE;AAAA,IAEzC;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAC1D,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,YAAM,OAAO,WAAW,SAAS,SAAS;AAG1C,WAAK,cAAc;AAAA,QACjB;AAAA,QACA,YAAY;AAAA,QACZ,SAAS;AAAA,MAAA,CACV;AAGD,YAAM,kBAAmC;AAAA,QACvC,IAAI,kBAAA;AAAA,QACJ;AAAA,QACA,WAAW,KAAK,IAAA;AAAA,QAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,MAAA;AAEhD,WAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,IACvD;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,iBAAe,MAAM,aAAa;AAAA,IAChC;AAAA,EAAA;AAIF,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}
+{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@11labs/client'\nimport type {\n  AnyClientTool,\n  AudioVisualization,\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeMessage,\n  RealtimeMode,\n  RealtimeSessionConfig,\n  RealtimeStatus,\n  RealtimeToken,\n} from '@tanstack/ai'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @11labs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n *   getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n *   adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n  options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n  return {\n    provider: 'elevenlabs',\n\n    async connect(\n      token: RealtimeToken,\n      clientToolDefs?: ReadonlyArray<AnyClientTool>,\n    ): Promise<RealtimeConnection> {\n      return createElevenLabsConnection(token, options, clientToolDefs)\n    },\n  }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n  token: RealtimeToken,\n  _options: ElevenLabsRealtimeOptions,\n  clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n  const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n  let conversation: Awaited<\n    ReturnType<typeof Conversation.startSession>\n  > | null = null\n  let messageIdCounter = 0\n\n  // Empty arrays for when visualization isn't available\n  const emptyFrequencyData = new Uint8Array(128)\n  const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n  // Helper to emit events\n  function emit<TEvent extends RealtimeEvent>(\n    event: TEvent,\n    payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n  ) {\n    const handlers = eventHandlers.get(event)\n    if (handlers) {\n      for (const handler of handlers) {\n        handler(payload)\n      }\n    }\n  }\n\n  function generateMessageId(): string {\n    return `el-msg-${Date.now()}-${++messageIdCounter}`\n  }\n\n  // Convert TanStack tool definitions to ElevenLabs clientTools format.\n  // @11labs/client@0.2.0 expects plain async functions, not objects.\n  const elevenLabsClientTools: Record<\n    string,\n    (params: unknown) => Promise<string>\n  > = {}\n\n  if (clientToolDefs) {\n    for (const tool of clientToolDefs) {\n      elevenLabsClientTools[tool.name] = async (params: unknown) => {\n        if (tool.execute) {\n          const result = await tool.execute(params)\n          return typeof result === 'string' ? result : JSON.stringify(result)\n        }\n        return JSON.stringify({\n          error: `No execute function for tool ${tool.name}`,\n        })\n      }\n    }\n  }\n\n  // Build session options\n  const sessionOptions: Record<string, unknown> = {\n    signedUrl: token.token,\n\n    onConnect: () => {\n      emit('status_change', { status: 'connected' as RealtimeStatus })\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    onDisconnect: () => {\n      emit('status_change', { status: 'idle' as RealtimeStatus })\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    onModeChange: ({ mode }: { mode: string }) => {\n      const mappedMode: RealtimeMode =\n        mode === 'speaking' ? 'speaking' : 'listening'\n      emit('mode_change', { mode: mappedMode })\n    },\n\n    onMessage: ({ message, source }: { message: string; source: string }) => {\n      const role = source === 'user' ? 'user' : 'assistant'\n\n      if (role === 'user') {\n        // User transcripts: only emit transcript — the RealtimeClient\n        // creates the user message when it receives a final transcript\n        emit('transcript', {\n          role,\n          transcript: message,\n          isFinal: true,\n        })\n      } else {\n        // Assistant messages: only emit message_complete — the canonical\n        // event for finalized assistant messages\n        const realtimeMessage: RealtimeMessage = {\n          id: generateMessageId(),\n          role,\n          timestamp: Date.now(),\n          parts: [{ type: 'audio', transcript: message }],\n        }\n        emit('message_complete', { message: realtimeMessage })\n      }\n    },\n\n    onError: (error: string | Error) => {\n      emit('error', {\n        error: new Error(\n          typeof error === 'string' ? error : error.message || 'Unknown error',\n        ),\n      })\n    },\n  }\n\n  // Only add clientTools if we have any\n  if (Object.keys(elevenLabsClientTools).length > 0) {\n    sessionOptions.clientTools = elevenLabsClientTools\n  }\n\n  // Start the conversation session\n  conversation = await Conversation.startSession(\n    sessionOptions as Parameters<typeof Conversation.startSession>[0],\n  )\n\n  // Connection implementation\n  const connection: RealtimeConnection = {\n    async disconnect() {\n      if (conversation) {\n        await conversation.endSession()\n        conversation = null\n      }\n      emit('status_change', { status: 'idle' as RealtimeStatus })\n    },\n\n    async startAudioCapture() {\n      // ElevenLabs SDK handles audio capture automatically\n      // This is called when the session starts\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    stopAudioCapture() {\n      // ElevenLabs SDK handles this\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    sendText(text: string) {\n      if (!conversation) return\n      conversation.sendUserMessage(text)\n    },\n\n    sendImage(_imageData: string, _mimeType: string) {\n      // ElevenLabs does not support direct image input in the conversation API\n      console.warn(\n        'ElevenLabs realtime does not support sending images directly.',\n      )\n    },\n\n    sendToolResult(_callId: string, _result: string) {\n      // ElevenLabs client tools are handled via the clientTools handlers\n      // registered at session start — results are returned automatically\n    },\n\n    updateSession(_config: Partial<RealtimeSessionConfig>) {\n      // ElevenLabs session config is set at creation time\n      console.warn(\n        'ElevenLabs does not support runtime session updates. Configure at connection time.',\n      )\n    },\n\n    interrupt() {\n      // ElevenLabs handles interruption automatically via barge-in\n      // No explicit API to call\n      emit('mode_change', { mode: 'listening' })\n      emit('interrupted', {})\n    },\n\n    on<TEvent extends RealtimeEvent>(\n      event: TEvent,\n      handler: RealtimeEventHandler<TEvent>,\n    ): () => void {\n      if (!eventHandlers.has(event)) {\n        eventHandlers.set(event, new Set())\n      }\n      eventHandlers.get(event)!.add(handler)\n\n      return () => {\n        eventHandlers.get(event)?.delete(handler)\n      }\n    },\n\n    getAudioVisualization(): AudioVisualization {\n      return {\n        get inputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getInputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        get outputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getOutputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        getInputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getInputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getOutputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getOutputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getInputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        getOutputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        get inputSampleRate() {\n          return 16000\n        },\n\n        get outputSampleRate() {\n          return 16000\n        },\n      }\n    },\n  }\n\n  return connection\n}\n"],"names":[],"mappings":";AAkCO,SAAS,mBACd,UAAqC,IACpB;AACjB,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,2BAA2B,OAAO,SAAS,cAAc;AAAA,IAClE;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAC1D,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,iBAAe,MAAM,aAAa;AAAA,IAChC;AAAA,EAAA;AAIF,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai-elevenlabs",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "ElevenLabs adapter for TanStack AI realtime voice",
   "author": "",
   "license": "MIT",
@@ -34,13 +34,13 @@
     "@11labs/client": "^0.2.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "^0.10.0",
-    "@tanstack/ai-client": "^0.7.7"
+    "@tanstack/ai": "^0.11.0",
+    "@tanstack/ai-client": "^0.7.11"
   },
   "devDependencies": {
     "@vitest/coverage-v8": "4.0.14",
-    "@tanstack/ai": "0.10.0",
-    "@tanstack/ai-client": "0.7.7"
+    "@tanstack/ai": "0.11.0",
+    "@tanstack/ai-client": "0.7.11"
   },
   "scripts": {
     "build": "vite build",

package/src/realtime/adapter.ts

@@ -82,32 +82,23 @@ async function createElevenLabsConnection(
     return `el-msg-${Date.now()}-${++messageIdCounter}`
   }
 
-  // Convert TanStack tool definitions to ElevenLabs clientTools format
+  // Convert TanStack tool definitions to ElevenLabs clientTools format.
+  // @11labs/client@0.2.0 expects plain async functions, not objects.
   const elevenLabsClientTools: Record<
     string,
-    {
-      handler: (params: unknown) => Promise<string>
-      description: string
-      parameters: Record<string, unknown>
-    }
+    (params: unknown) => Promise<string>
   > = {}
 
   if (clientToolDefs) {
     for (const tool of clientToolDefs) {
-      elevenLabsClientTools[tool.name] = {
-        handler: async (params: unknown) => {
-          if (tool.execute) {
-            const result = await tool.execute(params)
-            return typeof result === 'string' ? result : JSON.stringify(result)
-          }
-          return JSON.stringify({
-            error: `No execute function for tool ${tool.name}`,
-          })
-        },
-        description: tool.description,
-        parameters: tool.inputSchema
-          ? (tool.inputSchema as Record<string, unknown>)
-          : { type: 'object', properties: {} },
+      elevenLabsClientTools[tool.name] = async (params: unknown) => {
+        if (tool.execute) {
+          const result = await tool.execute(params)
+          return typeof result === 'string' ? result : JSON.stringify(result)
+        }
+        return JSON.stringify({
+          error: `No execute function for tool ${tool.name}`,
+        })
       }
     }
   }
@@ -135,21 +126,25 @@ async function createElevenLabsConnection(
     onMessage: ({ message, source }: { message: string; source: string }) => {
       const role = source === 'user' ? 'user' : 'assistant'
 
-      // Emit transcript update
-      emit('transcript', {
-        role,
-        transcript: message,
-        isFinal: true,
-      })
-
-      // Create and emit message
-      const realtimeMessage: RealtimeMessage = {
-        id: generateMessageId(),
-        role,
-        timestamp: Date.now(),
-        parts: [{ type: 'audio', transcript: message }],
+      if (role === 'user') {
+        // User transcripts: only emit transcript — the RealtimeClient
+        // creates the user message when it receives a final transcript
+        emit('transcript', {
+          role,
+          transcript: message,
+          isFinal: true,
+        })
+      } else {
+        // Assistant messages: only emit message_complete — the canonical
+        // event for finalized assistant messages
+        const realtimeMessage: RealtimeMessage = {
+          id: generateMessageId(),
+          role,
+          timestamp: Date.now(),
+          parts: [{ type: 'audio', transcript: message }],
+        }
+        emit('message_complete', { message: realtimeMessage })
       }
-      emit('message_complete', { message: realtimeMessage })
     },
 
     onError: (error: string | Error) => {