@tanstack/ai-elevenlabs 0.2.25 → 0.2.27

package/dist/esm/realtime/adapter.d.ts

@@ -1,4 +1,4 @@
-import { RealtimeAdapter } from '@tanstack/ai-client';
+import { RealtimeAdapter } from '@tanstack/ai';
 import { ElevenLabsRealtimeOptions } from './types.js';
 /**
  * Creates an ElevenLabs realtime adapter for client-side use.

package/dist/esm/realtime/adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@elevenlabs/client'\nimport { resolveDebugOption } from '@tanstack/ai/adapter-internals'\nimport type {\n  AnyClientTool,\n  AudioVisualization,\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeMessage,\n  RealtimeMode,\n  RealtimeSessionConfig,\n  RealtimeToken,\n} from '@tanstack/ai'\nimport type { InternalLogger } from '@tanstack/ai/adapter-internals'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @elevenlabs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n *   getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n *   adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n  options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n  const logger = resolveDebugOption(options.debug)\n\n  return {\n    provider: 'elevenlabs',\n\n    async connect(\n      token: RealtimeToken,\n      clientToolDefs?: ReadonlyArray<AnyClientTool>,\n    ): Promise<RealtimeConnection> {\n      logger.request(`activity=realtime provider=elevenlabs`, {\n        provider: 'elevenlabs',\n      })\n      return createElevenLabsConnection(token, options, logger, clientToolDefs)\n    },\n  }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n  token: RealtimeToken,\n  _options: ElevenLabsRealtimeOptions,\n  logger: InternalLogger,\n  clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n  const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n  let conversation: Awaited<\n    ReturnType<typeof Conversation.startSession>\n  > | null = null\n  let messageIdCounter = 0\n\n  // Empty arrays for when visualization isn't available\n  const emptyFrequencyData = new Uint8Array(128)\n  const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n  // Helper to emit events\n  function emit<TEvent extends RealtimeEvent>(\n    event: TEvent,\n    payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n  ) {\n    const handlers = eventHandlers.get(event)\n    if (handlers) {\n      for (const handler of handlers) {\n        handler(payload)\n      }\n    }\n  }\n\n  function generateMessageId(): string {\n    return `el-msg-${Date.now()}-${++messageIdCounter}`\n  }\n\n  // Convert TanStack tool definitions to ElevenLabs clientTools format.\n  // @elevenlabs/client expects plain async functions, not objects.\n  const elevenLabsClientTools: Record<\n    string,\n    (params: unknown) => Promise<string>\n  > = {}\n\n  if (clientToolDefs) {\n    for (const tool of clientToolDefs) {\n      elevenLabsClientTools[tool.name] = async (params: unknown) => {\n        if (tool.execute) {\n          const result = await tool.execute(params)\n          return typeof result === 'string' ? result : JSON.stringify(result)\n        }\n        return JSON.stringify({\n          error: `No execute function for tool ${tool.name}`,\n        })\n      }\n    }\n  }\n\n  // Build session options\n  const sessionOptions: Record<string, unknown> = {\n    signedUrl: token.token,\n\n    onConnect: () => {\n      logger.provider(`provider=elevenlabs direction=in type=connect`, {\n        frame: { type: 'connect' },\n      })\n      emit('status_change', { status: 'connected' })\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    onDisconnect: () => {\n      logger.provider(`provider=elevenlabs direction=in type=disconnect`, {\n        frame: { type: 'disconnect' },\n      })\n      emit('status_change', { status: 'idle' })\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    onModeChange: ({ mode }: { mode: string }) => {\n      logger.provider(`provider=elevenlabs direction=in type=mode_change`, {\n        frame: { type: 'mode_change', mode },\n      })\n      const mappedMode: RealtimeMode =\n        mode === 'speaking' ? 'speaking' : 'listening'\n      emit('mode_change', { mode: mappedMode })\n    },\n\n    onMessage: ({ message, source }: { message: string; source: string }) => {\n      logger.provider(`provider=elevenlabs direction=in type=message`, {\n        frame: { type: 'message', source, message },\n      })\n      const role = source === 'user' ? 'user' : 'assistant'\n\n      if (role === 'user') {\n        // User transcripts: only emit transcript — the RealtimeClient\n        // creates the user message when it receives a final transcript\n        emit('transcript', {\n          role,\n          transcript: message,\n          isFinal: true,\n        })\n      } else {\n        // Assistant messages: only emit message_complete — the canonical\n        // event for finalized assistant messages\n        const realtimeMessage: RealtimeMessage = {\n          id: generateMessageId(),\n          role,\n          timestamp: Date.now(),\n          parts: [{ type: 'audio', transcript: message }],\n        }\n        emit('message_complete', { message: realtimeMessage })\n      }\n    },\n\n    onError: (error: string | Error) => {\n      logger.errors('elevenlabs.realtime fatal', {\n        error,\n        source: 'elevenlabs.realtime',\n      })\n      emit('error', {\n        error: new Error(\n          typeof error === 'string' ? error : error.message || 'Unknown error',\n        ),\n      })\n    },\n  }\n\n  // Only add clientTools if we have any\n  if (Object.keys(elevenLabsClientTools).length > 0) {\n    sessionOptions.clientTools = elevenLabsClientTools\n  }\n\n  // Start the conversation session\n  try {\n    conversation = await Conversation.startSession(\n      sessionOptions as Parameters<typeof Conversation.startSession>[0],\n    )\n  } catch (error) {\n    logger.errors('elevenlabs.realtime fatal', {\n      error,\n      source: 'elevenlabs.realtime',\n    })\n    throw error\n  }\n\n  // Connection implementation\n  const connection: RealtimeConnection = {\n    async disconnect() {\n      if (conversation) {\n        await conversation.endSession()\n        conversation = null\n      }\n      emit('status_change', { status: 'idle' })\n    },\n\n    async startAudioCapture() {\n      // ElevenLabs SDK handles audio capture automatically\n      // This is called when the session starts\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    stopAudioCapture() {\n      // ElevenLabs SDK handles this\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    sendText(text: string) {\n      if (!conversation) return\n      logger.provider(`provider=elevenlabs direction=out type=user_message`, {\n        frame: { type: 'user_message', text },\n      })\n      conversation.sendUserMessage(text)\n    },\n\n    sendImage(_imageData: string, _mimeType: string) {\n      // ElevenLabs does not support direct image input in the conversation API\n      logger.errors('elevenlabs.realtime sendImage not supported', {\n        source: 'elevenlabs.realtime',\n      })\n    },\n\n    sendToolResult(_callId: string, _result: string) {\n      // ElevenLabs client tools are handled via the clientTools handlers\n      // registered at session start — results are returned automatically\n    },\n\n    updateSession(_config: Partial<RealtimeSessionConfig>) {\n      // ElevenLabs session config is set at creation time\n      logger.errors('elevenlabs.realtime updateSession not supported', {\n        source: 'elevenlabs.realtime',\n      })\n    },\n\n    interrupt() {\n      // ElevenLabs handles interruption automatically via barge-in\n      // No explicit API to call\n      emit('mode_change', { mode: 'listening' })\n      emit('interrupted', {})\n    },\n\n    on<TEvent extends RealtimeEvent>(\n      event: TEvent,\n      handler: RealtimeEventHandler<TEvent>,\n    ): () => void {\n      let handlers = eventHandlers.get(event)\n      if (!handlers) {\n        handlers = new Set()\n        eventHandlers.set(event, handlers)\n      }\n      handlers.add(handler)\n\n      return () => {\n        eventHandlers.get(event)?.delete(handler)\n      }\n    },\n\n    getAudioVisualization(): AudioVisualization {\n      return {\n        get inputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getInputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        get outputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getOutputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        getInputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getInputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getOutputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getOutputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getInputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        getOutputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        get inputSampleRate() {\n          return 16000\n        },\n\n        get outputSampleRate() {\n          return 16000\n        },\n      }\n    },\n  }\n\n  return connection\n}\n"],"names":[],"mappings":";;AAmCO,SAAS,mBACd,UAAqC,IACpB;AACjB,QAAM,SAAS,mBAAmB,QAAQ,KAAK;AAE/C,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,QAAQ,yCAAyC;AAAA,QACtD,UAAU;AAAA,MAAA,CACX;AACD,aAAO,2BAA2B,OAAO,SAAS,QAAQ,cAAc;AAAA,IAC1E;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,QACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,UAAA;AAAA,MAAU,CAC1B;AACD,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAAa;AAC7C,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,aAAO,SAAS,oDAAoD;AAAA,QAClE,OAAO,EAAE,MAAM,aAAA;AAAA,MAAa,CAC7B;AACD,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAAQ;AACxC,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,aAAO,SAAS,qDAAqD;AAAA,QACnE,OAAO,EAAE,MAAM,eAAe,KAAA;AAAA,MAAK,CACpC;AACD,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,WAAW,QAAQ,QAAA;AAAA,MAAQ,CAC3C;AACD,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,aAAO,OAAO,6BAA6B;AAAA,QACzC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,MAAI;AACF,mBAAe,MAAM,aAAa;AAAA,MAChC;AAAA,IAAA;AAAA,EAEJ,SAAS,OAAO;AACd,WAAO,OAAO,6BAA6B;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AACD,UAAM;AAAA,EACR;AAGA,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAAQ;AAAA,IAC1C;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,aAAO,SAAS,uDAAuD;AAAA,QACrE,OAAO,EAAE,MAAM,gBAAgB,KAAA;AAAA,MAAK,CACrC;AACD,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,aAAO,OAAO,+CAA+C;AAAA,QAC3D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,aAAO,OAAO,mDAAmD;AAAA,QAC/D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,WAAW,cAAc,IAAI,KAAK;AACtC,UAAI,CAAC,UAAU;AACb,uCAAe,IAAA;AACf,sBAAc,IAAI,OAAO,QAAQ;AAAA,MACnC;AACA,eAAS,IAAI,OAAO;AAEpB,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}
+{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@elevenlabs/client'\nimport { resolveDebugOption } from '@tanstack/ai/adapter-internals'\nimport type {\n  AnyClientTool,\n  AudioVisualization,\n  RealtimeAdapter,\n  RealtimeConnection,\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeMessage,\n  RealtimeMode,\n  RealtimeSessionConfig,\n  RealtimeToken,\n} from '@tanstack/ai'\nimport type { InternalLogger } from '@tanstack/ai/adapter-internals'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @elevenlabs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n *   getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n *   adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n  options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n  const logger = resolveDebugOption(options.debug)\n\n  return {\n    provider: 'elevenlabs',\n\n    async connect(\n      token: RealtimeToken,\n      clientToolDefs?: ReadonlyArray<AnyClientTool>,\n    ): Promise<RealtimeConnection> {\n      logger.request(`activity=realtime provider=elevenlabs`, {\n        provider: 'elevenlabs',\n      })\n      return createElevenLabsConnection(token, options, logger, clientToolDefs)\n    },\n  }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n  token: RealtimeToken,\n  _options: ElevenLabsRealtimeOptions,\n  logger: InternalLogger,\n  clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n  const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n  let conversation: Awaited<\n    ReturnType<typeof Conversation.startSession>\n  > | null = null\n  let messageIdCounter = 0\n\n  // Empty arrays for when visualization isn't available\n  const emptyFrequencyData = new Uint8Array(128)\n  const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n  // Helper to emit events\n  function emit<TEvent extends RealtimeEvent>(\n    event: TEvent,\n    payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n  ) {\n    const handlers = eventHandlers.get(event)\n    if (handlers) {\n      for (const handler of handlers) {\n        handler(payload)\n      }\n    }\n  }\n\n  function generateMessageId(): string {\n    return `el-msg-${Date.now()}-${++messageIdCounter}`\n  }\n\n  // Convert TanStack tool definitions to ElevenLabs clientTools format.\n  // @elevenlabs/client expects plain async functions, not objects.\n  const elevenLabsClientTools: Record<\n    string,\n    (params: unknown) => Promise<string>\n  > = {}\n\n  if (clientToolDefs) {\n    for (const tool of clientToolDefs) {\n      elevenLabsClientTools[tool.name] = async (params: unknown) => {\n        if (tool.execute) {\n          const result = await tool.execute(params)\n          return typeof result === 'string' ? result : JSON.stringify(result)\n        }\n        return JSON.stringify({\n          error: `No execute function for tool ${tool.name}`,\n        })\n      }\n    }\n  }\n\n  // Build session options\n  const sessionOptions: Record<string, unknown> = {\n    signedUrl: token.token,\n\n    onConnect: () => {\n      logger.provider(`provider=elevenlabs direction=in type=connect`, {\n        frame: { type: 'connect' },\n      })\n      emit('status_change', { status: 'connected' })\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    onDisconnect: () => {\n      logger.provider(`provider=elevenlabs direction=in type=disconnect`, {\n        frame: { type: 'disconnect' },\n      })\n      emit('status_change', { status: 'idle' })\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    onModeChange: ({ mode }: { mode: string }) => {\n      logger.provider(`provider=elevenlabs direction=in type=mode_change`, {\n        frame: { type: 'mode_change', mode },\n      })\n      const mappedMode: RealtimeMode =\n        mode === 'speaking' ? 'speaking' : 'listening'\n      emit('mode_change', { mode: mappedMode })\n    },\n\n    onMessage: ({ message, source }: { message: string; source: string }) => {\n      logger.provider(`provider=elevenlabs direction=in type=message`, {\n        frame: { type: 'message', source, message },\n      })\n      const role = source === 'user' ? 'user' : 'assistant'\n\n      if (role === 'user') {\n        // User transcripts: only emit transcript — the RealtimeClient\n        // creates the user message when it receives a final transcript\n        emit('transcript', {\n          role,\n          transcript: message,\n          isFinal: true,\n        })\n      } else {\n        // Assistant messages: only emit message_complete — the canonical\n        // event for finalized assistant messages\n        const realtimeMessage: RealtimeMessage = {\n          id: generateMessageId(),\n          role,\n          timestamp: Date.now(),\n          parts: [{ type: 'audio', transcript: message }],\n        }\n        emit('message_complete', { message: realtimeMessage })\n      }\n    },\n\n    onError: (error: string | Error) => {\n      logger.errors('elevenlabs.realtime fatal', {\n        error,\n        source: 'elevenlabs.realtime',\n      })\n      emit('error', {\n        error: new Error(\n          typeof error === 'string' ? error : error.message || 'Unknown error',\n        ),\n      })\n    },\n  }\n\n  // Only add clientTools if we have any\n  if (Object.keys(elevenLabsClientTools).length > 0) {\n    sessionOptions.clientTools = elevenLabsClientTools\n  }\n\n  // Start the conversation session\n  try {\n    conversation = await Conversation.startSession(\n      sessionOptions as Parameters<typeof Conversation.startSession>[0],\n    )\n  } catch (error) {\n    logger.errors('elevenlabs.realtime fatal', {\n      error,\n      source: 'elevenlabs.realtime',\n    })\n    throw error\n  }\n\n  // Connection implementation\n  const connection: RealtimeConnection = {\n    async disconnect() {\n      if (conversation) {\n        await conversation.endSession()\n        conversation = null\n      }\n      emit('status_change', { status: 'idle' })\n    },\n\n    async startAudioCapture() {\n      // ElevenLabs SDK handles audio capture automatically\n      // This is called when the session starts\n      emit('mode_change', { mode: 'listening' })\n    },\n\n    stopAudioCapture() {\n      // ElevenLabs SDK handles this\n      emit('mode_change', { mode: 'idle' })\n    },\n\n    sendText(text: string) {\n      if (!conversation) return\n      logger.provider(`provider=elevenlabs direction=out type=user_message`, {\n        frame: { type: 'user_message', text },\n      })\n      conversation.sendUserMessage(text)\n    },\n\n    sendImage(_imageData: string, _mimeType: string) {\n      // ElevenLabs does not support direct image input in the conversation API\n      logger.errors('elevenlabs.realtime sendImage not supported', {\n        source: 'elevenlabs.realtime',\n      })\n    },\n\n    sendToolResult(_callId: string, _result: string) {\n      // ElevenLabs client tools are handled via the clientTools handlers\n      // registered at session start — results are returned automatically\n    },\n\n    updateSession(_config: Partial<RealtimeSessionConfig>) {\n      // ElevenLabs session config is set at creation time\n      logger.errors('elevenlabs.realtime updateSession not supported', {\n        source: 'elevenlabs.realtime',\n      })\n    },\n\n    interrupt() {\n      // ElevenLabs handles interruption automatically via barge-in\n      // No explicit API to call\n      emit('mode_change', { mode: 'listening' })\n      emit('interrupted', {})\n    },\n\n    on<TEvent extends RealtimeEvent>(\n      event: TEvent,\n      handler: RealtimeEventHandler<TEvent>,\n    ): () => void {\n      let handlers = eventHandlers.get(event)\n      if (!handlers) {\n        handlers = new Set()\n        eventHandlers.set(event, handlers)\n      }\n      handlers.add(handler)\n\n      return () => {\n        eventHandlers.get(event)?.delete(handler)\n      }\n    },\n\n    getAudioVisualization(): AudioVisualization {\n      return {\n        get inputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getInputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        get outputLevel() {\n          if (!conversation) return 0\n          try {\n            return conversation.getOutputVolume()\n          } catch {\n            return 0\n          }\n        },\n\n        getInputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getInputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getOutputFrequencyData() {\n          if (!conversation) return emptyFrequencyData\n          try {\n            return conversation.getOutputByteFrequencyData()\n          } catch {\n            return emptyFrequencyData\n          }\n        },\n\n        getInputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        getOutputTimeDomainData() {\n          // ElevenLabs SDK doesn't expose time domain data\n          return emptyTimeDomainData\n        },\n\n        get inputSampleRate() {\n          return 16000\n        },\n\n        get outputSampleRate() {\n          return 16000\n        },\n      }\n    },\n  }\n\n  return connection\n}\n"],"names":[],"mappings":";;AAoCO,SAAS,mBACd,UAAqC,IACpB;AACjB,QAAM,SAAS,mBAAmB,QAAQ,KAAK;AAE/C,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,QAAQ,yCAAyC;AAAA,QACtD,UAAU;AAAA,MAAA,CACX;AACD,aAAO,2BAA2B,OAAO,SAAS,QAAQ,cAAc;AAAA,IAC1E;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,QACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,UAAA;AAAA,MAAU,CAC1B;AACD,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAAa;AAC7C,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,aAAO,SAAS,oDAAoD;AAAA,QAClE,OAAO,EAAE,MAAM,aAAA;AAAA,MAAa,CAC7B;AACD,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAAQ;AACxC,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,aAAO,SAAS,qDAAqD;AAAA,QACnE,OAAO,EAAE,MAAM,eAAe,KAAA;AAAA,MAAK,CACpC;AACD,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,WAAW,QAAQ,QAAA;AAAA,MAAQ,CAC3C;AACD,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,aAAO,OAAO,6BAA6B;AAAA,QACzC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,MAAI;AACF,mBAAe,MAAM,aAAa;AAAA,MAChC;AAAA,IAAA;AAAA,EAEJ,SAAS,OAAO;AACd,WAAO,OAAO,6BAA6B;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AACD,UAAM;AAAA,EACR;AAGA,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAAQ;AAAA,IAC1C;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,aAAO,SAAS,uDAAuD;AAAA,QACrE,OAAO,EAAE,MAAM,gBAAgB,KAAA;AAAA,MAAK,CACrC;AACD,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,aAAO,OAAO,+CAA+C;AAAA,QAC3D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,aAAO,OAAO,mDAAmD;AAAA,QAC/D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,WAAW,cAAc,IAAI,KAAK;AACtC,UAAI,CAAC,UAAU;AACb,uCAAe,IAAA;AACf,sBAAc,IAAI,OAAO,QAAQ;AAAA,MACnC;AACA,eAAS,IAAI,OAAO;AAEpB,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai-elevenlabs",
-  "version": "0.2.25",
+  "version": "0.2.27",
   "description": "ElevenLabs adapter for TanStack AI realtime voice, text-to-speech, transcription, music, and sound effects.",
   "author": "Tanner Linsley",
   "license": "MIT",
@@ -50,16 +50,14 @@
   "dependencies": {
     "@elevenlabs/client": "^1.3.1",
     "@elevenlabs/elevenlabs-js": "^2.44.0",
-    "@tanstack/ai-utils": "0.2.2"
+    "@tanstack/ai-utils": "0.3.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "^0.33.0",
-    "@tanstack/ai-client": "^0.18.1"
+    "@tanstack/ai": "^0.35.0"
   },
   "devDependencies": {
     "@vitest/coverage-v8": "4.0.14",
-    "@tanstack/ai": "0.33.0",
-    "@tanstack/ai-client": "0.18.1"
+    "@tanstack/ai": "0.35.0"
   },
   "scripts": {
     "build": "vite build",

package/src/realtime/adapter.ts

@@ -3,6 +3,8 @@ import { resolveDebugOption } from '@tanstack/ai/adapter-internals'
 import type {
   AnyClientTool,
   AudioVisualization,
+  RealtimeAdapter,
+  RealtimeConnection,
   RealtimeEvent,
   RealtimeEventHandler,
   RealtimeMessage,
@@ -11,7 +13,6 @@ import type {
   RealtimeToken,
 } from '@tanstack/ai'
 import type { InternalLogger } from '@tanstack/ai/adapter-internals'
-import type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'
 import type { ElevenLabsRealtimeOptions } from './types'
 
 /**