@tanstack/ai-elevenlabs 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,13 +1,18 @@
1
1
  import { Conversation } from "@11labs/client";
2
+ import { resolveDebugOption } from "@tanstack/ai/adapter-internals";
2
3
  function elevenlabsRealtime(options = {}) {
4
+ const logger = resolveDebugOption(options.debug);
3
5
  return {
4
6
  provider: "elevenlabs",
5
7
  async connect(token, clientToolDefs) {
6
- return createElevenLabsConnection(token, options, clientToolDefs);
8
+ logger.request(`activity=realtime provider=elevenlabs`, {
9
+ provider: "elevenlabs"
10
+ });
11
+ return createElevenLabsConnection(token, options, logger, clientToolDefs);
7
12
  }
8
13
  };
9
14
  }
10
- async function createElevenLabsConnection(token, _options, clientToolDefs) {
15
+ async function createElevenLabsConnection(token, _options, logger, clientToolDefs) {
11
16
  const eventHandlers = /* @__PURE__ */ new Map();
12
17
  let conversation = null;
13
18
  let messageIdCounter = 0;
@@ -41,18 +46,30 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
41
46
  const sessionOptions = {
42
47
  signedUrl: token.token,
43
48
  onConnect: () => {
49
+ logger.provider(`provider=elevenlabs direction=in type=connect`, {
50
+ frame: { type: "connect" }
51
+ });
44
52
  emit("status_change", { status: "connected" });
45
53
  emit("mode_change", { mode: "listening" });
46
54
  },
47
55
  onDisconnect: () => {
56
+ logger.provider(`provider=elevenlabs direction=in type=disconnect`, {
57
+ frame: { type: "disconnect" }
58
+ });
48
59
  emit("status_change", { status: "idle" });
49
60
  emit("mode_change", { mode: "idle" });
50
61
  },
51
62
  onModeChange: ({ mode }) => {
63
+ logger.provider(`provider=elevenlabs direction=in type=mode_change`, {
64
+ frame: { type: "mode_change", mode }
65
+ });
52
66
  const mappedMode = mode === "speaking" ? "speaking" : "listening";
53
67
  emit("mode_change", { mode: mappedMode });
54
68
  },
55
69
  onMessage: ({ message, source }) => {
70
+ logger.provider(`provider=elevenlabs direction=in type=message`, {
71
+ frame: { type: "message", source, message }
72
+ });
56
73
  const role = source === "user" ? "user" : "assistant";
57
74
  if (role === "user") {
58
75
  emit("transcript", {
@@ -71,6 +88,10 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
71
88
  }
72
89
  },
73
90
  onError: (error) => {
91
+ logger.errors("elevenlabs.realtime fatal", {
92
+ error,
93
+ source: "elevenlabs.realtime"
94
+ });
74
95
  emit("error", {
75
96
  error: new Error(
76
97
  typeof error === "string" ? error : error.message || "Unknown error"
@@ -81,9 +102,17 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
81
102
  if (Object.keys(elevenLabsClientTools).length > 0) {
82
103
  sessionOptions.clientTools = elevenLabsClientTools;
83
104
  }
84
- conversation = await Conversation.startSession(
85
- sessionOptions
86
- );
105
+ try {
106
+ conversation = await Conversation.startSession(
107
+ sessionOptions
108
+ );
109
+ } catch (error) {
110
+ logger.errors("elevenlabs.realtime fatal", {
111
+ error,
112
+ source: "elevenlabs.realtime"
113
+ });
114
+ throw error;
115
+ }
87
116
  const connection = {
88
117
  async disconnect() {
89
118
  if (conversation) {
@@ -100,19 +129,22 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
100
129
  },
101
130
  sendText(text) {
102
131
  if (!conversation) return;
132
+ logger.provider(`provider=elevenlabs direction=out type=user_message`, {
133
+ frame: { type: "user_message", text }
134
+ });
103
135
  conversation.sendUserMessage(text);
104
136
  },
105
137
  sendImage(_imageData, _mimeType) {
106
- console.warn(
107
- "ElevenLabs realtime does not support sending images directly."
108
- );
138
+ logger.errors("elevenlabs.realtime sendImage not supported", {
139
+ source: "elevenlabs.realtime"
140
+ });
109
141
  },
110
142
  sendToolResult(_callId, _result) {
111
143
  },
112
144
  updateSession(_config) {
113
- console.warn(
114
- "ElevenLabs does not support runtime session updates. Configure at connection time."
115
- );
145
+ logger.errors("elevenlabs.realtime updateSession not supported", {
146
+ source: "elevenlabs.realtime"
147
+ });
116
148
  },
117
149
  interrupt() {
118
150
  emit("mode_change", { mode: "listening" });
@@ -1 +1 @@
1
- {"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@11labs/client'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeMessage,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @11labs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n return {\n provider: 'elevenlabs',\n\n async connect(\n token: RealtimeToken,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n ): Promise<RealtimeConnection> {\n return createElevenLabsConnection(token, options, clientToolDefs)\n },\n }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n token: RealtimeToken,\n _options: ElevenLabsRealtimeOptions,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n let conversation: Awaited<\n ReturnType<typeof Conversation.startSession>\n > | null = null\n let messageIdCounter = 0\n\n // Empty arrays for when visualization isn't available\n const emptyFrequencyData = new Uint8Array(128)\n const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n // Helper to emit events\n function emit<TEvent extends RealtimeEvent>(\n event: TEvent,\n payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n ) {\n const handlers = eventHandlers.get(event)\n if (handlers) {\n for (const handler of handlers) {\n handler(payload)\n }\n }\n }\n\n function generateMessageId(): string {\n return `el-msg-${Date.now()}-${++messageIdCounter}`\n }\n\n // Convert TanStack tool definitions to ElevenLabs clientTools format.\n // @11labs/client@0.2.0 expects plain async functions, not objects.\n const elevenLabsClientTools: Record<\n string,\n (params: unknown) => Promise<string>\n > = {}\n\n if (clientToolDefs) {\n for (const tool of clientToolDefs) {\n elevenLabsClientTools[tool.name] = async (params: unknown) => {\n if (tool.execute) {\n const result = await tool.execute(params)\n return typeof result === 'string' ? result : JSON.stringify(result)\n }\n return JSON.stringify({\n error: `No execute function for tool ${tool.name}`,\n })\n }\n }\n }\n\n // Build session options\n const sessionOptions: Record<string, unknown> = {\n signedUrl: token.token,\n\n onConnect: () => {\n emit('status_change', { status: 'connected' as RealtimeStatus })\n emit('mode_change', { mode: 'listening' })\n },\n\n onDisconnect: () => {\n emit('status_change', { status: 'idle' as RealtimeStatus })\n emit('mode_change', { mode: 'idle' })\n },\n\n onModeChange: ({ mode }: { mode: string }) => {\n const mappedMode: RealtimeMode =\n mode === 'speaking' ? 'speaking' : 'listening'\n emit('mode_change', { mode: mappedMode })\n },\n\n onMessage: ({ message, source }: { message: string; source: string }) => {\n const role = source === 'user' ? 'user' : 'assistant'\n\n if (role === 'user') {\n // User transcripts: only emit transcript — the RealtimeClient\n // creates the user message when it receives a final transcript\n emit('transcript', {\n role,\n transcript: message,\n isFinal: true,\n })\n } else {\n // Assistant messages: only emit message_complete — the canonical\n // event for finalized assistant messages\n const realtimeMessage: RealtimeMessage = {\n id: generateMessageId(),\n role,\n timestamp: Date.now(),\n parts: [{ type: 'audio', transcript: message }],\n }\n emit('message_complete', { message: realtimeMessage })\n }\n },\n\n onError: (error: string | Error) => {\n emit('error', {\n error: new Error(\n typeof error === 'string' ? error : error.message || 'Unknown error',\n ),\n })\n },\n }\n\n // Only add clientTools if we have any\n if (Object.keys(elevenLabsClientTools).length > 0) {\n sessionOptions.clientTools = elevenLabsClientTools\n }\n\n // Start the conversation session\n conversation = await Conversation.startSession(\n sessionOptions as Parameters<typeof Conversation.startSession>[0],\n )\n\n // Connection implementation\n const connection: RealtimeConnection = {\n async disconnect() {\n if (conversation) {\n await conversation.endSession()\n conversation = null\n }\n emit('status_change', { status: 'idle' as RealtimeStatus })\n },\n\n async startAudioCapture() {\n // ElevenLabs SDK handles audio capture automatically\n // This is called when the session starts\n emit('mode_change', { mode: 'listening' })\n },\n\n stopAudioCapture() {\n // ElevenLabs SDK handles this\n emit('mode_change', { mode: 'idle' })\n },\n\n sendText(text: string) {\n if (!conversation) return\n conversation.sendUserMessage(text)\n },\n\n sendImage(_imageData: string, _mimeType: string) {\n // ElevenLabs does not support direct image input in the conversation API\n console.warn(\n 'ElevenLabs realtime does not support sending images directly.',\n )\n },\n\n sendToolResult(_callId: string, _result: string) {\n // ElevenLabs client tools are handled via the clientTools handlers\n // registered at session start — results are returned automatically\n },\n\n updateSession(_config: Partial<RealtimeSessionConfig>) {\n // ElevenLabs session config is set at creation time\n console.warn(\n 'ElevenLabs does not support runtime session updates. Configure at connection time.',\n )\n },\n\n interrupt() {\n // ElevenLabs handles interruption automatically via barge-in\n // No explicit API to call\n emit('mode_change', { mode: 'listening' })\n emit('interrupted', {})\n },\n\n on<TEvent extends RealtimeEvent>(\n event: TEvent,\n handler: RealtimeEventHandler<TEvent>,\n ): () => void {\n if (!eventHandlers.has(event)) {\n eventHandlers.set(event, new Set())\n }\n eventHandlers.get(event)!.add(handler)\n\n return () => {\n eventHandlers.get(event)?.delete(handler)\n }\n },\n\n getAudioVisualization(): AudioVisualization {\n return {\n get inputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getInputVolume()\n } catch {\n return 0\n }\n },\n\n get outputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getOutputVolume()\n } catch {\n return 0\n }\n },\n\n getInputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getInputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getOutputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getOutputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getInputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n getOutputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n get inputSampleRate() {\n return 16000\n },\n\n get outputSampleRate() {\n return 16000\n },\n }\n },\n }\n\n return connection\n}\n"],"names":[],"mappings":";AAkCO,SAAS,mBACd,UAAqC,IACpB;AACjB,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,2BAA2B,OAAO,SAAS,cAAc;AAAA,IAClE;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAC1D,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,iBAAe,MAAM,aAAa;AAAA,IAChC;AAAA,EAAA;AAIF,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}
1
+ {"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@11labs/client'\nimport { resolveDebugOption } from '@tanstack/ai/adapter-internals'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeMessage,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type { InternalLogger } from '@tanstack/ai/adapter-internals'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @11labs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n const logger = resolveDebugOption(options.debug)\n\n return {\n provider: 'elevenlabs',\n\n async connect(\n token: RealtimeToken,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n ): Promise<RealtimeConnection> {\n logger.request(`activity=realtime provider=elevenlabs`, {\n provider: 'elevenlabs',\n })\n return createElevenLabsConnection(token, options, logger, clientToolDefs)\n },\n }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n token: RealtimeToken,\n _options: ElevenLabsRealtimeOptions,\n logger: InternalLogger,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n let conversation: Awaited<\n ReturnType<typeof Conversation.startSession>\n > | null = null\n let messageIdCounter = 0\n\n // Empty arrays for when visualization isn't available\n const emptyFrequencyData = new Uint8Array(128)\n const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n // Helper to emit events\n function emit<TEvent extends RealtimeEvent>(\n event: TEvent,\n payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n ) {\n const handlers = eventHandlers.get(event)\n if (handlers) {\n for (const handler of handlers) {\n handler(payload)\n }\n }\n }\n\n function generateMessageId(): string {\n return `el-msg-${Date.now()}-${++messageIdCounter}`\n }\n\n // Convert TanStack tool definitions to ElevenLabs clientTools format.\n // @11labs/client@0.2.0 expects plain async functions, not objects.\n const elevenLabsClientTools: Record<\n string,\n (params: unknown) => Promise<string>\n > = {}\n\n if (clientToolDefs) {\n for (const tool of clientToolDefs) {\n elevenLabsClientTools[tool.name] = async (params: unknown) => {\n if (tool.execute) {\n const result = await tool.execute(params)\n return typeof result === 'string' ? result : JSON.stringify(result)\n }\n return JSON.stringify({\n error: `No execute function for tool ${tool.name}`,\n })\n }\n }\n }\n\n // Build session options\n const sessionOptions: Record<string, unknown> = {\n signedUrl: token.token,\n\n onConnect: () => {\n logger.provider(`provider=elevenlabs direction=in type=connect`, {\n frame: { type: 'connect' },\n })\n emit('status_change', { status: 'connected' as RealtimeStatus })\n emit('mode_change', { mode: 'listening' })\n },\n\n onDisconnect: () => {\n logger.provider(`provider=elevenlabs direction=in type=disconnect`, {\n frame: { type: 'disconnect' },\n })\n emit('status_change', { status: 'idle' as RealtimeStatus })\n emit('mode_change', { mode: 'idle' })\n },\n\n onModeChange: ({ mode }: { mode: string }) => {\n logger.provider(`provider=elevenlabs direction=in type=mode_change`, {\n frame: { type: 'mode_change', mode },\n })\n const mappedMode: RealtimeMode =\n mode === 'speaking' ? 'speaking' : 'listening'\n emit('mode_change', { mode: mappedMode })\n },\n\n onMessage: ({ message, source }: { message: string; source: string }) => {\n logger.provider(`provider=elevenlabs direction=in type=message`, {\n frame: { type: 'message', source, message },\n })\n const role = source === 'user' ? 'user' : 'assistant'\n\n if (role === 'user') {\n // User transcripts: only emit transcript — the RealtimeClient\n // creates the user message when it receives a final transcript\n emit('transcript', {\n role,\n transcript: message,\n isFinal: true,\n })\n } else {\n // Assistant messages: only emit message_complete — the canonical\n // event for finalized assistant messages\n const realtimeMessage: RealtimeMessage = {\n id: generateMessageId(),\n role,\n timestamp: Date.now(),\n parts: [{ type: 'audio', transcript: message }],\n }\n emit('message_complete', { message: realtimeMessage })\n }\n },\n\n onError: (error: string | Error) => {\n logger.errors('elevenlabs.realtime fatal', {\n error,\n source: 'elevenlabs.realtime',\n })\n emit('error', {\n error: new Error(\n typeof error === 'string' ? error : error.message || 'Unknown error',\n ),\n })\n },\n }\n\n // Only add clientTools if we have any\n if (Object.keys(elevenLabsClientTools).length > 0) {\n sessionOptions.clientTools = elevenLabsClientTools\n }\n\n // Start the conversation session\n try {\n conversation = await Conversation.startSession(\n sessionOptions as Parameters<typeof Conversation.startSession>[0],\n )\n } catch (error) {\n logger.errors('elevenlabs.realtime fatal', {\n error,\n source: 'elevenlabs.realtime',\n })\n throw error\n }\n\n // Connection implementation\n const connection: RealtimeConnection = {\n async disconnect() {\n if (conversation) {\n await conversation.endSession()\n conversation = null\n }\n emit('status_change', { status: 'idle' as RealtimeStatus })\n },\n\n async startAudioCapture() {\n // ElevenLabs SDK handles audio capture automatically\n // This is called when the session starts\n emit('mode_change', { mode: 'listening' })\n },\n\n stopAudioCapture() {\n // ElevenLabs SDK handles this\n emit('mode_change', { mode: 'idle' })\n },\n\n sendText(text: string) {\n if (!conversation) return\n logger.provider(`provider=elevenlabs direction=out type=user_message`, {\n frame: { type: 'user_message', text },\n })\n conversation.sendUserMessage(text)\n },\n\n sendImage(_imageData: string, _mimeType: string) {\n // ElevenLabs does not support direct image input in the conversation API\n logger.errors('elevenlabs.realtime sendImage not supported', {\n source: 'elevenlabs.realtime',\n })\n },\n\n sendToolResult(_callId: string, _result: string) {\n // ElevenLabs client tools are handled via the clientTools handlers\n // registered at session start — results are returned automatically\n },\n\n updateSession(_config: Partial<RealtimeSessionConfig>) {\n // ElevenLabs session config is set at creation time\n logger.errors('elevenlabs.realtime updateSession not supported', {\n source: 'elevenlabs.realtime',\n })\n },\n\n interrupt() {\n // ElevenLabs handles interruption automatically via barge-in\n // No explicit API to call\n emit('mode_change', { mode: 'listening' })\n emit('interrupted', {})\n },\n\n on<TEvent extends RealtimeEvent>(\n event: TEvent,\n handler: RealtimeEventHandler<TEvent>,\n ): () => void {\n if (!eventHandlers.has(event)) {\n eventHandlers.set(event, new Set())\n }\n eventHandlers.get(event)!.add(handler)\n\n return () => {\n eventHandlers.get(event)?.delete(handler)\n }\n },\n\n getAudioVisualization(): AudioVisualization {\n return {\n get inputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getInputVolume()\n } catch {\n return 0\n }\n },\n\n get outputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getOutputVolume()\n } catch {\n return 0\n }\n },\n\n getInputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getInputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getOutputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getOutputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getInputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n getOutputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n get inputSampleRate() {\n return 16000\n },\n\n get outputSampleRate() {\n return 16000\n },\n }\n },\n }\n\n return connection\n}\n"],"names":[],"mappings":";;AAoCO,SAAS,mBACd,UAAqC,IACpB;AACjB,QAAM,SAAS,mBAAmB,QAAQ,KAAK;AAE/C,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,QAAQ,yCAAyC;AAAA,QACtD,UAAU;AAAA,MAAA,CACX;AACD,aAAO,2BAA2B,OAAO,SAAS,QAAQ,cAAc;AAAA,IAC1E;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,QACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,UAAA;AAAA,MAAU,CAC1B;AACD,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,aAAO,SAAS,oDAAoD;AAAA,QAClE,OAAO,EAAE,MAAM,aAAA;AAAA,MAAa,CAC7B;AACD,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAC1D,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,aAAO,SAAS,qDAAqD;AAAA,QACnE,OAAO,EAAE,MAAM,eAAe,KAAA;AAAA,MAAK,CACpC;AACD,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,WAAW,QAAQ,QAAA;AAAA,MAAQ,CAC3C;AACD,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,aAAO,OAAO,6BAA6B;AAAA,QACzC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,MAAI;AACF,mBAAe,MAAM,aAAa;AAAA,MAChC;AAAA,IAAA;AAAA,EAEJ,SAAS,OAAO;AACd,WAAO,OAAO,6BAA6B;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AACD,UAAM;AAAA,EACR;AAGA,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,aAAO,SAAS,uDAAuD;AAAA,QACrE,OAAO,EAAE,MAAM,gBAAgB,KAAA;AAAA,MAAK,CACrC;AACD,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,aAAO,OAAO,+CAA+C;AAAA,QAC3D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,aAAO,OAAO,mDAAmD;AAAA,QAC/D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}
@@ -1,3 +1,4 @@
1
+ import { DebugOption } from '@tanstack/ai';
1
2
  /**
2
3
  * Options for the ElevenLabs realtime token adapter
3
4
  */
@@ -22,8 +23,13 @@ export interface ElevenLabsRealtimeTokenOptions {
22
23
  export interface ElevenLabsRealtimeOptions {
23
24
  /** Connection mode (default: auto-detect) */
24
25
  connectionMode?: 'websocket' | 'webrtc';
25
- /** Enable debug logging */
26
- debug?: boolean;
26
+ /**
27
+ * Enable debug logging for this adapter.
28
+ *
29
+ * - `true` enables all categories (`request`, `response`, `provider`, `errors`).
30
+ * - A {@link DebugConfig} object selects categories and/or a custom sink.
31
+ */
32
+ debug?: DebugOption;
27
33
  }
28
34
  /**
29
35
  * ElevenLabs conversation mode
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tanstack/ai-elevenlabs",
3
- "version": "0.1.6",
3
+ "version": "0.1.8",
4
4
  "description": "ElevenLabs adapter for TanStack AI realtime voice",
5
5
  "author": "",
6
6
  "license": "MIT",
@@ -34,13 +34,13 @@
34
34
  "@11labs/client": "^0.2.0"
35
35
  },
36
36
  "peerDependencies": {
37
- "@tanstack/ai": "^0.12.0",
38
- "@tanstack/ai-client": "^0.7.13"
37
+ "@tanstack/ai": "^0.14.0",
38
+ "@tanstack/ai-client": "^0.8.0"
39
39
  },
40
40
  "devDependencies": {
41
41
  "@vitest/coverage-v8": "4.0.14",
42
- "@tanstack/ai": "0.12.0",
43
- "@tanstack/ai-client": "0.7.13"
42
+ "@tanstack/ai": "0.14.0",
43
+ "@tanstack/ai-client": "0.8.0"
44
44
  },
45
45
  "scripts": {
46
46
  "build": "vite build",
@@ -1,4 +1,5 @@
1
1
  import { Conversation } from '@11labs/client'
2
+ import { resolveDebugOption } from '@tanstack/ai/adapter-internals'
2
3
  import type {
3
4
  AnyClientTool,
4
5
  AudioVisualization,
@@ -10,6 +11,7 @@ import type {
10
11
  RealtimeStatus,
11
12
  RealtimeToken,
12
13
  } from '@tanstack/ai'
14
+ import type { InternalLogger } from '@tanstack/ai/adapter-internals'
13
15
  import type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'
14
16
  import type { ElevenLabsRealtimeOptions } from './types'
15
17
 
@@ -35,6 +37,8 @@ import type { ElevenLabsRealtimeOptions } from './types'
35
37
  export function elevenlabsRealtime(
36
38
  options: ElevenLabsRealtimeOptions = {},
37
39
  ): RealtimeAdapter {
40
+ const logger = resolveDebugOption(options.debug)
41
+
38
42
  return {
39
43
  provider: 'elevenlabs',
40
44
 
@@ -42,7 +46,10 @@ export function elevenlabsRealtime(
42
46
  token: RealtimeToken,
43
47
  clientToolDefs?: ReadonlyArray<AnyClientTool>,
44
48
  ): Promise<RealtimeConnection> {
45
- return createElevenLabsConnection(token, options, clientToolDefs)
49
+ logger.request(`activity=realtime provider=elevenlabs`, {
50
+ provider: 'elevenlabs',
51
+ })
52
+ return createElevenLabsConnection(token, options, logger, clientToolDefs)
46
53
  },
47
54
  }
48
55
  }
@@ -53,6 +60,7 @@ export function elevenlabsRealtime(
53
60
  async function createElevenLabsConnection(
54
61
  token: RealtimeToken,
55
62
  _options: ElevenLabsRealtimeOptions,
63
+ logger: InternalLogger,
56
64
  clientToolDefs?: ReadonlyArray<AnyClientTool>,
57
65
  ): Promise<RealtimeConnection> {
58
66
  const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()
@@ -108,22 +116,34 @@ async function createElevenLabsConnection(
108
116
  signedUrl: token.token,
109
117
 
110
118
  onConnect: () => {
119
+ logger.provider(`provider=elevenlabs direction=in type=connect`, {
120
+ frame: { type: 'connect' },
121
+ })
111
122
  emit('status_change', { status: 'connected' as RealtimeStatus })
112
123
  emit('mode_change', { mode: 'listening' })
113
124
  },
114
125
 
115
126
  onDisconnect: () => {
127
+ logger.provider(`provider=elevenlabs direction=in type=disconnect`, {
128
+ frame: { type: 'disconnect' },
129
+ })
116
130
  emit('status_change', { status: 'idle' as RealtimeStatus })
117
131
  emit('mode_change', { mode: 'idle' })
118
132
  },
119
133
 
120
134
  onModeChange: ({ mode }: { mode: string }) => {
135
+ logger.provider(`provider=elevenlabs direction=in type=mode_change`, {
136
+ frame: { type: 'mode_change', mode },
137
+ })
121
138
  const mappedMode: RealtimeMode =
122
139
  mode === 'speaking' ? 'speaking' : 'listening'
123
140
  emit('mode_change', { mode: mappedMode })
124
141
  },
125
142
 
126
143
  onMessage: ({ message, source }: { message: string; source: string }) => {
144
+ logger.provider(`provider=elevenlabs direction=in type=message`, {
145
+ frame: { type: 'message', source, message },
146
+ })
127
147
  const role = source === 'user' ? 'user' : 'assistant'
128
148
 
129
149
  if (role === 'user') {
@@ -148,6 +168,10 @@ async function createElevenLabsConnection(
148
168
  },
149
169
 
150
170
  onError: (error: string | Error) => {
171
+ logger.errors('elevenlabs.realtime fatal', {
172
+ error,
173
+ source: 'elevenlabs.realtime',
174
+ })
151
175
  emit('error', {
152
176
  error: new Error(
153
177
  typeof error === 'string' ? error : error.message || 'Unknown error',
@@ -162,9 +186,17 @@ async function createElevenLabsConnection(
162
186
  }
163
187
 
164
188
  // Start the conversation session
165
- conversation = await Conversation.startSession(
166
- sessionOptions as Parameters<typeof Conversation.startSession>[0],
167
- )
189
+ try {
190
+ conversation = await Conversation.startSession(
191
+ sessionOptions as Parameters<typeof Conversation.startSession>[0],
192
+ )
193
+ } catch (error) {
194
+ logger.errors('elevenlabs.realtime fatal', {
195
+ error,
196
+ source: 'elevenlabs.realtime',
197
+ })
198
+ throw error
199
+ }
168
200
 
169
201
  // Connection implementation
170
202
  const connection: RealtimeConnection = {
@@ -189,14 +221,17 @@ async function createElevenLabsConnection(
189
221
 
190
222
  sendText(text: string) {
191
223
  if (!conversation) return
224
+ logger.provider(`provider=elevenlabs direction=out type=user_message`, {
225
+ frame: { type: 'user_message', text },
226
+ })
192
227
  conversation.sendUserMessage(text)
193
228
  },
194
229
 
195
230
  sendImage(_imageData: string, _mimeType: string) {
196
231
  // ElevenLabs does not support direct image input in the conversation API
197
- console.warn(
198
- 'ElevenLabs realtime does not support sending images directly.',
199
- )
232
+ logger.errors('elevenlabs.realtime sendImage not supported', {
233
+ source: 'elevenlabs.realtime',
234
+ })
200
235
  },
201
236
 
202
237
  sendToolResult(_callId: string, _result: string) {
@@ -206,9 +241,9 @@ async function createElevenLabsConnection(
206
241
 
207
242
  updateSession(_config: Partial<RealtimeSessionConfig>) {
208
243
  // ElevenLabs session config is set at creation time
209
- console.warn(
210
- 'ElevenLabs does not support runtime session updates. Configure at connection time.',
211
- )
244
+ logger.errors('elevenlabs.realtime updateSession not supported', {
245
+ source: 'elevenlabs.realtime',
246
+ })
212
247
  },
213
248
 
214
249
  interrupt() {
@@ -1,3 +1,5 @@
1
+ import type { DebugOption } from '@tanstack/ai'
2
+
1
3
  /**
2
4
  * Options for the ElevenLabs realtime token adapter
3
5
  */
@@ -23,8 +25,13 @@ export interface ElevenLabsRealtimeTokenOptions {
23
25
  export interface ElevenLabsRealtimeOptions {
24
26
  /** Connection mode (default: auto-detect) */
25
27
  connectionMode?: 'websocket' | 'webrtc'
26
- /** Enable debug logging */
27
- debug?: boolean
28
+ /**
29
+ * Enable debug logging for this adapter.
30
+ *
31
+ * - `true` enables all categories (`request`, `response`, `provider`, `errors`).
32
+ * - A {@link DebugConfig} object selects categories and/or a custom sink.
33
+ */
34
+ debug?: DebugOption
28
35
  }
29
36
 
30
37
  /**