@tanstack/ai-elevenlabs 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
import { Conversation } from "@11labs/client";
|
|
2
|
+
import { resolveDebugOption } from "@tanstack/ai/adapter-internals";
|
|
2
3
|
function elevenlabsRealtime(options = {}) {
|
|
4
|
+
const logger = resolveDebugOption(options.debug);
|
|
3
5
|
return {
|
|
4
6
|
provider: "elevenlabs",
|
|
5
7
|
async connect(token, clientToolDefs) {
|
|
6
|
-
|
|
8
|
+
logger.request(`activity=realtime provider=elevenlabs`, {
|
|
9
|
+
provider: "elevenlabs"
|
|
10
|
+
});
|
|
11
|
+
return createElevenLabsConnection(token, options, logger, clientToolDefs);
|
|
7
12
|
}
|
|
8
13
|
};
|
|
9
14
|
}
|
|
10
|
-
async function createElevenLabsConnection(token, _options, clientToolDefs) {
|
|
15
|
+
async function createElevenLabsConnection(token, _options, logger, clientToolDefs) {
|
|
11
16
|
const eventHandlers = /* @__PURE__ */ new Map();
|
|
12
17
|
let conversation = null;
|
|
13
18
|
let messageIdCounter = 0;
|
|
@@ -41,18 +46,30 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
|
|
|
41
46
|
const sessionOptions = {
|
|
42
47
|
signedUrl: token.token,
|
|
43
48
|
onConnect: () => {
|
|
49
|
+
logger.provider(`provider=elevenlabs direction=in type=connect`, {
|
|
50
|
+
frame: { type: "connect" }
|
|
51
|
+
});
|
|
44
52
|
emit("status_change", { status: "connected" });
|
|
45
53
|
emit("mode_change", { mode: "listening" });
|
|
46
54
|
},
|
|
47
55
|
onDisconnect: () => {
|
|
56
|
+
logger.provider(`provider=elevenlabs direction=in type=disconnect`, {
|
|
57
|
+
frame: { type: "disconnect" }
|
|
58
|
+
});
|
|
48
59
|
emit("status_change", { status: "idle" });
|
|
49
60
|
emit("mode_change", { mode: "idle" });
|
|
50
61
|
},
|
|
51
62
|
onModeChange: ({ mode }) => {
|
|
63
|
+
logger.provider(`provider=elevenlabs direction=in type=mode_change`, {
|
|
64
|
+
frame: { type: "mode_change", mode }
|
|
65
|
+
});
|
|
52
66
|
const mappedMode = mode === "speaking" ? "speaking" : "listening";
|
|
53
67
|
emit("mode_change", { mode: mappedMode });
|
|
54
68
|
},
|
|
55
69
|
onMessage: ({ message, source }) => {
|
|
70
|
+
logger.provider(`provider=elevenlabs direction=in type=message`, {
|
|
71
|
+
frame: { type: "message", source, message }
|
|
72
|
+
});
|
|
56
73
|
const role = source === "user" ? "user" : "assistant";
|
|
57
74
|
if (role === "user") {
|
|
58
75
|
emit("transcript", {
|
|
@@ -71,6 +88,10 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
|
|
|
71
88
|
}
|
|
72
89
|
},
|
|
73
90
|
onError: (error) => {
|
|
91
|
+
logger.errors("elevenlabs.realtime fatal", {
|
|
92
|
+
error,
|
|
93
|
+
source: "elevenlabs.realtime"
|
|
94
|
+
});
|
|
74
95
|
emit("error", {
|
|
75
96
|
error: new Error(
|
|
76
97
|
typeof error === "string" ? error : error.message || "Unknown error"
|
|
@@ -81,9 +102,17 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
|
|
|
81
102
|
if (Object.keys(elevenLabsClientTools).length > 0) {
|
|
82
103
|
sessionOptions.clientTools = elevenLabsClientTools;
|
|
83
104
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
105
|
+
try {
|
|
106
|
+
conversation = await Conversation.startSession(
|
|
107
|
+
sessionOptions
|
|
108
|
+
);
|
|
109
|
+
} catch (error) {
|
|
110
|
+
logger.errors("elevenlabs.realtime fatal", {
|
|
111
|
+
error,
|
|
112
|
+
source: "elevenlabs.realtime"
|
|
113
|
+
});
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
87
116
|
const connection = {
|
|
88
117
|
async disconnect() {
|
|
89
118
|
if (conversation) {
|
|
@@ -100,19 +129,22 @@ async function createElevenLabsConnection(token, _options, clientToolDefs) {
|
|
|
100
129
|
},
|
|
101
130
|
sendText(text) {
|
|
102
131
|
if (!conversation) return;
|
|
132
|
+
logger.provider(`provider=elevenlabs direction=out type=user_message`, {
|
|
133
|
+
frame: { type: "user_message", text }
|
|
134
|
+
});
|
|
103
135
|
conversation.sendUserMessage(text);
|
|
104
136
|
},
|
|
105
137
|
sendImage(_imageData, _mimeType) {
|
|
106
|
-
|
|
107
|
-
"
|
|
108
|
-
);
|
|
138
|
+
logger.errors("elevenlabs.realtime sendImage not supported", {
|
|
139
|
+
source: "elevenlabs.realtime"
|
|
140
|
+
});
|
|
109
141
|
},
|
|
110
142
|
sendToolResult(_callId, _result) {
|
|
111
143
|
},
|
|
112
144
|
updateSession(_config) {
|
|
113
|
-
|
|
114
|
-
"
|
|
115
|
-
);
|
|
145
|
+
logger.errors("elevenlabs.realtime updateSession not supported", {
|
|
146
|
+
source: "elevenlabs.realtime"
|
|
147
|
+
});
|
|
116
148
|
},
|
|
117
149
|
interrupt() {
|
|
118
150
|
emit("mode_change", { mode: "listening" });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@11labs/client'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeMessage,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @11labs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n return {\n provider: 'elevenlabs',\n\n async connect(\n token: RealtimeToken,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n ): Promise<RealtimeConnection> {\n return createElevenLabsConnection(token, options, clientToolDefs)\n },\n }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n token: RealtimeToken,\n _options: ElevenLabsRealtimeOptions,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n let conversation: Awaited<\n ReturnType<typeof Conversation.startSession>\n > | null = null\n let messageIdCounter = 0\n\n // Empty arrays for when visualization isn't available\n const emptyFrequencyData = new Uint8Array(128)\n const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n // Helper to emit events\n function emit<TEvent extends RealtimeEvent>(\n event: TEvent,\n payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n ) {\n const handlers = eventHandlers.get(event)\n if (handlers) {\n for (const handler of handlers) {\n handler(payload)\n }\n }\n }\n\n function generateMessageId(): string {\n return `el-msg-${Date.now()}-${++messageIdCounter}`\n }\n\n // Convert TanStack tool definitions to ElevenLabs clientTools format.\n // @11labs/client@0.2.0 expects plain async functions, not objects.\n const elevenLabsClientTools: Record<\n string,\n (params: unknown) => Promise<string>\n > = {}\n\n if (clientToolDefs) {\n for (const tool of clientToolDefs) {\n elevenLabsClientTools[tool.name] = async (params: unknown) => {\n if (tool.execute) {\n const result = await tool.execute(params)\n return typeof result === 'string' ? result : JSON.stringify(result)\n }\n return JSON.stringify({\n error: `No execute function for tool ${tool.name}`,\n })\n }\n }\n }\n\n // Build session options\n const sessionOptions: Record<string, unknown> = {\n signedUrl: token.token,\n\n onConnect: () => {\n emit('status_change', { status: 'connected' as RealtimeStatus })\n emit('mode_change', { mode: 'listening' })\n },\n\n onDisconnect: () => {\n emit('status_change', { status: 'idle' as RealtimeStatus })\n emit('mode_change', { mode: 'idle' })\n },\n\n onModeChange: ({ mode }: { mode: string }) => {\n const mappedMode: RealtimeMode =\n mode === 'speaking' ? 'speaking' : 'listening'\n emit('mode_change', { mode: mappedMode })\n },\n\n onMessage: ({ message, source }: { message: string; source: string }) => {\n const role = source === 'user' ? 'user' : 'assistant'\n\n if (role === 'user') {\n // User transcripts: only emit transcript — the RealtimeClient\n // creates the user message when it receives a final transcript\n emit('transcript', {\n role,\n transcript: message,\n isFinal: true,\n })\n } else {\n // Assistant messages: only emit message_complete — the canonical\n // event for finalized assistant messages\n const realtimeMessage: RealtimeMessage = {\n id: generateMessageId(),\n role,\n timestamp: Date.now(),\n parts: [{ type: 'audio', transcript: message }],\n }\n emit('message_complete', { message: realtimeMessage })\n }\n },\n\n onError: (error: string | Error) => {\n emit('error', {\n error: new Error(\n typeof error === 'string' ? error : error.message || 'Unknown error',\n ),\n })\n },\n }\n\n // Only add clientTools if we have any\n if (Object.keys(elevenLabsClientTools).length > 0) {\n sessionOptions.clientTools = elevenLabsClientTools\n }\n\n // Start the conversation session\n conversation = await Conversation.startSession(\n sessionOptions as Parameters<typeof Conversation.startSession>[0],\n )\n\n // Connection implementation\n const connection: RealtimeConnection = {\n async disconnect() {\n if (conversation) {\n await conversation.endSession()\n conversation = null\n }\n emit('status_change', { status: 'idle' as RealtimeStatus })\n },\n\n async startAudioCapture() {\n // ElevenLabs SDK handles audio capture automatically\n // This is called when the session starts\n emit('mode_change', { mode: 'listening' })\n },\n\n stopAudioCapture() {\n // ElevenLabs SDK handles this\n emit('mode_change', { mode: 'idle' })\n },\n\n sendText(text: string) {\n if (!conversation) return\n conversation.sendUserMessage(text)\n },\n\n sendImage(_imageData: string, _mimeType: string) {\n // ElevenLabs does not support direct image input in the conversation API\n console.warn(\n 'ElevenLabs realtime does not support sending images directly.',\n )\n },\n\n sendToolResult(_callId: string, _result: string) {\n // ElevenLabs client tools are handled via the clientTools handlers\n // registered at session start — results are returned automatically\n },\n\n updateSession(_config: Partial<RealtimeSessionConfig>) {\n // ElevenLabs session config is set at creation time\n console.warn(\n 'ElevenLabs does not support runtime session updates. Configure at connection time.',\n )\n },\n\n interrupt() {\n // ElevenLabs handles interruption automatically via barge-in\n // No explicit API to call\n emit('mode_change', { mode: 'listening' })\n emit('interrupted', {})\n },\n\n on<TEvent extends RealtimeEvent>(\n event: TEvent,\n handler: RealtimeEventHandler<TEvent>,\n ): () => void {\n if (!eventHandlers.has(event)) {\n eventHandlers.set(event, new Set())\n }\n eventHandlers.get(event)!.add(handler)\n\n return () => {\n eventHandlers.get(event)?.delete(handler)\n }\n },\n\n getAudioVisualization(): AudioVisualization {\n return {\n get inputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getInputVolume()\n } catch {\n return 0\n }\n },\n\n get outputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getOutputVolume()\n } catch {\n return 0\n }\n },\n\n getInputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getInputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getOutputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getOutputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getInputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n getOutputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n get inputSampleRate() {\n return 16000\n },\n\n get outputSampleRate() {\n return 16000\n },\n }\n },\n }\n\n return connection\n}\n"],"names":[],"mappings":";AAkCO,SAAS,mBACd,UAAqC,IACpB;AACjB,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,2BAA2B,OAAO,SAAS,cAAc;AAAA,IAClE;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAC1D,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,iBAAe,MAAM,aAAa;AAAA,IAChC;AAAA,EAAA;AAIF,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,cAAQ;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}
|
|
1
|
+
{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { Conversation } from '@11labs/client'\nimport { resolveDebugOption } from '@tanstack/ai/adapter-internals'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeMessage,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type { InternalLogger } from '@tanstack/ai/adapter-internals'\nimport type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'\nimport type { ElevenLabsRealtimeOptions } from './types'\n\n/**\n * Creates an ElevenLabs realtime adapter for client-side use.\n *\n * Wraps the @11labs/client SDK for voice conversations.\n *\n * @param options - Optional configuration\n * @returns A RealtimeAdapter for use with RealtimeClient\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { elevenlabsRealtime } from '@tanstack/ai-elevenlabs'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: elevenlabsRealtime(),\n * })\n * ```\n */\nexport function elevenlabsRealtime(\n options: ElevenLabsRealtimeOptions = {},\n): RealtimeAdapter {\n const logger = resolveDebugOption(options.debug)\n\n return {\n provider: 'elevenlabs',\n\n async connect(\n token: RealtimeToken,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n ): Promise<RealtimeConnection> {\n logger.request(`activity=realtime provider=elevenlabs`, {\n provider: 'elevenlabs',\n })\n return createElevenLabsConnection(token, options, logger, clientToolDefs)\n },\n }\n}\n\n/**\n * Creates a connection to ElevenLabs conversational AI\n */\nasync function createElevenLabsConnection(\n token: RealtimeToken,\n _options: ElevenLabsRealtimeOptions,\n logger: InternalLogger,\n clientToolDefs?: ReadonlyArray<AnyClientTool>,\n): Promise<RealtimeConnection> {\n const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n let conversation: Awaited<\n ReturnType<typeof Conversation.startSession>\n > | null = null\n let messageIdCounter = 0\n\n // Empty arrays for when visualization isn't available\n const emptyFrequencyData = new Uint8Array(128)\n const emptyTimeDomainData = new Uint8Array(128).fill(128)\n\n // Helper to emit events\n function emit<TEvent extends RealtimeEvent>(\n event: TEvent,\n payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n ) {\n const handlers = eventHandlers.get(event)\n if (handlers) {\n for (const handler of handlers) {\n handler(payload)\n }\n }\n }\n\n function generateMessageId(): string {\n return `el-msg-${Date.now()}-${++messageIdCounter}`\n }\n\n // Convert TanStack tool definitions to ElevenLabs clientTools format.\n // @11labs/client@0.2.0 expects plain async functions, not objects.\n const elevenLabsClientTools: Record<\n string,\n (params: unknown) => Promise<string>\n > = {}\n\n if (clientToolDefs) {\n for (const tool of clientToolDefs) {\n elevenLabsClientTools[tool.name] = async (params: unknown) => {\n if (tool.execute) {\n const result = await tool.execute(params)\n return typeof result === 'string' ? result : JSON.stringify(result)\n }\n return JSON.stringify({\n error: `No execute function for tool ${tool.name}`,\n })\n }\n }\n }\n\n // Build session options\n const sessionOptions: Record<string, unknown> = {\n signedUrl: token.token,\n\n onConnect: () => {\n logger.provider(`provider=elevenlabs direction=in type=connect`, {\n frame: { type: 'connect' },\n })\n emit('status_change', { status: 'connected' as RealtimeStatus })\n emit('mode_change', { mode: 'listening' })\n },\n\n onDisconnect: () => {\n logger.provider(`provider=elevenlabs direction=in type=disconnect`, {\n frame: { type: 'disconnect' },\n })\n emit('status_change', { status: 'idle' as RealtimeStatus })\n emit('mode_change', { mode: 'idle' })\n },\n\n onModeChange: ({ mode }: { mode: string }) => {\n logger.provider(`provider=elevenlabs direction=in type=mode_change`, {\n frame: { type: 'mode_change', mode },\n })\n const mappedMode: RealtimeMode =\n mode === 'speaking' ? 'speaking' : 'listening'\n emit('mode_change', { mode: mappedMode })\n },\n\n onMessage: ({ message, source }: { message: string; source: string }) => {\n logger.provider(`provider=elevenlabs direction=in type=message`, {\n frame: { type: 'message', source, message },\n })\n const role = source === 'user' ? 'user' : 'assistant'\n\n if (role === 'user') {\n // User transcripts: only emit transcript — the RealtimeClient\n // creates the user message when it receives a final transcript\n emit('transcript', {\n role,\n transcript: message,\n isFinal: true,\n })\n } else {\n // Assistant messages: only emit message_complete — the canonical\n // event for finalized assistant messages\n const realtimeMessage: RealtimeMessage = {\n id: generateMessageId(),\n role,\n timestamp: Date.now(),\n parts: [{ type: 'audio', transcript: message }],\n }\n emit('message_complete', { message: realtimeMessage })\n }\n },\n\n onError: (error: string | Error) => {\n logger.errors('elevenlabs.realtime fatal', {\n error,\n source: 'elevenlabs.realtime',\n })\n emit('error', {\n error: new Error(\n typeof error === 'string' ? error : error.message || 'Unknown error',\n ),\n })\n },\n }\n\n // Only add clientTools if we have any\n if (Object.keys(elevenLabsClientTools).length > 0) {\n sessionOptions.clientTools = elevenLabsClientTools\n }\n\n // Start the conversation session\n try {\n conversation = await Conversation.startSession(\n sessionOptions as Parameters<typeof Conversation.startSession>[0],\n )\n } catch (error) {\n logger.errors('elevenlabs.realtime fatal', {\n error,\n source: 'elevenlabs.realtime',\n })\n throw error\n }\n\n // Connection implementation\n const connection: RealtimeConnection = {\n async disconnect() {\n if (conversation) {\n await conversation.endSession()\n conversation = null\n }\n emit('status_change', { status: 'idle' as RealtimeStatus })\n },\n\n async startAudioCapture() {\n // ElevenLabs SDK handles audio capture automatically\n // This is called when the session starts\n emit('mode_change', { mode: 'listening' })\n },\n\n stopAudioCapture() {\n // ElevenLabs SDK handles this\n emit('mode_change', { mode: 'idle' })\n },\n\n sendText(text: string) {\n if (!conversation) return\n logger.provider(`provider=elevenlabs direction=out type=user_message`, {\n frame: { type: 'user_message', text },\n })\n conversation.sendUserMessage(text)\n },\n\n sendImage(_imageData: string, _mimeType: string) {\n // ElevenLabs does not support direct image input in the conversation API\n logger.errors('elevenlabs.realtime sendImage not supported', {\n source: 'elevenlabs.realtime',\n })\n },\n\n sendToolResult(_callId: string, _result: string) {\n // ElevenLabs client tools are handled via the clientTools handlers\n // registered at session start — results are returned automatically\n },\n\n updateSession(_config: Partial<RealtimeSessionConfig>) {\n // ElevenLabs session config is set at creation time\n logger.errors('elevenlabs.realtime updateSession not supported', {\n source: 'elevenlabs.realtime',\n })\n },\n\n interrupt() {\n // ElevenLabs handles interruption automatically via barge-in\n // No explicit API to call\n emit('mode_change', { mode: 'listening' })\n emit('interrupted', {})\n },\n\n on<TEvent extends RealtimeEvent>(\n event: TEvent,\n handler: RealtimeEventHandler<TEvent>,\n ): () => void {\n if (!eventHandlers.has(event)) {\n eventHandlers.set(event, new Set())\n }\n eventHandlers.get(event)!.add(handler)\n\n return () => {\n eventHandlers.get(event)?.delete(handler)\n }\n },\n\n getAudioVisualization(): AudioVisualization {\n return {\n get inputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getInputVolume()\n } catch {\n return 0\n }\n },\n\n get outputLevel() {\n if (!conversation) return 0\n try {\n return conversation.getOutputVolume()\n } catch {\n return 0\n }\n },\n\n getInputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getInputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getOutputFrequencyData() {\n if (!conversation) return emptyFrequencyData\n try {\n return conversation.getOutputByteFrequencyData()\n } catch {\n return emptyFrequencyData\n }\n },\n\n getInputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n getOutputTimeDomainData() {\n // ElevenLabs SDK doesn't expose time domain data\n return emptyTimeDomainData\n },\n\n get inputSampleRate() {\n return 16000\n },\n\n get outputSampleRate() {\n return 16000\n },\n }\n },\n }\n\n return connection\n}\n"],"names":[],"mappings":";;AAoCO,SAAS,mBACd,UAAqC,IACpB;AACjB,QAAM,SAAS,mBAAmB,QAAQ,KAAK;AAE/C,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,gBAC6B;AAC7B,aAAO,QAAQ,yCAAyC;AAAA,QACtD,UAAU;AAAA,MAAA,CACX;AACD,aAAO,2BAA2B,OAAO,SAAS,QAAQ,cAAc;AAAA,IAC1E;AAAA,EAAA;AAEJ;AAKA,eAAe,2BACb,OACA,UACA,QACA,gBAC6B;AAC7B,QAAM,oCAAoB,IAAA;AAC1B,MAAI,eAEO;AACX,MAAI,mBAAmB;AAGvB,QAAM,qBAAqB,IAAI,WAAW,GAAG;AAC7C,QAAM,sBAAsB,IAAI,WAAW,GAAG,EAAE,KAAK,GAAG;AAGxD,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAA4B;AACnC,WAAO,UAAU,KAAK,IAAA,CAAK,IAAI,EAAE,gBAAgB;AAAA,EACnD;AAIA,QAAM,wBAGF,CAAA;AAEJ,MAAI,gBAAgB;AAClB,eAAW,QAAQ,gBAAgB;AACjC,4BAAsB,KAAK,IAAI,IAAI,OAAO,WAAoB;AAC5D,YAAI,KAAK,SAAS;AAChB,gBAAM,SAAS,MAAM,KAAK,QAAQ,MAAM;AACxC,iBAAO,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,QACpE;AACA,eAAO,KAAK,UAAU;AAAA,UACpB,OAAO,gCAAgC,KAAK,IAAI;AAAA,QAAA,CACjD;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAA0C;AAAA,IAC9C,WAAW,MAAM;AAAA,IAEjB,WAAW,MAAM;AACf,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,UAAA;AAAA,MAAU,CAC1B;AACD,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,cAAc,MAAM;AAClB,aAAO,SAAS,oDAAoD;AAAA,QAClE,OAAO,EAAE,MAAM,aAAA;AAAA,MAAa,CAC7B;AACD,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAC1D,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,cAAc,CAAC,EAAE,WAA6B;AAC5C,aAAO,SAAS,qDAAqD;AAAA,QACnE,OAAO,EAAE,MAAM,eAAe,KAAA;AAAA,MAAK,CACpC;AACD,YAAM,aACJ,SAAS,aAAa,aAAa;AACrC,WAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,IAC1C;AAAA,IAEA,WAAW,CAAC,EAAE,SAAS,aAAkD;AACvE,aAAO,SAAS,iDAAiD;AAAA,QAC/D,OAAO,EAAE,MAAM,WAAW,QAAQ,QAAA;AAAA,MAAQ,CAC3C;AACD,YAAM,OAAO,WAAW,SAAS,SAAS;AAE1C,UAAI,SAAS,QAAQ;AAGnB,aAAK,cAAc;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AAAA,MACH,OAAO;AAGL,cAAM,kBAAmC;AAAA,UACvC,IAAI,kBAAA;AAAA,UACJ;AAAA,UACA,WAAW,KAAK,IAAA;AAAA,UAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,SAAS;AAAA,QAAA;AAEhD,aAAK,oBAAoB,EAAE,SAAS,gBAAA,CAAiB;AAAA,MACvD;AAAA,IACF;AAAA,IAEA,SAAS,CAAC,UAA0B;AAClC,aAAO,OAAO,6BAA6B;AAAA,QACzC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,WAAK,SAAS;AAAA,QACZ,OAAO,IAAI;AAAA,UACT,OAAO,UAAU,WAAW,QAAQ,MAAM,WAAW;AAAA,QAAA;AAAA,MACvD,CACD;AAAA,IACH;AAAA,EAAA;AAIF,MAAI,OAAO,KAAK,qBAAqB,EAAE,SAAS,GAAG;AACjD,mBAAe,cAAc;AAAA,EAC/B;AAGA,MAAI;AACF,mBAAe,MAAM,aAAa;AAAA,MAChC;AAAA,IAAA;AAAA,EAEJ,SAAS,OAAO;AACd,WAAO,OAAO,6BAA6B;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AACD,UAAM;AAAA,EACR;AAGA,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AACjB,UAAI,cAAc;AAChB,cAAM,aAAa,WAAA;AACnB,uBAAe;AAAA,MACjB;AACA,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AAGxB,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AAEjB,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,UAAI,CAAC,aAAc;AACnB,aAAO,SAAS,uDAAuD;AAAA,QACrE,OAAO,EAAE,MAAM,gBAAgB,KAAA;AAAA,MAAK,CACrC;AACD,mBAAa,gBAAgB,IAAI;AAAA,IACnC;AAAA,IAEA,UAAU,YAAoB,WAAmB;AAE/C,aAAO,OAAO,+CAA+C;AAAA,QAC3D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,eAAe,SAAiB,SAAiB;AAAA,IAGjD;AAAA,IAEA,cAAc,SAAyC;AAErD,aAAO,OAAO,mDAAmD;AAAA,QAC/D,QAAQ;AAAA,MAAA,CACT;AAAA,IACH;AAAA,IAEA,YAAY;AAGV,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE;AAAA,IACxB;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,eAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,gBAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,0BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC,aAAc,QAAO;AAC1B,cAAI;AACF,mBAAO,aAAa,2BAAA;AAAA,UACtB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QAEA,yBAAyB;AAEvB,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AAExB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAGF,SAAO;AACT;"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DebugOption } from '@tanstack/ai';
|
|
1
2
|
/**
|
|
2
3
|
* Options for the ElevenLabs realtime token adapter
|
|
3
4
|
*/
|
|
@@ -22,8 +23,13 @@ export interface ElevenLabsRealtimeTokenOptions {
|
|
|
22
23
|
export interface ElevenLabsRealtimeOptions {
|
|
23
24
|
/** Connection mode (default: auto-detect) */
|
|
24
25
|
connectionMode?: 'websocket' | 'webrtc';
|
|
25
|
-
/**
|
|
26
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Enable debug logging for this adapter.
|
|
28
|
+
*
|
|
29
|
+
* - `true` enables all categories (`request`, `response`, `provider`, `errors`).
|
|
30
|
+
* - A {@link DebugConfig} object selects categories and/or a custom sink.
|
|
31
|
+
*/
|
|
32
|
+
debug?: DebugOption;
|
|
27
33
|
}
|
|
28
34
|
/**
|
|
29
35
|
* ElevenLabs conversation mode
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tanstack/ai-elevenlabs",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "ElevenLabs adapter for TanStack AI realtime voice",
|
|
5
5
|
"author": "",
|
|
6
6
|
"license": "MIT",
|
|
@@ -34,13 +34,13 @@
|
|
|
34
34
|
"@11labs/client": "^0.2.0"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"@tanstack/ai": "^0.
|
|
38
|
-
"@tanstack/ai-client": "^0.7.
|
|
37
|
+
"@tanstack/ai": "^0.13.0",
|
|
38
|
+
"@tanstack/ai-client": "^0.7.14"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@vitest/coverage-v8": "4.0.14",
|
|
42
|
-
"@tanstack/ai": "0.
|
|
43
|
-
"@tanstack/ai-client": "0.7.
|
|
42
|
+
"@tanstack/ai": "0.13.0",
|
|
43
|
+
"@tanstack/ai-client": "0.7.14"
|
|
44
44
|
},
|
|
45
45
|
"scripts": {
|
|
46
46
|
"build": "vite build",
|
package/src/realtime/adapter.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Conversation } from '@11labs/client'
|
|
2
|
+
import { resolveDebugOption } from '@tanstack/ai/adapter-internals'
|
|
2
3
|
import type {
|
|
3
4
|
AnyClientTool,
|
|
4
5
|
AudioVisualization,
|
|
@@ -10,6 +11,7 @@ import type {
|
|
|
10
11
|
RealtimeStatus,
|
|
11
12
|
RealtimeToken,
|
|
12
13
|
} from '@tanstack/ai'
|
|
14
|
+
import type { InternalLogger } from '@tanstack/ai/adapter-internals'
|
|
13
15
|
import type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai-client'
|
|
14
16
|
import type { ElevenLabsRealtimeOptions } from './types'
|
|
15
17
|
|
|
@@ -35,6 +37,8 @@ import type { ElevenLabsRealtimeOptions } from './types'
|
|
|
35
37
|
export function elevenlabsRealtime(
|
|
36
38
|
options: ElevenLabsRealtimeOptions = {},
|
|
37
39
|
): RealtimeAdapter {
|
|
40
|
+
const logger = resolveDebugOption(options.debug)
|
|
41
|
+
|
|
38
42
|
return {
|
|
39
43
|
provider: 'elevenlabs',
|
|
40
44
|
|
|
@@ -42,7 +46,10 @@ export function elevenlabsRealtime(
|
|
|
42
46
|
token: RealtimeToken,
|
|
43
47
|
clientToolDefs?: ReadonlyArray<AnyClientTool>,
|
|
44
48
|
): Promise<RealtimeConnection> {
|
|
45
|
-
|
|
49
|
+
logger.request(`activity=realtime provider=elevenlabs`, {
|
|
50
|
+
provider: 'elevenlabs',
|
|
51
|
+
})
|
|
52
|
+
return createElevenLabsConnection(token, options, logger, clientToolDefs)
|
|
46
53
|
},
|
|
47
54
|
}
|
|
48
55
|
}
|
|
@@ -53,6 +60,7 @@ export function elevenlabsRealtime(
|
|
|
53
60
|
async function createElevenLabsConnection(
|
|
54
61
|
token: RealtimeToken,
|
|
55
62
|
_options: ElevenLabsRealtimeOptions,
|
|
63
|
+
logger: InternalLogger,
|
|
56
64
|
clientToolDefs?: ReadonlyArray<AnyClientTool>,
|
|
57
65
|
): Promise<RealtimeConnection> {
|
|
58
66
|
const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()
|
|
@@ -108,22 +116,34 @@ async function createElevenLabsConnection(
|
|
|
108
116
|
signedUrl: token.token,
|
|
109
117
|
|
|
110
118
|
onConnect: () => {
|
|
119
|
+
logger.provider(`provider=elevenlabs direction=in type=connect`, {
|
|
120
|
+
frame: { type: 'connect' },
|
|
121
|
+
})
|
|
111
122
|
emit('status_change', { status: 'connected' as RealtimeStatus })
|
|
112
123
|
emit('mode_change', { mode: 'listening' })
|
|
113
124
|
},
|
|
114
125
|
|
|
115
126
|
onDisconnect: () => {
|
|
127
|
+
logger.provider(`provider=elevenlabs direction=in type=disconnect`, {
|
|
128
|
+
frame: { type: 'disconnect' },
|
|
129
|
+
})
|
|
116
130
|
emit('status_change', { status: 'idle' as RealtimeStatus })
|
|
117
131
|
emit('mode_change', { mode: 'idle' })
|
|
118
132
|
},
|
|
119
133
|
|
|
120
134
|
onModeChange: ({ mode }: { mode: string }) => {
|
|
135
|
+
logger.provider(`provider=elevenlabs direction=in type=mode_change`, {
|
|
136
|
+
frame: { type: 'mode_change', mode },
|
|
137
|
+
})
|
|
121
138
|
const mappedMode: RealtimeMode =
|
|
122
139
|
mode === 'speaking' ? 'speaking' : 'listening'
|
|
123
140
|
emit('mode_change', { mode: mappedMode })
|
|
124
141
|
},
|
|
125
142
|
|
|
126
143
|
onMessage: ({ message, source }: { message: string; source: string }) => {
|
|
144
|
+
logger.provider(`provider=elevenlabs direction=in type=message`, {
|
|
145
|
+
frame: { type: 'message', source, message },
|
|
146
|
+
})
|
|
127
147
|
const role = source === 'user' ? 'user' : 'assistant'
|
|
128
148
|
|
|
129
149
|
if (role === 'user') {
|
|
@@ -148,6 +168,10 @@ async function createElevenLabsConnection(
|
|
|
148
168
|
},
|
|
149
169
|
|
|
150
170
|
onError: (error: string | Error) => {
|
|
171
|
+
logger.errors('elevenlabs.realtime fatal', {
|
|
172
|
+
error,
|
|
173
|
+
source: 'elevenlabs.realtime',
|
|
174
|
+
})
|
|
151
175
|
emit('error', {
|
|
152
176
|
error: new Error(
|
|
153
177
|
typeof error === 'string' ? error : error.message || 'Unknown error',
|
|
@@ -162,9 +186,17 @@ async function createElevenLabsConnection(
|
|
|
162
186
|
}
|
|
163
187
|
|
|
164
188
|
// Start the conversation session
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
189
|
+
try {
|
|
190
|
+
conversation = await Conversation.startSession(
|
|
191
|
+
sessionOptions as Parameters<typeof Conversation.startSession>[0],
|
|
192
|
+
)
|
|
193
|
+
} catch (error) {
|
|
194
|
+
logger.errors('elevenlabs.realtime fatal', {
|
|
195
|
+
error,
|
|
196
|
+
source: 'elevenlabs.realtime',
|
|
197
|
+
})
|
|
198
|
+
throw error
|
|
199
|
+
}
|
|
168
200
|
|
|
169
201
|
// Connection implementation
|
|
170
202
|
const connection: RealtimeConnection = {
|
|
@@ -189,14 +221,17 @@ async function createElevenLabsConnection(
|
|
|
189
221
|
|
|
190
222
|
sendText(text: string) {
|
|
191
223
|
if (!conversation) return
|
|
224
|
+
logger.provider(`provider=elevenlabs direction=out type=user_message`, {
|
|
225
|
+
frame: { type: 'user_message', text },
|
|
226
|
+
})
|
|
192
227
|
conversation.sendUserMessage(text)
|
|
193
228
|
},
|
|
194
229
|
|
|
195
230
|
sendImage(_imageData: string, _mimeType: string) {
|
|
196
231
|
// ElevenLabs does not support direct image input in the conversation API
|
|
197
|
-
|
|
198
|
-
'
|
|
199
|
-
)
|
|
232
|
+
logger.errors('elevenlabs.realtime sendImage not supported', {
|
|
233
|
+
source: 'elevenlabs.realtime',
|
|
234
|
+
})
|
|
200
235
|
},
|
|
201
236
|
|
|
202
237
|
sendToolResult(_callId: string, _result: string) {
|
|
@@ -206,9 +241,9 @@ async function createElevenLabsConnection(
|
|
|
206
241
|
|
|
207
242
|
updateSession(_config: Partial<RealtimeSessionConfig>) {
|
|
208
243
|
// ElevenLabs session config is set at creation time
|
|
209
|
-
|
|
210
|
-
'
|
|
211
|
-
)
|
|
244
|
+
logger.errors('elevenlabs.realtime updateSession not supported', {
|
|
245
|
+
source: 'elevenlabs.realtime',
|
|
246
|
+
})
|
|
212
247
|
},
|
|
213
248
|
|
|
214
249
|
interrupt() {
|
package/src/realtime/types.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { DebugOption } from '@tanstack/ai'
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Options for the ElevenLabs realtime token adapter
|
|
3
5
|
*/
|
|
@@ -23,8 +25,13 @@ export interface ElevenLabsRealtimeTokenOptions {
|
|
|
23
25
|
export interface ElevenLabsRealtimeOptions {
|
|
24
26
|
/** Connection mode (default: auto-detect) */
|
|
25
27
|
connectionMode?: 'websocket' | 'webrtc'
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
+
/**
|
|
29
|
+
* Enable debug logging for this adapter.
|
|
30
|
+
*
|
|
31
|
+
* - `true` enables all categories (`request`, `response`, `provider`, `errors`).
|
|
32
|
+
* - A {@link DebugConfig} object selects categories and/or a custom sink.
|
|
33
|
+
*/
|
|
34
|
+
debug?: DebugOption
|
|
28
35
|
}
|
|
29
36
|
|
|
30
37
|
/**
|