@tanstack/ai 0.34.1 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/client.d.ts +1 -1
- package/dist/esm/client.js.map +1 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/realtime/types.d.ts +48 -0
- package/package.json +1 -1
- package/src/client.ts +2 -0
- package/src/index.ts +2 -0
- package/src/realtime/types.ts +77 -0
package/dist/esm/client.d.ts
CHANGED
|
@@ -41,4 +41,4 @@ export type { ChunkRecording, ChunkStrategy, InternalToolCallState, JSONParser,
|
|
|
41
41
|
export { uiMessagesToWire } from './utilities/ag-ui-wire.js';
|
|
42
42
|
export type { WireMessage } from './utilities/ag-ui-wire.js';
|
|
43
43
|
export type { AudioPart, ContentPart, ContentPartDataSource, ContentPartSource, ContentPartUrlSource, CustomEvent, DocumentPart, ImagePart, MediaInputMetadata, MediaInputRole, MediaPrompt, MediaPromptPart, MessagePart, ModelMessage, RunErrorEvent, RunFinishedEvent, SchemaInput, StreamChunk, StructuredOutputPart, TextPart, ThinkingPart, ToolCall, ToolCallPart, ToolResultPart, UIMessage, VideoPart, InferSchemaType, } from './types.js';
|
|
44
|
-
export type { AudioVisualization, RealtimeError, RealtimeErrorCode, RealtimeEvent, RealtimeEventHandler, RealtimeEventPayloads, RealtimeMessage, RealtimeMessagePart, RealtimeMode, RealtimeSessionConfig, RealtimeStatus, RealtimeToken, RealtimeAudioPart, RealtimeImagePart, RealtimeTextPart, RealtimeToolCallPart, RealtimeToolResultPart, VADConfig, } from './realtime/types.js';
|
|
44
|
+
export type { AudioVisualization, RealtimeAdapter, RealtimeConnection, RealtimeError, RealtimeErrorCode, RealtimeEvent, RealtimeEventHandler, RealtimeEventPayloads, RealtimeMessage, RealtimeMessagePart, RealtimeMode, RealtimeSessionConfig, RealtimeStatus, RealtimeToken, RealtimeAudioPart, RealtimeImagePart, RealtimeTextPart, RealtimeToolCallPart, RealtimeToolResultPart, VADConfig, } from './realtime/types.js';
|
package/dist/esm/client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","sources":["../../src/client.ts"],"sourcesContent":["export enum EventType {\n TEXT_MESSAGE_START = 'TEXT_MESSAGE_START',\n TEXT_MESSAGE_CONTENT = 'TEXT_MESSAGE_CONTENT',\n TEXT_MESSAGE_END = 'TEXT_MESSAGE_END',\n TEXT_MESSAGE_CHUNK = 'TEXT_MESSAGE_CHUNK',\n TOOL_CALL_START = 'TOOL_CALL_START',\n TOOL_CALL_ARGS = 'TOOL_CALL_ARGS',\n TOOL_CALL_END = 'TOOL_CALL_END',\n TOOL_CALL_CHUNK = 'TOOL_CALL_CHUNK',\n TOOL_CALL_RESULT = 'TOOL_CALL_RESULT',\n THINKING_START = 'THINKING_START',\n THINKING_END = 'THINKING_END',\n THINKING_TEXT_MESSAGE_START = 'THINKING_TEXT_MESSAGE_START',\n THINKING_TEXT_MESSAGE_CONTENT = 'THINKING_TEXT_MESSAGE_CONTENT',\n THINKING_TEXT_MESSAGE_END = 'THINKING_TEXT_MESSAGE_END',\n STATE_SNAPSHOT = 'STATE_SNAPSHOT',\n STATE_DELTA = 'STATE_DELTA',\n MESSAGES_SNAPSHOT = 'MESSAGES_SNAPSHOT',\n ACTIVITY_SNAPSHOT = 'ACTIVITY_SNAPSHOT',\n ACTIVITY_DELTA = 'ACTIVITY_DELTA',\n RAW = 'RAW',\n CUSTOM = 'CUSTOM',\n RUN_STARTED = 'RUN_STARTED',\n RUN_FINISHED = 'RUN_FINISHED',\n RUN_ERROR = 'RUN_ERROR',\n STEP_STARTED = 'STEP_STARTED',\n STEP_FINISHED = 'STEP_FINISHED',\n REASONING_START = 'REASONING_START',\n REASONING_MESSAGE_START = 'REASONING_MESSAGE_START',\n REASONING_MESSAGE_CONTENT = 'REASONING_MESSAGE_CONTENT',\n REASONING_MESSAGE_END = 'REASONING_MESSAGE_END',\n REASONING_MESSAGE_CHUNK = 'REASONING_MESSAGE_CHUNK',\n REASONING_END = 'REASONING_END',\n REASONING_ENCRYPTED_VALUE = 'REASONING_ENCRYPTED_VALUE',\n}\n\nexport {\n toolDefinition,\n type AnyClientTool,\n type ClientTool,\n type InferToolInput,\n type InferToolName,\n type InferToolOutput,\n type ToolDefinition,\n type ToolDefinitionConfig,\n type ToolDefinitionInstance,\n} from './activities/chat/tools/tool-definition'\n\nexport {\n convertSchemaToJsonSchema,\n isStandardSchema,\n parseWithStandardSchema,\n} from './activities/chat/tools/schema-converter'\n\nexport {\n convertMessagesToModelMessages,\n generateMessageId,\n modelMessageToUIMessage,\n modelMessagesToUIMessages,\n normalizeToUIMessage,\n uiMessageToModelMessages,\n} from './activities/chat/messages'\n\nexport {\n BatchStrategy,\n CompositeStrategy,\n defaultJSONParser,\n ImmediateStrategy,\n parsePartialJSON,\n PartialJSONParser,\n PunctuationStrategy,\n StreamProcessor,\n WordBoundaryStrategy,\n} from './activities/chat/stream/index'\nexport type {\n ChunkRecording,\n ChunkStrategy,\n InternalToolCallState,\n JSONParser,\n ProcessorResult,\n ProcessorState,\n StreamProcessorEvents,\n StreamProcessorOptions,\n ToolCallState,\n ToolResultState,\n} from './activities/chat/stream/index'\n\nexport { uiMessagesToWire } from './utilities/ag-ui-wire'\nexport type { WireMessage } from './utilities/ag-ui-wire'\n\nexport type {\n AudioPart,\n ContentPart,\n ContentPartDataSource,\n ContentPartSource,\n ContentPartUrlSource,\n CustomEvent,\n DocumentPart,\n ImagePart,\n MediaInputMetadata,\n MediaInputRole,\n MediaPrompt,\n MediaPromptPart,\n MessagePart,\n ModelMessage,\n RunErrorEvent,\n RunFinishedEvent,\n SchemaInput,\n StreamChunk,\n StructuredOutputPart,\n TextPart,\n ThinkingPart,\n ToolCall,\n ToolCallPart,\n ToolResultPart,\n UIMessage,\n VideoPart,\n InferSchemaType,\n} from './types'\n\nexport type {\n AudioVisualization,\n RealtimeError,\n RealtimeErrorCode,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeEventPayloads,\n RealtimeMessage,\n RealtimeMessagePart,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n RealtimeAudioPart,\n RealtimeImagePart,\n RealtimeTextPart,\n RealtimeToolCallPart,\n RealtimeToolResultPart,\n VADConfig,\n} from './realtime/types'\n"],"names":["EventType"],"mappings":";;;;;;;AAAO,IAAK,8BAAAA,eAAL;AACLA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,sBAAA,IAAuB;AACvBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,cAAA,IAAe;AACfA,aAAA,6BAAA,IAA8B;AAC9BA,aAAA,+BAAA,IAAgC;AAChCA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,aAAA,IAAc;AACdA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,KAAA,IAAM;AACNA,aAAA,QAAA,IAAS;AACTA,aAAA,aAAA,IAAc;AACdA,aAAA,cAAA,IAAe;AACfA,aAAA,WAAA,IAAY;AACZA,aAAA,cAAA,IAAe;AACfA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,uBAAA,IAAwB;AACxBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,eAAA,IAAgB;AAChBA,aAAA,2BAAA,IAA4B;AAjClB,SAAAA;AAAA,GAAA,aAAA,CAAA,CAAA;"}
|
|
1
|
+
{"version":3,"file":"client.js","sources":["../../src/client.ts"],"sourcesContent":["export enum EventType {\n TEXT_MESSAGE_START = 'TEXT_MESSAGE_START',\n TEXT_MESSAGE_CONTENT = 'TEXT_MESSAGE_CONTENT',\n TEXT_MESSAGE_END = 'TEXT_MESSAGE_END',\n TEXT_MESSAGE_CHUNK = 'TEXT_MESSAGE_CHUNK',\n TOOL_CALL_START = 'TOOL_CALL_START',\n TOOL_CALL_ARGS = 'TOOL_CALL_ARGS',\n TOOL_CALL_END = 'TOOL_CALL_END',\n TOOL_CALL_CHUNK = 'TOOL_CALL_CHUNK',\n TOOL_CALL_RESULT = 'TOOL_CALL_RESULT',\n THINKING_START = 'THINKING_START',\n THINKING_END = 'THINKING_END',\n THINKING_TEXT_MESSAGE_START = 'THINKING_TEXT_MESSAGE_START',\n THINKING_TEXT_MESSAGE_CONTENT = 'THINKING_TEXT_MESSAGE_CONTENT',\n THINKING_TEXT_MESSAGE_END = 'THINKING_TEXT_MESSAGE_END',\n STATE_SNAPSHOT = 'STATE_SNAPSHOT',\n STATE_DELTA = 'STATE_DELTA',\n MESSAGES_SNAPSHOT = 'MESSAGES_SNAPSHOT',\n ACTIVITY_SNAPSHOT = 'ACTIVITY_SNAPSHOT',\n ACTIVITY_DELTA = 'ACTIVITY_DELTA',\n RAW = 'RAW',\n CUSTOM = 'CUSTOM',\n RUN_STARTED = 'RUN_STARTED',\n RUN_FINISHED = 'RUN_FINISHED',\n RUN_ERROR = 'RUN_ERROR',\n STEP_STARTED = 'STEP_STARTED',\n STEP_FINISHED = 'STEP_FINISHED',\n REASONING_START = 'REASONING_START',\n REASONING_MESSAGE_START = 'REASONING_MESSAGE_START',\n REASONING_MESSAGE_CONTENT = 'REASONING_MESSAGE_CONTENT',\n REASONING_MESSAGE_END = 'REASONING_MESSAGE_END',\n REASONING_MESSAGE_CHUNK = 'REASONING_MESSAGE_CHUNK',\n REASONING_END = 'REASONING_END',\n REASONING_ENCRYPTED_VALUE = 'REASONING_ENCRYPTED_VALUE',\n}\n\nexport {\n toolDefinition,\n type AnyClientTool,\n type ClientTool,\n type InferToolInput,\n type InferToolName,\n type InferToolOutput,\n type ToolDefinition,\n type ToolDefinitionConfig,\n type ToolDefinitionInstance,\n} from './activities/chat/tools/tool-definition'\n\nexport {\n convertSchemaToJsonSchema,\n isStandardSchema,\n parseWithStandardSchema,\n} from './activities/chat/tools/schema-converter'\n\nexport {\n convertMessagesToModelMessages,\n generateMessageId,\n modelMessageToUIMessage,\n modelMessagesToUIMessages,\n normalizeToUIMessage,\n uiMessageToModelMessages,\n} from './activities/chat/messages'\n\nexport {\n BatchStrategy,\n CompositeStrategy,\n defaultJSONParser,\n ImmediateStrategy,\n parsePartialJSON,\n PartialJSONParser,\n PunctuationStrategy,\n StreamProcessor,\n WordBoundaryStrategy,\n} from './activities/chat/stream/index'\nexport type {\n ChunkRecording,\n ChunkStrategy,\n InternalToolCallState,\n JSONParser,\n ProcessorResult,\n ProcessorState,\n StreamProcessorEvents,\n StreamProcessorOptions,\n ToolCallState,\n ToolResultState,\n} from './activities/chat/stream/index'\n\nexport { uiMessagesToWire } from './utilities/ag-ui-wire'\nexport type { WireMessage } from './utilities/ag-ui-wire'\n\nexport type {\n AudioPart,\n ContentPart,\n ContentPartDataSource,\n ContentPartSource,\n ContentPartUrlSource,\n CustomEvent,\n DocumentPart,\n ImagePart,\n MediaInputMetadata,\n MediaInputRole,\n MediaPrompt,\n MediaPromptPart,\n MessagePart,\n ModelMessage,\n RunErrorEvent,\n RunFinishedEvent,\n SchemaInput,\n StreamChunk,\n StructuredOutputPart,\n TextPart,\n ThinkingPart,\n ToolCall,\n ToolCallPart,\n ToolResultPart,\n UIMessage,\n VideoPart,\n InferSchemaType,\n} from './types'\n\nexport type {\n AudioVisualization,\n RealtimeAdapter,\n RealtimeConnection,\n RealtimeError,\n RealtimeErrorCode,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeEventPayloads,\n RealtimeMessage,\n RealtimeMessagePart,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n RealtimeAudioPart,\n RealtimeImagePart,\n RealtimeTextPart,\n RealtimeToolCallPart,\n RealtimeToolResultPart,\n VADConfig,\n} from './realtime/types'\n"],"names":["EventType"],"mappings":";;;;;;;AAAO,IAAK,8BAAAA,eAAL;AACLA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,sBAAA,IAAuB;AACvBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,oBAAA,IAAqB;AACrBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,kBAAA,IAAmB;AACnBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,cAAA,IAAe;AACfA,aAAA,6BAAA,IAA8B;AAC9BA,aAAA,+BAAA,IAAgC;AAChCA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,aAAA,IAAc;AACdA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,mBAAA,IAAoB;AACpBA,aAAA,gBAAA,IAAiB;AACjBA,aAAA,KAAA,IAAM;AACNA,aAAA,QAAA,IAAS;AACTA,aAAA,aAAA,IAAc;AACdA,aAAA,cAAA,IAAe;AACfA,aAAA,WAAA,IAAY;AACZA,aAAA,cAAA,IAAe;AACfA,aAAA,eAAA,IAAgB;AAChBA,aAAA,iBAAA,IAAkB;AAClBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,2BAAA,IAA4B;AAC5BA,aAAA,uBAAA,IAAwB;AACxBA,aAAA,yBAAA,IAA0B;AAC1BA,aAAA,eAAA,IAAgB;AAChBA,aAAA,2BAAA,IAA4B;AAjClB,SAAAA;AAAA,GAAA,aAAA,CAAA,CAAA;"}
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -30,7 +30,7 @@ export type { SystemPrompt, NormalizedSystemPrompt } from './system-prompts.js';
|
|
|
30
30
|
export { normalizeSystemPrompts } from './system-prompts.js';
|
|
31
31
|
export { detectImageMimeType } from './utils.js';
|
|
32
32
|
export { realtimeToken } from './realtime/index.js';
|
|
33
|
-
export type { RealtimeToken, RealtimeTokenAdapter, RealtimeTokenOptions, RealtimeSessionConfig, VADConfig, RealtimeMessage, RealtimeMessagePart, RealtimeTextPart, RealtimeAudioPart, RealtimeToolCallPart, RealtimeToolResultPart, RealtimeImagePart, RealtimeStatus, RealtimeMode, AudioVisualization, RealtimeEvent, RealtimeEventPayloads, RealtimeEventHandler, RealtimeErrorCode, RealtimeError, } from './realtime/index.js';
|
|
33
|
+
export type { RealtimeToken, RealtimeTokenAdapter, RealtimeTokenOptions, RealtimeSessionConfig, VADConfig, RealtimeMessage, RealtimeMessagePart, RealtimeTextPart, RealtimeAudioPart, RealtimeToolCallPart, RealtimeToolResultPart, RealtimeImagePart, RealtimeStatus, RealtimeMode, AudioVisualization, RealtimeEvent, RealtimeEventPayloads, RealtimeEventHandler, RealtimeErrorCode, RealtimeError, RealtimeAdapter, RealtimeConnection, } from './realtime/index.js';
|
|
34
34
|
export { convertMessagesToModelMessages, generateMessageId, uiMessageToModelMessages, modelMessageToUIMessage, modelMessagesToUIMessages, normalizeToUIMessage, } from './activities/chat/messages.js';
|
|
35
35
|
export { StreamProcessor, createReplayStream, ImmediateStrategy, PunctuationStrategy, BatchStrategy, WordBoundaryStrategy, CompositeStrategy, PartialJSONParser, defaultJSONParser, parsePartialJSON, } from './activities/chat/stream/index.js';
|
|
36
36
|
export type { ChunkStrategy, ChunkRecording, InternalToolCallState, ProcessorResult, ProcessorState, StreamProcessorEvents, StreamProcessorOptions, ToolCallState, ToolResultState, JSONParser, } from './activities/chat/stream/index.js';
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AnyClientTool } from '../activities/chat/tools/tool-definition.js';
|
|
1
2
|
/**
|
|
2
3
|
* Voice activity detection configuration
|
|
3
4
|
*/
|
|
@@ -232,3 +233,50 @@ export interface RealtimeError extends Error {
|
|
|
232
233
|
provider?: string;
|
|
233
234
|
details?: unknown;
|
|
234
235
|
}
|
|
236
|
+
/**
|
|
237
|
+
* Adapter interface for connecting to realtime providers.
|
|
238
|
+
* Each provider (OpenAI, ElevenLabs, etc.) implements this interface.
|
|
239
|
+
*
|
|
240
|
+
* Defined here in `@tanstack/ai` — the shared layer that both provider adapter
|
|
241
|
+
* packages and the client runtime (`@tanstack/ai-client`) already depend on —
|
|
242
|
+
* so a provider package can describe its realtime adapter without taking a
|
|
243
|
+
* dependency on the client-only `@tanstack/ai-client`. `@tanstack/ai-client`
|
|
244
|
+
* re-exports this type for backwards compatibility.
|
|
245
|
+
*/
|
|
246
|
+
export interface RealtimeAdapter {
|
|
247
|
+
/** Provider identifier */
|
|
248
|
+
provider: string;
|
|
249
|
+
/**
|
|
250
|
+
* Create a connection using the provided token
|
|
251
|
+
* @param token - The ephemeral token from the server
|
|
252
|
+
* @param clientTools - Optional client-side tools to register with the provider
|
|
253
|
+
* @returns A connection instance
|
|
254
|
+
*/
|
|
255
|
+
connect: (token: RealtimeToken, clientTools?: ReadonlyArray<AnyClientTool>) => Promise<RealtimeConnection>;
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Connection interface representing an active realtime session.
|
|
259
|
+
* Handles audio I/O, events, and session management.
|
|
260
|
+
*/
|
|
261
|
+
export interface RealtimeConnection {
|
|
262
|
+
/** Disconnect from the realtime session */
|
|
263
|
+
disconnect: () => Promise<void>;
|
|
264
|
+
/** Start capturing audio from the microphone */
|
|
265
|
+
startAudioCapture: () => Promise<void>;
|
|
266
|
+
/** Stop capturing audio */
|
|
267
|
+
stopAudioCapture: () => void;
|
|
268
|
+
/** Send a text message (fallback for when voice isn't available) */
|
|
269
|
+
sendText: (text: string) => void;
|
|
270
|
+
/** Send an image to the conversation */
|
|
271
|
+
sendImage: (imageData: string, mimeType: string) => void;
|
|
272
|
+
/** Send a tool execution result back to the provider */
|
|
273
|
+
sendToolResult: (callId: string, result: string) => void;
|
|
274
|
+
/** Update session configuration */
|
|
275
|
+
updateSession: (config: Partial<RealtimeSessionConfig>) => void;
|
|
276
|
+
/** Interrupt the current response */
|
|
277
|
+
interrupt: () => void;
|
|
278
|
+
/** Subscribe to connection events */
|
|
279
|
+
on: <TEvent extends RealtimeEvent>(event: TEvent, handler: RealtimeEventHandler<TEvent>) => () => void;
|
|
280
|
+
/** Get audio visualization data */
|
|
281
|
+
getAudioVisualization: () => AudioVisualization;
|
|
282
|
+
}
|
package/package.json
CHANGED
package/src/client.ts
CHANGED
package/src/index.ts
CHANGED
package/src/realtime/types.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { AnyClientTool } from '../activities/chat/tools/tool-definition'
|
|
2
|
+
|
|
1
3
|
// ============================================================================
|
|
2
4
|
// Token Types
|
|
3
5
|
// ============================================================================
|
|
@@ -292,3 +294,78 @@ export interface RealtimeError extends Error {
|
|
|
292
294
|
provider?: string
|
|
293
295
|
details?: unknown
|
|
294
296
|
}
|
|
297
|
+
|
|
298
|
+
// ============================================================================
|
|
299
|
+
// Adapter Contract
|
|
300
|
+
// ============================================================================
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Adapter interface for connecting to realtime providers.
|
|
304
|
+
* Each provider (OpenAI, ElevenLabs, etc.) implements this interface.
|
|
305
|
+
*
|
|
306
|
+
* Defined here in `@tanstack/ai` — the shared layer that both provider adapter
|
|
307
|
+
* packages and the client runtime (`@tanstack/ai-client`) already depend on —
|
|
308
|
+
* so a provider package can describe its realtime adapter without taking a
|
|
309
|
+
* dependency on the client-only `@tanstack/ai-client`. `@tanstack/ai-client`
|
|
310
|
+
* re-exports this type for backwards compatibility.
|
|
311
|
+
*/
|
|
312
|
+
export interface RealtimeAdapter {
|
|
313
|
+
/** Provider identifier */
|
|
314
|
+
provider: string
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Create a connection using the provided token
|
|
318
|
+
* @param token - The ephemeral token from the server
|
|
319
|
+
* @param clientTools - Optional client-side tools to register with the provider
|
|
320
|
+
* @returns A connection instance
|
|
321
|
+
*/
|
|
322
|
+
connect: (
|
|
323
|
+
token: RealtimeToken,
|
|
324
|
+
clientTools?: ReadonlyArray<AnyClientTool>,
|
|
325
|
+
) => Promise<RealtimeConnection>
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Connection interface representing an active realtime session.
|
|
330
|
+
* Handles audio I/O, events, and session management.
|
|
331
|
+
*/
|
|
332
|
+
export interface RealtimeConnection {
|
|
333
|
+
// Lifecycle
|
|
334
|
+
/** Disconnect from the realtime session */
|
|
335
|
+
disconnect: () => Promise<void>
|
|
336
|
+
|
|
337
|
+
// Audio I/O
|
|
338
|
+
/** Start capturing audio from the microphone */
|
|
339
|
+
startAudioCapture: () => Promise<void>
|
|
340
|
+
/** Stop capturing audio */
|
|
341
|
+
stopAudioCapture: () => void
|
|
342
|
+
|
|
343
|
+
// Text input
|
|
344
|
+
/** Send a text message (fallback for when voice isn't available) */
|
|
345
|
+
sendText: (text: string) => void
|
|
346
|
+
|
|
347
|
+
// Image input
|
|
348
|
+
/** Send an image to the conversation */
|
|
349
|
+
sendImage: (imageData: string, mimeType: string) => void
|
|
350
|
+
|
|
351
|
+
// Tool results
|
|
352
|
+
/** Send a tool execution result back to the provider */
|
|
353
|
+
sendToolResult: (callId: string, result: string) => void
|
|
354
|
+
|
|
355
|
+
// Session management
|
|
356
|
+
/** Update session configuration */
|
|
357
|
+
updateSession: (config: Partial<RealtimeSessionConfig>) => void
|
|
358
|
+
/** Interrupt the current response */
|
|
359
|
+
interrupt: () => void
|
|
360
|
+
|
|
361
|
+
// Events
|
|
362
|
+
/** Subscribe to connection events */
|
|
363
|
+
on: <TEvent extends RealtimeEvent>(
|
|
364
|
+
event: TEvent,
|
|
365
|
+
handler: RealtimeEventHandler<TEvent>,
|
|
366
|
+
) => () => void
|
|
367
|
+
|
|
368
|
+
// Audio visualization
|
|
369
|
+
/** Get audio visualization data */
|
|
370
|
+
getAudioVisualization: () => AudioVisualization
|
|
371
|
+
}
|