@tanstack/ai 0.39.1 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/esm/activities/chat/index.d.ts +2 -2
  2. package/dist/esm/activities/chat/index.js +20 -3
  3. package/dist/esm/activities/chat/index.js.map +1 -1
  4. package/dist/esm/activities/chat/messages.js +7 -0
  5. package/dist/esm/activities/chat/messages.js.map +1 -1
  6. package/dist/esm/activities/chat/middleware/compose.d.ts +2 -2
  7. package/dist/esm/activities/chat/middleware/compose.js.map +1 -1
  8. package/dist/esm/activities/chat/middleware/index.d.ts +1 -1
  9. package/dist/esm/activities/chat/middleware/sandbox-runtime.d.ts +7 -2
  10. package/dist/esm/activities/chat/middleware/sandbox-runtime.js.map +1 -1
  11. package/dist/esm/activities/chat/middleware/types.d.ts +16 -4
  12. package/dist/esm/activities/chat/stream/message-updaters.d.ts +2 -0
  13. package/dist/esm/activities/chat/stream/message-updaters.js +3 -1
  14. package/dist/esm/activities/chat/stream/message-updaters.js.map +1 -1
  15. package/dist/esm/activities/chat/stream/processor.js +18 -1
  16. package/dist/esm/activities/chat/stream/processor.js.map +1 -1
  17. package/dist/esm/activities/chat/tools/tool-definition.d.ts +14 -11
  18. package/dist/esm/activities/chat/tools/tool-definition.js.map +1 -1
  19. package/dist/esm/activities/generateAudio/index.d.ts +1 -1
  20. package/dist/esm/activities/generateAudio/index.js.map +1 -1
  21. package/dist/esm/activities/generateImage/index.d.ts +1 -1
  22. package/dist/esm/activities/generateImage/index.js +1 -1
  23. package/dist/esm/activities/generateImage/index.js.map +1 -1
  24. package/dist/esm/activities/generateSpeech/index.d.ts +1 -1
  25. package/dist/esm/activities/generateSpeech/index.js.map +1 -1
  26. package/dist/esm/activities/generateTranscription/index.d.ts +3 -3
  27. package/dist/esm/activities/generateTranscription/index.js.map +1 -1
  28. package/dist/esm/activities/generateVideo/index.d.ts +12 -12
  29. package/dist/esm/activities/generateVideo/index.js.map +1 -1
  30. package/dist/esm/index.d.ts +3 -3
  31. package/dist/esm/index.js +2 -0
  32. package/dist/esm/index.js.map +1 -1
  33. package/dist/esm/realtime/event-emitter.d.ts +5 -0
  34. package/dist/esm/realtime/event-emitter.js +27 -0
  35. package/dist/esm/realtime/event-emitter.js.map +1 -0
  36. package/dist/esm/realtime/index.d.ts +1 -0
  37. package/dist/esm/realtime/index.js.map +1 -1
  38. package/dist/esm/realtime/types.d.ts +9 -1
  39. package/dist/esm/types.d.ts +124 -1
  40. package/package.json +1 -1
  41. package/skills/ai-core/adapter-configuration/SKILL.md +10 -0
  42. package/skills/ai-core/ag-ui-protocol/SKILL.md +59 -0
  43. package/skills/ai-core/media-generation/SKILL.md +50 -5
  44. package/skills/ai-core/middleware/SKILL.md +106 -0
  45. package/skills/ai-core/tool-calling/SKILL.md +13 -1
  46. package/src/activities/chat/index.ts +26 -6
  47. package/src/activities/chat/messages.ts +9 -0
  48. package/src/activities/chat/middleware/compose.ts +2 -2
  49. package/src/activities/chat/middleware/index.ts +1 -0
  50. package/src/activities/chat/middleware/sandbox-runtime.ts +4 -2
  51. package/src/activities/chat/middleware/types.ts +17 -4
  52. package/src/activities/chat/stream/message-updaters.ts +7 -1
  53. package/src/activities/chat/stream/processor.ts +30 -2
  54. package/src/activities/chat/tools/tool-definition.ts +33 -11
  55. package/src/activities/generateAudio/index.ts +2 -2
  56. package/src/activities/generateImage/index.ts +2 -2
  57. package/src/activities/generateSpeech/index.ts +2 -2
  58. package/src/activities/generateTranscription/index.ts +8 -4
  59. package/src/activities/generateVideo/index.ts +29 -15
  60. package/src/index.ts +3 -1
  61. package/src/realtime/event-emitter.ts +46 -0
  62. package/src/realtime/index.ts +2 -0
  63. package/src/realtime/types.ts +9 -0
  64. package/src/types.ts +116 -1
@@ -45,6 +45,7 @@ import type {
45
45
  import type {
46
46
  AgentLoopStrategy,
47
47
  AnyTool,
48
+ ChatStream,
48
49
  ConstrainedModelMessage,
49
50
  CustomEvent,
50
51
  InferSchemaType,
@@ -69,7 +70,7 @@ import type {
69
70
  ChatMiddleware,
70
71
  ChatMiddlewareConfig,
71
72
  ChatMiddlewareContext,
72
- SandboxFileEvent,
73
+ SandboxFileHookEvent,
73
74
  StructuredOutputMiddlewareConfig,
74
75
  } from './middleware/types'
75
76
  import type { CheckCoverage } from './middleware/builder'
@@ -404,7 +405,7 @@ export type TextActivityResult<
404
405
  : Promise<InferSchemaType<TSchema>>
405
406
  : [TStream] extends [false]
406
407
  ? Promise<string>
407
- : AsyncIterable<StreamChunk>
408
+ : ChatStream
408
409
 
409
410
  // ===========================
410
411
  // ChatEngine Implementation
@@ -712,11 +713,30 @@ class TextEngine<
712
713
  // a `sandbox.file` custom chunk to be drained into the public stream.
713
714
  provideSandboxRuntime(this.middlewareCtx, {
714
715
  logger: this.logger,
715
- emit: (event: SandboxFileEvent) => {
716
- this.logger.sandbox(`file ${event.type} ${event.path}`, { event })
717
- void this.middlewareRunner.runSandboxFile(this.middlewareCtx, event)
716
+ emit: (event: SandboxFileHookEvent) => {
717
+ this.logger.sandbox(`file ${event.type} ${event.path}`, {
718
+ event: {
719
+ type: event.type,
720
+ path: event.path,
721
+ timestamp: event.timestamp,
722
+ },
723
+ })
724
+ void this.middlewareRunner
725
+ .runSandboxFile(this.middlewareCtx, event)
726
+ .catch((err: unknown) => {
727
+ this.logger.errors('sandbox file hook failed', { error: err })
728
+ })
729
+ this.sandboxFileQueue.push(
730
+ this.createCustomEventChunk('sandbox.file', {
731
+ type: event.type,
732
+ path: event.path,
733
+ timestamp: event.timestamp,
734
+ }),
735
+ )
736
+ },
737
+ emitFileDiff: (value: { path: string; diff: string }) => {
718
738
  this.sandboxFileQueue.push(
719
- this.createCustomEventChunk('sandbox.file', { ...event }),
739
+ this.createCustomEventChunk('sandbox.file.diff', value),
720
740
  )
721
741
  },
722
742
  })
@@ -444,12 +444,21 @@ export function modelMessageToUIMessage(
444
444
  // Handle tool calls
445
445
  if (modelMessage.toolCalls && modelMessage.toolCalls.length > 0) {
446
446
  for (const toolCall of modelMessage.toolCalls) {
447
+ // Model-message arguments are complete, so surface the parsed input.
448
+ // A malformed arguments string just leaves `input` undefined.
449
+ let input: unknown
450
+ try {
451
+ input = JSON.parse(toolCall.function.arguments)
452
+ } catch {
453
+ input = undefined
454
+ }
447
455
  parts.push({
448
456
  type: 'tool-call',
449
457
  id: toolCall.id,
450
458
  name: toolCall.function.name,
451
459
  arguments: toolCall.function.arguments,
452
460
  state: 'input-complete', // Model messages have complete arguments
461
+ ...(input !== undefined && { input }),
453
462
  ...(toolCall.metadata !== undefined && { metadata: toolCall.metadata }),
454
463
  })
455
464
  }
@@ -11,7 +11,7 @@ import type {
11
11
  ErrorInfo,
12
12
  FinishInfo,
13
13
  IterationInfo,
14
- SandboxFileEvent,
14
+ SandboxFileHookEvent,
15
15
  StructuredOutputMiddlewareConfig,
16
16
  ToolCallHookContext,
17
17
  ToolPhaseCompleteInfo,
@@ -352,7 +352,7 @@ export class MiddlewareRunner<TContext = unknown> {
352
352
  */
353
353
  async runSandboxFile(
354
354
  ctx: ChatMiddlewareContext<TContext>,
355
- event: SandboxFileEvent,
355
+ event: SandboxFileHookEvent,
356
356
  ): Promise<void> {
357
357
  const typed = (
358
358
  {
@@ -14,6 +14,7 @@ export type {
14
14
  AbortInfo,
15
15
  ErrorInfo,
16
16
  SandboxFileEvent,
17
+ SandboxFileHookEvent,
17
18
  ChatSandboxHooks,
18
19
  } from './types'
19
20
 
@@ -7,10 +7,12 @@
7
7
  */
8
8
  import { createCapability } from './capabilities'
9
9
  import type { InternalLogger } from '../../../logger/internal-logger'
10
- import type { SandboxFileEvent } from './types'
10
+ import type { SandboxFileHookEvent } from './types'
11
11
 
12
12
  export interface SandboxRuntime {
13
- emit: (event: SandboxFileEvent) => void
13
+ emit: (event: SandboxFileHookEvent) => void
14
+ /** Emit an opt-in per-file `sandbox.file.diff` CUSTOM chunk. */
15
+ emitFileDiff: (value: { path: string; diff: string }) => void
14
16
  logger: InternalLogger
15
17
  }
16
18
 
@@ -21,6 +21,19 @@ export interface SandboxFileEvent {
21
21
  timestamp: number
22
22
  }
23
23
 
24
+ /** The file event a sandbox hook receives: the serializable {@link SandboxFileEvent}
25
+ * plus lazy, git-backed content accessors. Accessors compute on call, so a hook
26
+ * that only reads `path`/`type` pays nothing. Never present on the serialized
27
+ * `sandbox.file` CUSTOM chunk. */
28
+ export interface SandboxFileHookEvent extends SandboxFileEvent {
29
+ /** Content at the session baseline (`''` for a new file or non-git workspace). */
30
+ before: () => Promise<string>
31
+ /** Current content (`''` when the event is a delete). */
32
+ after: () => Promise<string>
33
+ /** Unified patch vs the session baseline (synthesized add-patch when non-git). */
34
+ diff: () => Promise<string>
35
+ }
36
+
24
37
  /**
25
38
  * Sandbox file-event hooks a chat middleware can declare. Fire server-side for
26
39
  * every file create/change/delete observed in the sandbox during the run.
@@ -28,19 +41,19 @@ export interface SandboxFileEvent {
28
41
  export interface ChatSandboxHooks<TContext = unknown> {
29
42
  onFile?: (
30
43
  ctx: ChatMiddlewareContext<TContext>,
31
- e: SandboxFileEvent,
44
+ e: SandboxFileHookEvent,
32
45
  ) => void | Promise<void>
33
46
  onFileCreate?: (
34
47
  ctx: ChatMiddlewareContext<TContext>,
35
- e: SandboxFileEvent,
48
+ e: SandboxFileHookEvent,
36
49
  ) => void | Promise<void>
37
50
  onFileChange?: (
38
51
  ctx: ChatMiddlewareContext<TContext>,
39
- e: SandboxFileEvent,
52
+ e: SandboxFileHookEvent,
40
53
  ) => void | Promise<void>
41
54
  onFileDelete?: (
42
55
  ctx: ChatMiddlewareContext<TContext>,
43
- e: SandboxFileEvent,
56
+ e: SandboxFileHookEvent,
44
57
  ) => void | Promise<void>
45
58
  }
46
59
 
@@ -58,6 +58,8 @@ export function updateToolCallPart(
58
58
  name: string
59
59
  arguments: string
60
60
  state: ToolCallState
61
+ /** Parsed input — set when the arguments are complete. */
62
+ input?: unknown
61
63
  metadata?: Record<string, unknown>
62
64
  },
63
65
  ): Array<UIMessage> {
@@ -76,6 +78,9 @@ export function updateToolCallPart(
76
78
  // Gemini's thoughtSignature on TOOL_CALL_START) we must not lose it on
77
79
  // subsequent updates that don't re-supply it.
78
80
  const metadata = toolCall.metadata ?? existing?.metadata
81
+ // Same for the parsed input: it's supplied once at completion, so
82
+ // subsequent arg-less updates (approval, etc.) must not drop it.
83
+ const input = toolCall.input ?? existing?.input
79
84
 
80
85
  const toolCallPart: ToolCallPart = {
81
86
  type: 'tool-call',
@@ -83,9 +88,10 @@ export function updateToolCallPart(
83
88
  name: toolCall.name,
84
89
  arguments: toolCall.arguments,
85
90
  state: toolCall.state,
86
- // Carry forward approval and output from the existing part
91
+ // Carry forward approval, output and parsed input from the existing part
87
92
  ...(existing?.approval && { approval: { ...existing.approval } }),
88
93
  ...(existing?.output !== undefined && { output: existing.output }),
94
+ ...(input !== undefined && { input }),
89
95
  ...(metadata !== undefined && { metadata }),
90
96
  }
91
97
 
@@ -1298,15 +1298,35 @@ export class StreamProcessor {
1298
1298
  // received, back-fill the arguments string so the UIMessage ToolCallPart
1299
1299
  // carries the correct value (defensive against adapters that skip ARGS).
1300
1300
  if (chunk.input !== undefined && !existingToolCall.arguments) {
1301
- existingToolCall.arguments = JSON.stringify(chunk.input)
1301
+ try {
1302
+ existingToolCall.arguments = JSON.stringify(chunk.input)
1303
+ } catch {
1304
+ // circular refs, BigInt, etc. — leave arguments empty rather than
1305
+ // aborting stream processing
1306
+ }
1302
1307
  }
1303
1308
 
1304
1309
  const index = msgState.toolCallOrder.indexOf(chunk.toolCallId)
1305
1310
  this.completeToolCall(messageId, index, existingToolCall)
1306
1311
  // If TOOL_CALL_END provides parsed input, use it as the canonical parsed
1307
1312
  // arguments (overrides the accumulated string parse from completeToolCall)
1313
+ // and refresh the rendered part's `input` so it reflects the canonical
1314
+ // value rather than the possibly-divergent accumulated-args parse that
1315
+ // completeToolCall wrote (e.g. an adapter that coerces values differently
1316
+ // between the streamed args and the final structured input).
1308
1317
  if (chunk.input !== undefined) {
1309
1318
  existingToolCall.parsedArguments = chunk.input
1319
+ this.messages = updateToolCallPart(this.messages, messageId, {
1320
+ id: existingToolCall.id,
1321
+ name: existingToolCall.name,
1322
+ arguments: existingToolCall.arguments,
1323
+ state: 'input-complete',
1324
+ input: chunk.input,
1325
+ ...(existingToolCall.metadata !== undefined && {
1326
+ metadata: existingToolCall.metadata,
1327
+ }),
1328
+ })
1329
+ this.emitMessagesChange()
1310
1330
  }
1311
1331
  }
1312
1332
 
@@ -1890,12 +1910,20 @@ export class StreamProcessor {
1890
1910
  return
1891
1911
  }
1892
1912
 
1893
- // Update UIMessage
1913
+ // Update UIMessage. The arguments are complete now, so surface the parsed
1914
+ // input on the part. For adapters that skip TOOL_CALL_ARGS the arguments
1915
+ // string was back-filled from TOOL_CALL_END.input, so this parse matches
1916
+ // the canonical input. If a TOOL_CALL_END.input diverges from the
1917
+ // accumulated args, handleToolCallEndEvent re-updates the part with the
1918
+ // canonical value after this call.
1894
1919
  this.messages = updateToolCallPart(this.messages, messageId, {
1895
1920
  id: toolCall.id,
1896
1921
  name: toolCall.name,
1897
1922
  arguments: toolCall.arguments,
1898
1923
  state: 'input-complete',
1924
+ ...(toolCall.parsedArguments !== undefined && {
1925
+ input: toolCall.parsedArguments,
1926
+ }),
1899
1927
  ...(toolCall.metadata !== undefined && { metadata: toolCall.metadata }),
1900
1928
  })
1901
1929
  this.emitMessagesChange()
@@ -26,6 +26,10 @@ export interface ClientTool<
26
26
  TOutput extends SchemaInput = SchemaInput,
27
27
  TName extends string = string,
28
28
  TContext = unknown,
29
+ // Captured as a literal (`true` / `false`) so downstream types — notably
30
+ // the tool-call part's `approval` field — can be gated on it. Defaults to
31
+ // `false` when the tool config omits `needsApproval`.
32
+ TNeedsApproval extends boolean = false,
29
33
  > {
30
34
  __toolSide: 'client'
31
35
  name: TName
@@ -37,7 +41,7 @@ export interface ClientTool<
37
41
  // because `undefined` doesn't extend the schema constraint.
38
42
  inputSchema?: TInput
39
43
  outputSchema?: TOutput
40
- needsApproval?: boolean
44
+ needsApproval?: TNeedsApproval
41
45
  lazy?: boolean
42
46
  metadata?: Record<string, unknown>
43
47
  execute?: ToolExecuteFunction<TInput, TOutput, TContext>
@@ -51,18 +55,22 @@ export interface ToolDefinitionInstance<
51
55
  TOutput extends SchemaInput = SchemaInput,
52
56
  TName extends string = string,
53
57
  TContext = unknown,
58
+ TNeedsApproval extends boolean = false,
54
59
  > extends Tool<TInput, TOutput, TName, TContext> {
55
60
  __toolSide: 'definition'
61
+ // Narrow the base `needsApproval?: boolean` to the captured literal so it
62
+ // survives into `ToolCallPartForTool`'s approval gate.
63
+ needsApproval?: TNeedsApproval
56
64
  }
57
65
 
58
66
  /**
59
67
  * Union type for any kind of client-side tool (client tool or definition)
60
68
  */
61
69
  export type AnyClientTool =
62
- | (Omit<ClientTool<any, any, string, any>, 'execute'> & {
70
+ | (Omit<ClientTool<any, any, string, any, boolean>, 'execute'> & {
63
71
  execute?: ((args: any, context?: any) => any) | undefined
64
72
  })
65
- | (Omit<ToolDefinitionInstance<any, any, string, any>, 'execute'> & {
73
+ | (Omit<ToolDefinitionInstance<any, any, string, any, boolean>, 'execute'> & {
66
74
  execute?: ((args: any, context?: any) => any) | undefined
67
75
  })
68
76
 
@@ -100,12 +108,13 @@ export interface ToolDefinitionConfig<
100
108
  TInput extends SchemaInput = SchemaInput,
101
109
  TOutput extends SchemaInput = SchemaInput,
102
110
  TName extends string = string,
111
+ TNeedsApproval extends boolean = false,
103
112
  > {
104
113
  name: TName
105
114
  description: string
106
115
  inputSchema?: TInput
107
116
  outputSchema?: TOutput
108
- needsApproval?: boolean
117
+ needsApproval?: TNeedsApproval
109
118
  lazy?: boolean
110
119
  metadata?: Record<string, unknown>
111
120
  }
@@ -117,7 +126,14 @@ export interface ToolDefinition<
117
126
  TInput extends SchemaInput = SchemaInput,
118
127
  TOutput extends SchemaInput = SchemaInput,
119
128
  TName extends string = string,
120
- > extends ToolDefinitionInstance<TInput, TOutput, TName> {
129
+ TNeedsApproval extends boolean = false,
130
+ > extends ToolDefinitionInstance<
131
+ TInput,
132
+ TOutput,
133
+ TName,
134
+ unknown,
135
+ TNeedsApproval
136
+ > {
121
137
  /**
122
138
  * Create a server-side tool with execute function
123
139
  */
@@ -126,11 +142,13 @@ export interface ToolDefinition<
126
142
  ) => ServerTool<TInput, TOutput, TName, TContext>
127
143
 
128
144
  /**
129
- * Create a client-side tool with optional execute function
145
+ * Create a client-side tool with optional execute function.
146
+ * Carries the definition's `needsApproval` literal through to the client
147
+ * tool so the tool-call part's `approval` field stays gated on it.
130
148
  */
131
149
  client: <TContext = unknown>(
132
150
  execute?: ToolExecuteFunction<TInput, TOutput, TContext>,
133
- ) => ClientTool<TInput, TOutput, TName, TContext>
151
+ ) => ClientTool<TInput, TOutput, TName, TContext, TNeedsApproval>
134
152
  }
135
153
 
136
154
  /**
@@ -192,10 +210,14 @@ export function toolDefinition<
192
210
  TInput extends SchemaInput = SchemaInput,
193
211
  TOutput extends SchemaInput = SchemaInput,
194
212
  TName extends string = string,
213
+ // `const` forces the literal (`true` / `false`) to be captured from the
214
+ // config's optional `needsApproval` — without it TS widens to `boolean`,
215
+ // which collapses the approval gate in `ToolCallPartForTool`.
216
+ const TNeedsApproval extends boolean = false,
195
217
  >(
196
- config: ToolDefinitionConfig<TInput, TOutput, TName>,
197
- ): ToolDefinition<TInput, TOutput, TName> {
198
- const definition: ToolDefinition<TInput, TOutput, TName> = {
218
+ config: ToolDefinitionConfig<TInput, TOutput, TName, TNeedsApproval>,
219
+ ): ToolDefinition<TInput, TOutput, TName, TNeedsApproval> {
220
+ const definition: ToolDefinition<TInput, TOutput, TName, TNeedsApproval> = {
199
221
  __toolSide: 'definition',
200
222
  ...config,
201
223
  server<TContext = unknown>(
@@ -210,7 +232,7 @@ export function toolDefinition<
210
232
 
211
233
  client<TContext = unknown>(
212
234
  execute?: ToolExecuteFunction<TInput, TOutput, TContext>,
213
- ): ClientTool<TInput, TOutput, TName, TContext> {
235
+ ): ClientTool<TInput, TOutput, TName, TContext, TNeedsApproval> {
214
236
  return {
215
237
  __toolSide: 'client',
216
238
  ...config,
@@ -14,10 +14,10 @@ import {
14
14
  runGenerationFinish,
15
15
  runGenerationStart,
16
16
  runGenerationUsage,
17
- } from '../middleware'
17
+ } from '../middleware/run'
18
18
  import type { InternalLogger } from '../../logger/internal-logger'
19
19
  import type { DebugOption } from '../../logger/types'
20
- import type { GenerationMiddleware } from '../middleware'
20
+ import type { GenerationMiddleware } from '../middleware/types'
21
21
  import type { AudioAdapter } from './adapter'
22
22
  import type { AudioGenerationResult, StreamChunk } from '../../types'
23
23
 
@@ -14,11 +14,11 @@ import {
14
14
  runGenerationFinish,
15
15
  runGenerationStart,
16
16
  runGenerationUsage,
17
- } from '../middleware'
17
+ } from '../middleware/run'
18
18
  import { resolveMediaPrompt } from '../../utilities/media-prompt'
19
19
  import type { InternalLogger } from '../../logger/internal-logger'
20
20
  import type { DebugOption } from '../../logger/types'
21
- import type { GenerationMiddleware } from '../middleware'
21
+ import type { GenerationMiddleware } from '../middleware/types'
22
22
  import type { ImageAdapter } from './adapter'
23
23
  import type {
24
24
  ImageGenerationResult,
@@ -14,10 +14,10 @@ import {
14
14
  runGenerationFinish,
15
15
  runGenerationStart,
16
16
  runGenerationUsage,
17
- } from '../middleware'
17
+ } from '../middleware/run'
18
18
  import type { InternalLogger } from '../../logger/internal-logger'
19
19
  import type { DebugOption } from '../../logger/types'
20
- import type { GenerationMiddleware } from '../middleware'
20
+ import type { GenerationMiddleware } from '../middleware/types'
21
21
  import type { TTSAdapter } from './adapter'
22
22
  import type { StreamChunk, TTSResult } from '../../types'
23
23
 
@@ -14,12 +14,16 @@ import {
14
14
  runGenerationFinish,
15
15
  runGenerationStart,
16
16
  runGenerationUsage,
17
- } from '../middleware'
17
+ } from '../middleware/run'
18
18
  import type { InternalLogger } from '../../logger/internal-logger'
19
19
  import type { DebugOption } from '../../logger/types'
20
- import type { GenerationMiddleware } from '../middleware'
20
+ import type { GenerationMiddleware } from '../middleware/types'
21
21
  import type { TranscriptionAdapter } from './adapter'
22
- import type { StreamChunk, TranscriptionResult } from '../../types'
22
+ import type {
23
+ StreamChunk,
24
+ TranscriptionResponseFormat,
25
+ TranscriptionResult,
26
+ } from '../../types'
23
27
 
24
28
  // ===========================
25
29
  // Activity Kind
@@ -67,7 +71,7 @@ export interface TranscriptionActivityOptions<
67
71
  /** An optional prompt to guide the transcription */
68
72
  prompt?: string
69
73
  /** The format of the transcription output */
70
- responseFormat?: 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt'
74
+ responseFormat?: TranscriptionResponseFormat
71
75
  /** Provider-specific options for transcription */
72
76
  modelOptions?: TranscriptionProviderOptions<TAdapter>
73
77
  /**
@@ -17,10 +17,10 @@ import {
17
17
  runGenerationFinish,
18
18
  runGenerationStart,
19
19
  runGenerationUsage,
20
- } from '../middleware'
20
+ } from '../middleware/run'
21
21
  import type { InternalLogger } from '../../logger/internal-logger'
22
22
  import type { DebugOption } from '../../logger/types'
23
- import type { GenerationMiddleware } from '../middleware'
23
+ import type { GenerationMiddleware } from '../middleware/types'
24
24
  import type { VideoAdapter } from './adapter'
25
25
  import type {
26
26
  MediaPrompt,
@@ -47,7 +47,7 @@ export const kind = 'video' as const
47
47
  * Extract provider options from a VideoAdapter via ~types.
48
48
  */
49
49
  export type VideoProviderOptions<TAdapter> =
50
- TAdapter extends VideoAdapter<any, any, any, any>
50
+ TAdapter extends VideoAdapter<any, any, any, any, any, any>
51
51
  ? TAdapter['~types']['providerOptions']
52
52
  : object
53
53
 
@@ -55,7 +55,14 @@ export type VideoProviderOptions<TAdapter> =
55
55
  * Extract the size type for a VideoAdapter's model via ~types.
56
56
  */
57
57
  export type VideoSizeForAdapter<TAdapter> =
58
- TAdapter extends VideoAdapter<infer TModel, any, any, infer TSizeMap>
58
+ TAdapter extends VideoAdapter<
59
+ infer TModel,
60
+ any,
61
+ any,
62
+ infer TSizeMap,
63
+ any,
64
+ any
65
+ >
59
66
  ? TModel extends keyof TSizeMap
60
67
  ? TSizeMap[TModel]
61
68
  : string
@@ -68,7 +75,14 @@ export type VideoSizeForAdapter<TAdapter> =
68
75
  * without a map fall back to the full MediaPrompt.
69
76
  */
70
77
  export type VideoPromptForAdapter<TAdapter> =
71
- TAdapter extends VideoAdapter<infer TModel, any, any, any, infer ModsByName>
78
+ TAdapter extends VideoAdapter<
79
+ infer TModel,
80
+ any,
81
+ any,
82
+ any,
83
+ infer ModsByName,
84
+ any
85
+ >
72
86
  ? string extends keyof ModsByName
73
87
  ? MediaPrompt
74
88
  : TModel extends keyof ModsByName
@@ -108,7 +122,7 @@ function createId(prefix: string): string {
108
122
  * The model is extracted from the adapter's model property.
109
123
  */
110
124
  interface VideoActivityBaseOptions<
111
- TAdapter extends VideoAdapter<string, any, any, any>,
125
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
112
126
  > {
113
127
  /** The video adapter to use (must be created with a model) */
114
128
  adapter: TAdapter & { kind: typeof kind }
@@ -124,7 +138,7 @@ interface VideoActivityBaseOptions<
124
138
  * @experimental Video generation is an experimental feature and may change.
125
139
  */
126
140
  export type VideoCreateOptions<
127
- TAdapter extends VideoAdapter<string, any, any, any>,
141
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
128
142
  TStream extends boolean = false,
129
143
  > = VideoActivityBaseOptions<TAdapter> & {
130
144
  /** Request type - create a new job (default if not specified) */
@@ -191,7 +205,7 @@ export type VideoCreateOptions<
191
205
  * @experimental Video generation is an experimental feature and may change.
192
206
  */
193
207
  export interface VideoStatusOptions<
194
- TAdapter extends VideoAdapter<string, any, any, any>,
208
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
195
209
  > extends VideoActivityBaseOptions<TAdapter> {
196
210
  /** Request type - get job status */
197
211
  request: 'status'
@@ -205,7 +219,7 @@ export interface VideoStatusOptions<
205
219
  * @experimental Video generation is an experimental feature and may change.
206
220
  */
207
221
  export interface VideoUrlOptions<
208
- TAdapter extends VideoAdapter<string, any, any, any>,
222
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
209
223
  > extends VideoActivityBaseOptions<TAdapter> {
210
224
  /** Request type - get video URL */
211
225
  request: 'url'
@@ -220,7 +234,7 @@ export interface VideoUrlOptions<
220
234
  * @experimental Video generation is an experimental feature and may change.
221
235
  */
222
236
  export type VideoActivityOptions<
223
- TAdapter extends VideoAdapter<string, any, any, any>,
237
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
224
238
  TRequest extends 'create' | 'status' | 'url' = 'create',
225
239
  TStream extends boolean = false,
226
240
  > = TRequest extends 'status'
@@ -296,7 +310,7 @@ export type VideoActivityResult<
296
310
  * ```
297
311
  */
298
312
  export function generateVideo<
299
- TAdapter extends VideoAdapter<string, any, any, any>,
313
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
300
314
  TStream extends boolean = false,
301
315
  >(
302
316
  options: VideoCreateOptions<TAdapter, TStream>,
@@ -314,7 +328,7 @@ export function generateVideo<
314
328
  * Internal implementation of non-streaming video job creation.
315
329
  */
316
330
  async function runCreateVideoJob<
317
- TAdapter extends VideoAdapter<string, any, any, any>,
331
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
318
332
  >(options: VideoCreateOptions<TAdapter, boolean>): Promise<VideoJobResult> {
319
333
  const { adapter, prompt, size, duration, modelOptions, middleware } = options
320
334
  const model = adapter.model
@@ -383,7 +397,7 @@ function sleep(ms: number): Promise<void> {
383
397
  * Handles the full job lifecycle: create job → poll for status → stream updates → yield final result.
384
398
  */
385
399
  async function* runStreamingVideoGeneration<
386
- TAdapter extends VideoAdapter<string, any, any, any>,
400
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
387
401
  >(options: VideoCreateOptions<TAdapter, true>): AsyncIterable<StreamChunk> {
388
402
  const { adapter, prompt, size, duration, modelOptions, middleware } = options
389
403
  const model = adapter.model
@@ -582,7 +596,7 @@ async function* runStreamingVideoGeneration<
582
596
  * ```
583
597
  */
584
598
  export async function getVideoJobStatus<
585
- TAdapter extends VideoAdapter<string, any, any, any>,
599
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
586
600
  >(options: {
587
601
  adapter: TAdapter & { kind: typeof kind }
588
602
  jobId: string
@@ -692,7 +706,7 @@ export async function getVideoJobStatus<
692
706
  * Create typed options for the generateVideo() function without executing.
693
707
  */
694
708
  export function createVideoOptions<
695
- TAdapter extends VideoAdapter<string, any, any, any>,
709
+ TAdapter extends VideoAdapter<string, any, any, any, any, any>,
696
710
  TStream extends boolean = false,
697
711
  >(
698
712
  options: VideoCreateOptions<TAdapter, TStream>,
package/src/index.ts CHANGED
@@ -121,6 +121,7 @@ export type {
121
121
  AbortInfo,
122
122
  ErrorInfo,
123
123
  SandboxFileEvent,
124
+ SandboxFileHookEvent,
124
125
  ChatSandboxHooks,
125
126
  } from './activities/chat/middleware/index'
126
127
 
@@ -178,12 +179,13 @@ export { normalizeSystemPrompts } from './system-prompts'
178
179
  export { detectImageMimeType } from './utils'
179
180
 
180
181
  // Realtime
181
- export { realtimeToken } from './realtime/index'
182
+ export { realtimeToken, createRealtimeEventEmitter } from './realtime/index'
182
183
  export type {
183
184
  RealtimeToken,
184
185
  RealtimeTokenAdapter,
185
186
  RealtimeTokenOptions,
186
187
  RealtimeSessionConfig,
188
+ RealtimeToolConfig,
187
189
  VADConfig,
188
190
  RealtimeMessage,
189
191
  RealtimeMessagePart,
@@ -0,0 +1,46 @@
1
+ import type {
2
+ RealtimeEvent,
3
+ RealtimeEventHandler,
4
+ RealtimeEventPayloads,
5
+ } from './types'
6
+
7
+ /**
8
+ * Handlers are stored with a `never` payload so any specific
9
+ * `RealtimeEventHandler<TEvent>` is assignable in (contravariance), keeping the
10
+ * heterogeneous handler map type-safe without `any`. `emit` narrows back to the
11
+ * event's real payload type via its signature; the lone `as never` at the call
12
+ * site is the inverse of that stored `never`.
13
+ */
14
+ type StoredHandler = (payload: never) => void
15
+
16
+ export function createRealtimeEventEmitter() {
17
+ const eventHandlers = new Map<RealtimeEvent, Set<StoredHandler>>()
18
+
19
+ return {
20
+ emit<TEvent extends RealtimeEvent>(
21
+ event: TEvent,
22
+ payload: RealtimeEventPayloads[TEvent],
23
+ ) {
24
+ const handlers = eventHandlers.get(event)
25
+ if (!handlers) return
26
+ for (const handler of handlers) {
27
+ handler(payload as never)
28
+ }
29
+ },
30
+ on<TEvent extends RealtimeEvent>(
31
+ event: TEvent,
32
+ handler: RealtimeEventHandler<TEvent>,
33
+ ): () => void {
34
+ let handlers = eventHandlers.get(event)
35
+ if (!handlers) {
36
+ handlers = new Set<StoredHandler>()
37
+ eventHandlers.set(event, handlers)
38
+ }
39
+ handlers.add(handler)
40
+
41
+ return () => {
42
+ handlers.delete(handler)
43
+ }
44
+ },
45
+ }
46
+ }
@@ -1,5 +1,7 @@
1
1
  import type { RealtimeToken, RealtimeTokenOptions } from './types'
2
2
 
3
+ export { createRealtimeEventEmitter } from './event-emitter'
4
+
3
5
  // Re-export all types
4
6
  export type * from './types'
5
7