@tanstack/ai 0.5.0 → 0.6.1

package/src/activities/chat/index.ts

@@ -24,6 +24,7 @@ import type { AnyTextAdapter } from './adapter'
 import type {
   AgentLoopStrategy,
   ConstrainedModelMessage,
+  CustomEvent,
   InferSchemaType,
   ModelMessage,
   RunFinishedEvent,
@@ -601,17 +602,30 @@ class TextEngine<
 
     const { approvals, clientToolResults } = this.collectClientState()
 
-    const executionResult = await executeToolCalls(
+    const generator = executeToolCalls(
       pendingToolCalls,
       this.tools,
       approvals,
       clientToolResults,
+      (eventName, data) => this.createCustomEventChunk(eventName, data),
     )
 
+    // Consume the async generator, yielding custom events and collecting the return value
+    const executionResult = yield* this.drainToolCallGenerator(generator)
+
     if (
       executionResult.needsApproval.length > 0 ||
       executionResult.needsClientExecution.length > 0
     ) {
+      if (executionResult.results.length > 0) {
+        for (const chunk of this.emitToolResults(
+          executionResult.results,
+          finishEvent,
+        )) {
+          yield chunk
+        }
+      }
+
       for (const chunk of this.emitApprovalRequests(
         executionResult.needsApproval,
         finishEvent,
@@ -660,17 +674,30 @@ class TextEngine<
 
     const { approvals, clientToolResults } = this.collectClientState()
 
-    const executionResult = await executeToolCalls(
+    const generator = executeToolCalls(
       toolCalls,
       this.tools,
       approvals,
       clientToolResults,
+      (eventName, data) => this.createCustomEventChunk(eventName, data),
     )
 
+    // Consume the async generator, yielding custom events and collecting the return value
+    const executionResult = yield* this.drainToolCallGenerator(generator)
+
     if (
       executionResult.needsApproval.length > 0 ||
       executionResult.needsClientExecution.length > 0
     ) {
+      if (executionResult.results.length > 0) {
+        for (const chunk of this.emitToolResults(
+          executionResult.results,
+          finishEvent,
+        )) {
+          yield chunk
+        }
+      }
+
       for (const chunk of this.emitApprovalRequests(
         executionResult.needsApproval,
         finishEvent,
@@ -833,7 +860,7 @@ class TextEngine<
         timestamp: Date.now(),
         model: finishEvent.model,
         name: 'approval-requested',
-        data: {
+        value: {
           toolCallId: approval.toolCallId,
           toolName: approval.toolName,
           input: approval.input,
@@ -870,7 +897,7 @@ class TextEngine<
         timestamp: Date.now(),
         model: finishEvent.model,
         name: 'tool-input-available',
-        data: {
+        value: {
           toolCallId: clientTool.toolCallId,
           toolName: clientTool.toolName,
           input: clientTool.input,
@@ -1052,6 +1079,50 @@ class TextEngine<
     }
   }
 
+  /**
+   * Drain an executeToolCalls async generator, yielding any CustomEvent chunks
+   * and returning the final ExecuteToolCallsResult.
+   */
+  private async *drainToolCallGenerator(
+    generator: AsyncGenerator<
+      CustomEvent,
+      {
+        results: Array<ToolResult>
+        needsApproval: Array<ApprovalRequest>
+        needsClientExecution: Array<ClientToolRequest>
+      },
+      void
+    >,
+  ): AsyncGenerator<
+    StreamChunk,
+    {
+      results: Array<ToolResult>
+      needsApproval: Array<ApprovalRequest>
+      needsClientExecution: Array<ClientToolRequest>
+    },
+    void
+  > {
+    let next = await generator.next()
+    while (!next.done) {
+      yield next.value
+      next = await generator.next()
+    }
+    return next.value
+  }
+
+  private createCustomEventChunk(
+    eventName: string,
+    value: Record<string, any>,
+  ): CustomEvent {
+    return {
+      type: 'CUSTOM',
+      timestamp: Date.now(),
+      model: this.params.model,
+      name: eventName,
+      value,
+    }
+  }
+
   private createId(prefix: string): string {
     return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
   }

package/src/activities/chat/stream/processor.ts

@@ -73,6 +73,13 @@ export interface StreamProcessorEvents {
     approvalId: string
   }) => void
 
+  // Custom events from server-side tools
+  onCustomEvent?: (
+    eventType: string,
+    data: unknown,
+    context: { toolCallId?: string },
+  ) => void
+
   // Granular events for UI optimization (character-by-character, state tracking)
   onTextUpdate?: (messageId: string, content: string) => void
   onToolCallStateChange?: (
@@ -807,8 +814,8 @@ export class StreamProcessor {
     chunk: Extract<StreamChunk, { type: 'CUSTOM' }>,
   ): void {
     // Handle client tool input availability - trigger client-side execution
-    if (chunk.name === 'tool-input-available' && chunk.data) {
-      const { toolCallId, toolName, input } = chunk.data as {
+    if (chunk.name === 'tool-input-available' && chunk.value) {
+      const { toolCallId, toolName, input } = chunk.value as {
         toolCallId: string
         toolName: string
         input: any
@@ -820,11 +827,12 @@ export class StreamProcessor {
         toolName,
         input,
       })
+      return
     }
 
     // Handle approval requests
-    if (chunk.name === 'approval-requested' && chunk.data) {
-      const { toolCallId, toolName, input, approval } = chunk.data as {
+    if (chunk.name === 'approval-requested' && chunk.value) {
+      const { toolCallId, toolName, input, approval } = chunk.value as {
         toolCallId: string
         toolName: string
         input: any
@@ -849,6 +857,16 @@ export class StreamProcessor {
         input,
         approvalId: approval.id,
       })
+      return
+    }
+
+    // Forward non-system custom events to onCustomEvent callback
+    if (this.events.onCustomEvent) {
+      const toolCallId =
+        chunk.value && typeof chunk.value === 'object'
+          ? (chunk.value as any).toolCallId
+          : undefined
+      this.events.onCustomEvent(chunk.name, chunk.value, { toolCallId })
     }
   }
 

package/src/activities/chat/tools/tool-calls.ts

@@ -1,5 +1,6 @@
 import { isStandardSchema, parseWithStandardSchema } from './schema-converter'
 import type {
+  CustomEvent,
   ModelMessage,
   RunFinishedEvent,
   Tool,
@@ -7,6 +8,7 @@ import type {
   ToolCallArgsEvent,
   ToolCallEndEvent,
   ToolCallStartEvent,
+  ToolExecutionContext,
 } from '../../../types'
 
 /**
@@ -250,7 +252,46 @@ interface ExecuteToolCallsResult {
 }
 
 /**
- * Execute tool calls based on their configuration
+ * Helper that runs a tool execution promise while polling for pending custom events.
+ * Yields any custom events that are emitted during execution, then returns the
+ * execution result.
+ */
+async function* executeWithEventPolling<T>(
+  executionPromise: Promise<T>,
+  pendingEvents: Array<CustomEvent>,
+): AsyncGenerator<CustomEvent, T, void> {
+  // Use an object to track mutable state across the async boundary
+  const state = { done: false, result: undefined as T }
+  const executionWithFlag = executionPromise.then((r) => {
+    state.done = true
+    state.result = r
+    return r
+  })
+
+  while (!state.done) {
+    // Wait for either the execution to complete or a short timeout
+    await Promise.race([
+      executionWithFlag,
+      new Promise((resolve) => setTimeout(resolve, 10)),
+    ])
+
+    // Flush any pending events
+    while (pendingEvents.length > 0) {
+      yield pendingEvents.shift()!
+    }
+  }
+
+  // Final flush in case events were emitted right at completion
+  while (pendingEvents.length > 0) {
+    yield pendingEvents.shift()!
+  }
+
+  return state.result
+}
+
+/**
+ * Execute tool calls based on their configuration.
+ * Yields CustomEvent chunks during tool execution for real-time progress updates.
  *
  * Handles three cases:
  * 1. Client tools (no execute) - request client to execute
@@ -261,13 +302,18 @@ interface ExecuteToolCallsResult {
  * @param tools - Available tools with their configurations
  * @param approvals - Map of approval decisions (approval.id -> approved boolean)
  * @param clientResults - Map of client-side execution results (toolCallId -> result)
+ * @param createCustomEventChunk - Factory to create CustomEvent chunks (optional)
  */
-export async function executeToolCalls(
+export async function* executeToolCalls(
   toolCalls: Array<ToolCall>,
   tools: ReadonlyArray<Tool>,
   approvals: Map<string, boolean> = new Map(),
   clientResults: Map<string, any> = new Map(),
-): Promise<ExecuteToolCallsResult> {
+  createCustomEventChunk?: (
+    eventName: string,
+    value: Record<string, any>,
+  ) => CustomEvent,
+): AsyncGenerator<CustomEvent, ExecuteToolCallsResult, void> {
   const results: Array<ToolResult> = []
   const needsApproval: Array<ApprovalRequest> = []
   const needsClientExecution: Array<ClientToolRequest> = []
@@ -326,6 +372,29 @@ export async function executeToolCalls(
       }
     }
 
+    // Create a ToolExecutionContext for this tool call with event emission
+    const pendingEvents: Array<CustomEvent> = []
+    const context: ToolExecutionContext = {
+      toolCallId: toolCall.id,
+      emitCustomEvent: (eventName: string, value: Record<string, any>) => {
+        if (createCustomEventChunk) {
+          pendingEvents.push(
+            createCustomEventChunk(eventName, {
+              ...value,
+              toolCallId: toolCall.id,
+            }),
+          )
+        }
+      },
+    }
+
+    // Helper to flush any pending events
+    function* flushEvents(): Generator<CustomEvent> {
+      while (pendingEvents.length > 0) {
+        yield pendingEvents.shift()!
+      }
+    }
+
     // CASE 1: Client-side tool (no execute function)
     if (!tool.execute) {
       // Check if tool needs approval
@@ -402,8 +471,15 @@ export async function executeToolCalls(
           // Execute after approval
           const startTime = Date.now()
           try {
-            let result = await tool.execute(input)
+            const executionPromise = Promise.resolve(
+              tool.execute(input, context),
+            )
+            let result = yield* executeWithEventPolling(
+              executionPromise,
+              pendingEvents,
+            )
             const duration = Date.now() - startTime
+            yield* flushEvents()
 
             // Validate output against outputSchema if provided (for Standard Schema compliant schemas)
             if (
@@ -426,6 +502,7 @@ export async function executeToolCalls(
             })
           } catch (error: unknown) {
             const duration = Date.now() - startTime
+            yield* flushEvents()
             const message =
               error instanceof Error ? error.message : 'Unknown error'
             results.push({
@@ -460,8 +537,13 @@ export async function executeToolCalls(
     // CASE 3: Normal server tool - execute immediately
     const startTime = Date.now()
     try {
-      let result = await tool.execute(input)
+      const executionPromise = Promise.resolve(tool.execute(input, context))
+      let result = yield* executeWithEventPolling(
+        executionPromise,
+        pendingEvents,
+      )
       const duration = Date.now() - startTime
+      yield* flushEvents()
 
       // Validate output against outputSchema if provided (for Standard Schema compliant schemas)
       if (
@@ -482,6 +564,7 @@ export async function executeToolCalls(
       })
     } catch (error: unknown) {
       const duration = Date.now() - startTime
+      yield* flushEvents()
       const message = error instanceof Error ? error.message : 'Unknown error'
       results.push({
         toolCallId: toolCall.id,

package/src/activities/chat/tools/tool-definition.ts

@@ -4,6 +4,7 @@ import type {
   JSONSchema,
   SchemaInput,
   Tool,
+  ToolExecutionContext,
 } from '../../../types'
 
 /**
@@ -112,6 +113,7 @@ export interface ToolDefinition<
   server: (
     execute: (
       args: InferSchemaType<TInput>,
+      context?: ToolExecutionContext,
     ) => Promise<InferSchemaType<TOutput>> | InferSchemaType<TOutput>,
   ) => ServerTool<TInput, TOutput, TName>
 
@@ -193,6 +195,7 @@ export function toolDefinition<
     server(
       execute: (
         args: InferSchemaType<TInput>,
+        context?: ToolExecutionContext,
       ) => Promise<InferSchemaType<TOutput>> | InferSchemaType<TOutput>,
     ): ServerTool<TInput, TOutput, TName> {
       return {

package/src/activities/generateImage/adapter.ts

@@ -1,5 +1,12 @@
 import type { ImageGenerationOptions, ImageGenerationResult } from '../../types'
 
+/**
+ * Resolve the size type for a model from the model-size map.
+ * If the map has an index signature (i.e. no explicit keys), falls back to string.
+ * If the model is an explicit key, uses its mapped size type.
+ * Otherwise falls back to string.
+ */
+
 /**
  * Configuration for image adapter instances
  */
@@ -49,7 +56,7 @@ export interface ImageAdapter<
    * Generate images from a prompt
    */
   generateImages: (
-    options: ImageGenerationOptions<TProviderOptions>,
+    options: ImageGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
   ) => Promise<ImageGenerationResult>
 }
 
@@ -95,7 +102,7 @@ export abstract class BaseImageAdapter<
   }
 
   abstract generateImages(
-    options: ImageGenerationOptions<TProviderOptions>,
+    options: ImageGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
   ): Promise<ImageGenerationResult>
 
   protected generateId(): string {

package/src/activities/generateImage/index.ts

@@ -61,9 +61,9 @@ export type ImageSizeForModel<TAdapter, TModel extends string> =
  *
  * @template TAdapter - The image adapter type
  */
-export interface ImageActivityOptions<
-  TAdapter extends ImageAdapter<string, object, any, any>,
-> {
+export type ImageActivityOptions<
+  TAdapter extends ImageAdapter<string, any, any, any>,
+> = {
   /** The image adapter to use (must be created with a model) */
   adapter: TAdapter & { kind: typeof kind }
   /** Text description of the desired image(s) */
@@ -72,9 +72,19 @@ export interface ImageActivityOptions<
   numberOfImages?: number
   /** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
   size?: ImageSizeForModel<TAdapter, TAdapter['model']>
-  /** Provider-specific options for image generation */
-  modelOptions?: ImageProviderOptionsForModel<TAdapter, TAdapter['model']>
-}
+} & ({} extends ImageProviderOptionsForModel<TAdapter, TAdapter['model']>
+  ? {
+      /** Provider-specific options for image generation */ modelOptions?: ImageProviderOptionsForModel<
+        TAdapter,
+        TAdapter['model']
+      >
+    }
+  : {
+      /** Provider-specific options for image generation */ modelOptions: ImageProviderOptionsForModel<
+        TAdapter,
+        TAdapter['model']
+      >
+    })
 
 // ===========================
 // Activity Result Type
@@ -137,7 +147,7 @@ function createId(prefix: string): string {
  * ```
  */
 export async function generateImage<
-  TAdapter extends ImageAdapter<string, object, any, any>,
+  TAdapter extends ImageAdapter<string, any, any, any>,
 >(options: ImageActivityOptions<TAdapter>): ImageActivityResult {
   const { adapter, ...rest } = options
   const model = adapter.model
@@ -150,7 +160,7 @@ export async function generateImage<
     model,
     prompt: rest.prompt,
     numberOfImages: rest.numberOfImages,
-    size: rest.size as string | undefined,
+    size: rest.size,
     modelOptions: rest.modelOptions as Record<string, unknown> | undefined,
     timestamp: startTime,
   })
@@ -193,7 +203,7 @@ export async function generateImage<
  * Create typed options for the generateImage() function without executing.
  */
 export function createImageOptions<
-  TAdapter extends ImageAdapter<string, object, any, any>,
+  TAdapter extends ImageAdapter<string, any, any, any>,
 >(options: ImageActivityOptions<TAdapter>): ImageActivityOptions<TAdapter> {
   return options
 }

package/src/activities/generateVideo/adapter.ts

@@ -29,10 +29,14 @@ export interface VideoAdapterConfig {
  * Generic parameters:
  * - TModel: The specific model name (e.g., 'sora-2')
  * - TProviderOptions: Provider-specific options (already resolved)
+ * - TModelProviderOptionsByName: Map from model name to its specific provider options
+ * - TModelSizeByName: Map from model name to its supported sizes
  */
 export interface VideoAdapter<
   TModel extends string = string,
   TProviderOptions extends object = Record<string, unknown>,
+  TModelProviderOptionsByName extends Record<string, any> = Record<string, any>,
+  TModelSizeByName extends Record<string, string> = Record<string, string>,
 > {
   /** Discriminator for adapter kind - used to determine API shape */
   readonly kind: 'video'
@@ -46,6 +50,8 @@ export interface VideoAdapter<
    */
   '~types': {
     providerOptions: TProviderOptions
+    modelProviderOptionsByName: TModelProviderOptionsByName
+    modelSizeByName: TModelSizeByName
   }
 
   /**
@@ -53,7 +59,7 @@ export interface VideoAdapter<
    * Returns a job ID that can be used to poll for status and retrieve the video.
    */
   createVideoJob: (
-    options: VideoGenerationOptions<TProviderOptions>,
+    options: VideoGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
   ) => Promise<VideoJobResult>
 
   /**
@@ -72,7 +78,7 @@ export interface VideoAdapter<
  * A VideoAdapter with any/unknown type parameters.
  * Useful as a constraint in generic functions and interfaces.
  */
-export type AnyVideoAdapter = VideoAdapter<any, any>
+export type AnyVideoAdapter = VideoAdapter<any, any, any, any>
 
 /**
  * Abstract base class for video generation adapters.
@@ -85,7 +91,14 @@ export type AnyVideoAdapter = VideoAdapter<any, any>
 export abstract class BaseVideoAdapter<
   TModel extends string = string,
   TProviderOptions extends object = Record<string, unknown>,
-> implements VideoAdapter<TModel, TProviderOptions> {
+  TModelProviderOptionsByName extends Record<string, any> = Record<string, any>,
+  TModelSizeByName extends Record<string, string> = Record<string, string>,
+> implements VideoAdapter<
+  TModel,
+  TProviderOptions,
+  TModelProviderOptionsByName,
+  TModelSizeByName
+> {
   readonly kind = 'video' as const
   abstract readonly name: string
   readonly model: TModel
@@ -93,6 +106,8 @@ export abstract class BaseVideoAdapter<
   // Type-only property - never assigned at runtime
   declare '~types': {
     providerOptions: TProviderOptions
+    modelProviderOptionsByName: TModelProviderOptionsByName
+    modelSizeByName: TModelSizeByName
   }
 
   protected config: VideoAdapterConfig
@@ -103,7 +118,7 @@ export abstract class BaseVideoAdapter<
   }
 
   abstract createVideoJob(
-    options: VideoGenerationOptions<TProviderOptions>,
+    options: VideoGenerationOptions<TProviderOptions, TModelSizeByName[TModel]>,
   ): Promise<VideoJobResult>
 
   abstract getVideoStatus(jobId: string): Promise<VideoStatusResult>

package/src/activities/generateVideo/index.ts

@@ -30,10 +30,20 @@ export const kind = 'video' as const
  * Extract provider options from a VideoAdapter via ~types.
  */
 export type VideoProviderOptions<TAdapter> =
-  TAdapter extends VideoAdapter<any, any>
+  TAdapter extends VideoAdapter<any, any, any, any>
     ? TAdapter['~types']['providerOptions']
     : object
 
+/**
+ * Extract the size type for a VideoAdapter's model via ~types.
+ */
+export type VideoSizeForAdapter<TAdapter> =
+  TAdapter extends VideoAdapter<infer TModel, any, any, infer TSizeMap>
+    ? TModel extends keyof TSizeMap
+      ? TSizeMap[TModel]
+      : string
+    : string
+
 // ===========================
 // Activity Options Types
 
@@ -47,7 +57,7 @@ function createId(prefix: string): string {
  * The model is extracted from the adapter's model property.
  */
 interface VideoActivityBaseOptions<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
 > {
   /** The video adapter to use (must be created with a model) */
   adapter: TAdapter & { kind: typeof kind }
@@ -59,20 +69,24 @@ interface VideoActivityBaseOptions<
  *
  * @experimental Video generation is an experimental feature and may change.
  */
-export interface VideoCreateOptions<
-  TAdapter extends VideoAdapter<string, object>,
-> extends VideoActivityBaseOptions<TAdapter> {
+export type VideoCreateOptions<
+  TAdapter extends VideoAdapter<string, any, any, any>,
+> = VideoActivityBaseOptions<TAdapter> & {
   /** Request type - create a new job (default if not specified) */
   request?: 'create'
   /** Text description of the desired video */
   prompt: string
-  /** Video size in WIDTHxHEIGHT format (e.g., "1280x720") */
-  size?: string
+  /** Video size — format depends on the provider (e.g., "16:9", "1280x720") */
+  size?: VideoSizeForAdapter<TAdapter>
   /** Video duration in seconds */
   duration?: number
-  /** Provider-specific options for video generation */
-  modelOptions?: VideoProviderOptions<TAdapter>
-}
+} & ({} extends VideoProviderOptions<TAdapter>
+    ? {
+        /** Provider-specific options for video generation */ modelOptions?: VideoProviderOptions<TAdapter>
+      }
+    : {
+        /** Provider-specific options for video generation */ modelOptions: VideoProviderOptions<TAdapter>
+      })
 
 /**
  * Options for polling the status of a video generation job.
@@ -80,7 +94,7 @@ export interface VideoCreateOptions<
  * @experimental Video generation is an experimental feature and may change.
  */
 export interface VideoStatusOptions<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
 > extends VideoActivityBaseOptions<TAdapter> {
   /** Request type - get job status */
   request: 'status'
@@ -94,7 +108,7 @@ export interface VideoStatusOptions<
  * @experimental Video generation is an experimental feature and may change.
  */
 export interface VideoUrlOptions<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
 > extends VideoActivityBaseOptions<TAdapter> {
   /** Request type - get video URL */
   request: 'url'
@@ -109,7 +123,7 @@ export interface VideoUrlOptions<
  * @experimental Video generation is an experimental feature and may change.
  */
 export type VideoActivityOptions<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
   TRequest extends 'create' | 'status' | 'url' = 'create',
 > = TRequest extends 'status'
   ? VideoStatusOptions<TAdapter>
@@ -161,7 +175,7 @@ export type VideoActivityResult<
  * ```
  */
 export async function generateVideo<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
 >(options: VideoCreateOptions<TAdapter>): Promise<VideoJobResult> {
   const { adapter, prompt, size, duration, modelOptions } = options
   const model = adapter.model
@@ -201,7 +215,7 @@ export async function generateVideo<
  * ```
  */
 export async function getVideoJobStatus<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
 >(options: {
   adapter: TAdapter & { kind: typeof kind }
   jobId: string
@@ -301,7 +315,7 @@ export async function getVideoJobStatus<
  * Create typed options for the generateVideo() function without executing.
  */
 export function createVideoOptions<
-  TAdapter extends VideoAdapter<string, object>,
+  TAdapter extends VideoAdapter<string, any, any, any>,
 >(options: VideoCreateOptions<TAdapter>): VideoCreateOptions<TAdapter> {
   return options
 }