@tanstack/ai 0.6.2 → 0.8.0

package/src/activities/generateVideo/index.ts

@@ -7,9 +7,10 @@
  * @experimental Video generation is an experimental feature and may change.
  */
 
-import { aiEventClient } from '../../event-client.js'
+import { aiEventClient } from '@tanstack/ai-event-client'
 import type { VideoAdapter } from './adapter'
 import type {
+  StreamChunk,
   VideoJobResult,
   VideoStatusResult,
   VideoUrlResult,
@@ -67,10 +68,14 @@ interface VideoActivityBaseOptions<
  * Options for creating a new video generation job.
  * The model is extracted from the adapter's model property.
  *
+ * @template TAdapter - The video adapter type
+ * @template TStream - Whether to stream the output
+ *
  * @experimental Video generation is an experimental feature and may change.
  */
 export type VideoCreateOptions<
   TAdapter extends VideoAdapter<string, any, any, any>,
+  TStream extends boolean = false,
 > = VideoActivityBaseOptions<TAdapter> & {
   /** Request type - create a new job (default if not specified) */
   request?: 'create'
@@ -80,6 +85,21 @@ export type VideoCreateOptions<
   size?: VideoSizeForAdapter<TAdapter>
   /** Video duration in seconds */
   duration?: number
+  /**
+   * Whether to stream the video generation lifecycle.
+   * When true, returns an AsyncIterable<StreamChunk> that handles the full
+   * job lifecycle: create job, poll for status, yield updates, and yield final result.
+   * When false or not provided, returns a Promise<VideoJobResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /** Polling interval in milliseconds (stream mode only). @default 2000 */
+  pollingInterval?: number
+  /** Maximum time to wait before timing out in milliseconds (stream mode only). @default 600000 */
+  maxDuration?: number
+  /** Custom run ID (stream mode only) */
+  runId?: string
 } & ({} extends VideoProviderOptions<TAdapter>
     ? {
         /** Provider-specific options for video generation */ modelOptions?: VideoProviderOptions<TAdapter>
@@ -125,28 +145,34 @@ export interface VideoUrlOptions<
 export type VideoActivityOptions<
   TAdapter extends VideoAdapter<string, any, any, any>,
   TRequest extends 'create' | 'status' | 'url' = 'create',
+  TStream extends boolean = false,
 > = TRequest extends 'status'
   ? VideoStatusOptions<TAdapter>
   : TRequest extends 'url'
     ? VideoUrlOptions<TAdapter>
-    : VideoCreateOptions<TAdapter>
+    : VideoCreateOptions<TAdapter, TStream>
 
 // ===========================
 // Activity Result Types
 // ===========================
 
 /**
- * Result type for the video activity, based on request type.
+ * Result type for the video activity, based on request type and streaming.
+ * - If stream is true (create request): AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<VideoJobResult | VideoStatusResult | VideoUrlResult>
  *
  * @experimental Video generation is an experimental feature and may change.
  */
 export type VideoActivityResult<
   TRequest extends 'create' | 'status' | 'url' = 'create',
+  TStream extends boolean = false,
 > = TRequest extends 'status'
   ? Promise<VideoStatusResult>
   : TRequest extends 'url'
     ? Promise<VideoUrlResult>
-    : Promise<VideoJobResult>
+    : TStream extends true
+      ? AsyncIterable<StreamChunk>
+      : Promise<VideoJobResult>
 
 // ===========================
 // Activity Implementation
@@ -158,6 +184,9 @@ export type VideoActivityResult<
  * Uses AI video generation models to create videos based on natural language descriptions.
  * Unlike image generation, video generation is asynchronous and requires polling for completion.
  *
+ * When `stream: true` is passed, handles the full job lifecycle automatically:
+ * create job → poll for status → stream updates → yield final result.
+ *
  * @experimental Video generation is an experimental feature and may change.
  *
  * @example Create a video generation job
@@ -173,10 +202,43 @@ export type VideoActivityResult<
  *
  * console.log('Job started:', jobId)
  * ```
+ *
+ * @example Stream the full video generation lifecycle
+ * ```ts
+ * import { generateVideo, toServerSentEventsResponse } from '@tanstack/ai'
+ * import { openaiVideo } from '@tanstack/ai-openai'
+ *
+ * const stream = generateVideo({
+ *   adapter: openaiVideo('sora-2'),
+ *   prompt: 'A cat chasing a dog in a sunny park',
+ *   stream: true,
+ *   pollingInterval: 3000,
+ * })
+ *
+ * return toServerSentEventsResponse(stream)
+ * ```
+ */
+export function generateVideo<
+  TAdapter extends VideoAdapter<string, any, any, any>,
+  TStream extends boolean = false,
+>(
+  options: VideoCreateOptions<TAdapter, TStream>,
+): VideoActivityResult<'create', TStream> {
+  if (options.stream) {
+    return runStreamingVideoGeneration(
+      options as VideoCreateOptions<TAdapter, true>,
+    ) as VideoActivityResult<'create', TStream>
+  }
+
+  return runCreateVideoJob(options) as VideoActivityResult<'create', TStream>
+}
+
+/**
+ * Internal implementation of non-streaming video job creation.
  */
-export async function generateVideo<
+async function runCreateVideoJob<
   TAdapter extends VideoAdapter<string, any, any, any>,
->(options: VideoCreateOptions<TAdapter>): Promise<VideoJobResult> {
+>(options: VideoCreateOptions<TAdapter, boolean>): Promise<VideoJobResult> {
   const { adapter, prompt, size, duration, modelOptions } = options
   const model = adapter.model
 
@@ -189,6 +251,108 @@ export async function generateVideo<
   })
 }
 
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * Internal streaming implementation for video generation.
+ * Handles the full job lifecycle: create job → poll for status → stream updates → yield final result.
+ */
+async function* runStreamingVideoGeneration<
+  TAdapter extends VideoAdapter<string, any, any, any>,
+>(options: VideoCreateOptions<TAdapter, true>): AsyncIterable<StreamChunk> {
+  const { adapter, prompt, size, duration, modelOptions } = options
+  const model = adapter.model
+  const runId = options.runId ?? createId('run')
+  const pollingInterval = options.pollingInterval ?? 2000
+  const maxDuration = options.maxDuration ?? 600_000
+
+  yield {
+    type: 'RUN_STARTED',
+    runId,
+    timestamp: Date.now(),
+  }
+
+  try {
+    // Create the video generation job
+    const jobResult = await adapter.createVideoJob({
+      model,
+      prompt,
+      size,
+      duration,
+      modelOptions,
+    })
+
+    yield {
+      type: 'CUSTOM',
+      name: 'video:job:created',
+      value: { jobId: jobResult.jobId },
+      timestamp: Date.now(),
+    }
+
+    // Poll for completion
+    const startTime = Date.now()
+    while (Date.now() - startTime < maxDuration) {
+      await sleep(pollingInterval)
+
+      const statusResult = await adapter.getVideoStatus(jobResult.jobId)
+
+      yield {
+        type: 'CUSTOM',
+        name: 'video:status',
+        value: {
+          jobId: jobResult.jobId,
+          status: statusResult.status,
+          progress: statusResult.progress,
+          error: statusResult.error,
+        },
+        timestamp: Date.now(),
+      }
+
+      if (statusResult.status === 'completed') {
+        const urlResult = await adapter.getVideoUrl(jobResult.jobId)
+
+        yield {
+          type: 'CUSTOM',
+          name: 'generation:result',
+          value: {
+            jobId: jobResult.jobId,
+            status: 'completed',
+            url: urlResult.url,
+            expiresAt: urlResult.expiresAt,
+          },
+          timestamp: Date.now(),
+        }
+
+        yield {
+          type: 'RUN_FINISHED',
+          runId,
+          finishReason: 'stop',
+          timestamp: Date.now(),
+        }
+        return
+      }
+
+      if (statusResult.status === 'failed') {
+        throw new Error(statusResult.error || 'Video generation failed')
+      }
+    }
+
+    throw new Error('Video generation timed out')
+  } catch (error: any) {
+    yield {
+      type: 'RUN_ERROR',
+      runId,
+      error: {
+        message: error.message || 'Video generation failed',
+        code: error.code,
+      },
+      timestamp: Date.now(),
+    }
+  }
+}
+
 /**
  * Get video job status - returns the current status, progress, and URL if available.
  *
@@ -316,7 +480,10 @@ export async function getVideoJobStatus<
  */
 export function createVideoOptions<
   TAdapter extends VideoAdapter<string, any, any, any>,
->(options: VideoCreateOptions<TAdapter>): VideoCreateOptions<TAdapter> {
+  TStream extends boolean = false,
+>(
+  options: VideoCreateOptions<TAdapter, TStream>,
+): VideoCreateOptions<TAdapter, TStream> {
   return options
 }
 

package/src/activities/stream-generation-result.ts

@@ -0,0 +1,62 @@
+/**
+ * Internal helper for wrapping one-shot generation results as StreamChunk
+ * async iterables. NOT exported from the package — used only by activity
+ * implementations to support `stream: true`.
+ */
+
+import type { StreamChunk } from '../types'
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+/**
+ * Wrap a one-shot generation result as a StreamChunk async iterable.
+ *
+ * This allows non-streaming activities (image, speech, transcription, summarize)
+ * to be sent over the same streaming transport as chat.
+ *
+ * @param generator - An async function that performs the generation and returns the result
+ * @param options - Optional configuration (runId)
+ * @returns An AsyncIterable of StreamChunks with RUN_STARTED, CUSTOM(generation:result), and RUN_FINISHED events
+ */
+export async function* streamGenerationResult<TResult>(
+  generator: () => Promise<TResult>,
+  options?: { runId?: string },
+): AsyncIterable<StreamChunk> {
+  const runId = options?.runId ?? createId('run')
+
+  yield {
+    type: 'RUN_STARTED',
+    runId,
+    timestamp: Date.now(),
+  }
+
+  try {
+    const result = await generator()
+
+    yield {
+      type: 'CUSTOM',
+      name: 'generation:result',
+      value: result as unknown,
+      timestamp: Date.now(),
+    }
+
+    yield {
+      type: 'RUN_FINISHED',
+      runId,
+      finishReason: 'stop',
+      timestamp: Date.now(),
+    }
+  } catch (error: any) {
+    yield {
+      type: 'RUN_ERROR',
+      runId,
+      error: {
+        message: error.message || 'Generation failed',
+        code: error.code,
+      },
+      timestamp: Date.now(),
+    }
+  }
+}

package/src/activities/summarize/index.ts

@@ -5,7 +5,8 @@
  * This is a self-contained module with implementation, types, and JSDoc.
  */
 
-import { aiEventClient } from '../../event-client.js'
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
 import type { SummarizeAdapter } from './adapter'
 import type {
   StreamChunk,
@@ -239,28 +240,8 @@ async function* runStreamingSummarize(
     return
   }
 
-  // Fall back to non-streaming and yield as a single chunk
-  const result = await adapter.summarize(summarizeOptions)
-
-  // Yield TEXT_MESSAGE_CONTENT event with the summary
-  yield {
-    type: 'TEXT_MESSAGE_CONTENT',
-    messageId: result.id,
-    model: result.model,
-    timestamp: Date.now(),
-    delta: result.summary,
-    content: result.summary,
-  }
-
-  // Yield RUN_FINISHED event
-  yield {
-    type: 'RUN_FINISHED',
-    runId: result.id,
-    model: result.model,
-    timestamp: Date.now(),
-    finishReason: 'stop',
-    usage: result.usage,
-  }
+  // Fall back to non-streaming — wrap result with streamGenerationResult
+  yield* streamGenerationResult(() => adapter.summarize(summarizeOptions))
 }
 
 // ===========================

package/src/index.ts

@@ -70,14 +70,53 @@ export {
   combineStrategies,
 } from './activities/chat/agent-loop-strategies'
 
+// Chat middleware
+export type {
+  ChatMiddleware,
+  ChatMiddlewareContext,
+  ChatMiddlewarePhase,
+  ChatMiddlewareConfig,
+  ToolCallHookContext,
+  BeforeToolCallDecision,
+  AfterToolCallInfo,
+  IterationInfo,
+  ToolPhaseCompleteInfo,
+  UsageInfo,
+  FinishInfo,
+  AbortInfo,
+  ErrorInfo,
+} from './activities/chat/middleware/index'
+
 // All types
 export * from './types'
 
 // Utility functions
 export { detectImageMimeType } from './utils'
 
-// Event client + event types
-export * from './event-client'
+// Realtime
+export { realtimeToken } from './realtime/index'
+export type {
+  RealtimeToken,
+  RealtimeTokenAdapter,
+  RealtimeTokenOptions,
+  RealtimeSessionConfig,
+  VADConfig,
+  RealtimeMessage,
+  RealtimeMessagePart,
+  RealtimeTextPart,
+  RealtimeAudioPart,
+  RealtimeToolCallPart,
+  RealtimeToolResultPart,
+  RealtimeImagePart,
+  RealtimeStatus,
+  RealtimeMode,
+  AudioVisualization,
+  RealtimeEvent,
+  RealtimeEventPayloads,
+  RealtimeEventHandler,
+  RealtimeErrorCode,
+  RealtimeError,
+} from './realtime/index'
 
 // Message converters
 export {

package/src/middlewares/content-guard.ts

@@ -0,0 +1,285 @@
+import type {
+  ChatMiddleware,
+  ChatMiddlewareContext,
+} from '../activities/chat/middleware/types'
+import type { StreamChunk } from '../types'
+
+/**
+ * A content guard rule — either a regex pattern with replacement, or a transform function.
+ */
+export type ContentGuardRule =
+  | { pattern: RegExp; replacement: string }
+  | { fn: (text: string) => string }
+
+/**
+ * Information passed to the onFiltered callback.
+ */
+export interface ContentFilteredInfo {
+  /** The message ID being filtered */
+  messageId: string
+  /** The original text before filtering */
+  original: string
+  /** The filtered text after rules applied */
+  filtered: string
+  /** Which strategy was used */
+  strategy: 'delta' | 'buffered'
+}
+
+/**
+ * Options for the content guard middleware.
+ */
+export interface ContentGuardMiddlewareOptions {
+  /**
+   * Rules to apply to text content. Each rule is either a regex pattern
+   * with a replacement string, or a custom transform function.
+   * Rules are applied in order. Each rule receives the output of the previous.
+   */
+  rules: Array<ContentGuardRule>
+
+  /**
+   * Matching strategy:
+   * - 'delta': Apply rules to each delta as it arrives. Fast, real-time,
+   *   but patterns spanning chunk boundaries may be missed.
+   * - 'buffered': Accumulate content and apply rules to settled portions,
+   *   holding back a look-behind buffer to catch cross-boundary patterns.
+   *
+   * @default 'buffered'
+   */
+  strategy?: 'delta' | 'buffered'
+
+  /**
+   * Number of characters to hold back before emitting (buffered strategy only).
+   * Should be at least as long as the longest pattern you expect to match.
+   * Buffer is flushed when the stream ends.
+   *
+   * @default 50
+   */
+  bufferSize?: number
+
+  /**
+   * If true, drop the entire chunk when any rule changes the content.
+   * @default false
+   */
+  blockOnMatch?: boolean
+
+  /**
+   * Callback when content is filtered by any rule.
+   */
+  onFiltered?: (info: ContentFilteredInfo) => void
+}
+
+/**
+ * Apply all rules to a string, returning the transformed result.
+ */
+function applyRules(text: string, rules: Array<ContentGuardRule>): string {
+  let result = text
+  for (const rule of rules) {
+    if ('pattern' in rule) {
+      result = result.replace(rule.pattern, rule.replacement)
+    } else {
+      result = rule.fn(result)
+    }
+  }
+  return result
+}
+
+/**
+ * Creates a middleware that filters or transforms streamed text content.
+ *
+ * @example
+ * ```ts
+ * import { contentGuardMiddleware } from '@tanstack/ai/middlewares'
+ *
+ * const guard = contentGuardMiddleware({
+ *   rules: [
+ *     { pattern: /\b\d{3}-\d{2}-\d{4}\b/g, replacement: '[SSN REDACTED]' },
+ *   ],
+ *   strategy: 'buffered',
+ * })
+ * ```
+ */
+export function contentGuardMiddleware(
+  options: ContentGuardMiddlewareOptions,
+): ChatMiddleware {
+  const {
+    rules,
+    strategy = 'buffered',
+    bufferSize = 50,
+    blockOnMatch = false,
+    onFiltered,
+  } = options
+
+  if (strategy === 'delta') {
+    return createDeltaStrategy(rules, blockOnMatch, onFiltered)
+  }
+  return createBufferedStrategy(rules, bufferSize, blockOnMatch, onFiltered)
+}
+
+function createDeltaStrategy(
+  rules: Array<ContentGuardRule>,
+  blockOnMatch: boolean,
+  onFiltered?: (info: ContentFilteredInfo) => void,
+): ChatMiddleware {
+  return {
+    name: 'content-guard',
+
+    onChunk(_ctx: ChatMiddlewareContext, chunk: StreamChunk) {
+      if (chunk.type !== 'TEXT_MESSAGE_CONTENT') return
+
+      const original = chunk.delta
+      const filtered = applyRules(original, rules)
+
+      if (filtered === original) return // unchanged, pass through
+
+      if (onFiltered) {
+        onFiltered({
+          messageId: chunk.messageId,
+          original,
+          filtered,
+          strategy: 'delta',
+        })
+      }
+
+      if (blockOnMatch) return null // drop chunk
+
+      return {
+        ...chunk,
+        delta: filtered,
+        content: undefined,
+      } as StreamChunk
+    },
+  }
+}
+
+function createBufferedStrategy(
+  rules: Array<ContentGuardRule>,
+  bufferSize: number,
+  blockOnMatch: boolean,
+  onFiltered?: (info: ContentFilteredInfo) => void,
+): ChatMiddleware {
+  let rawAccumulated = ''
+  let emittedFilteredLength = 0
+  let lastMessageId = ''
+
+  function resetState() {
+    rawAccumulated = ''
+    emittedFilteredLength = 0
+    lastMessageId = ''
+  }
+
+  function flushBuffer(): StreamChunk | null {
+    if (rawAccumulated.length === 0) return null
+
+    const filtered = applyRules(rawAccumulated, rules)
+
+    if (blockOnMatch && filtered !== rawAccumulated) {
+      if (onFiltered) {
+        onFiltered({
+          messageId: lastMessageId,
+          original: rawAccumulated,
+          filtered,
+          strategy: 'buffered',
+        })
+      }
+      resetState()
+      return null
+    }
+
+    const remaining = filtered.slice(emittedFilteredLength)
+    if (remaining.length > 0) {
+      if (filtered !== rawAccumulated && onFiltered) {
+        onFiltered({
+          messageId: lastMessageId,
+          original: rawAccumulated,
+          filtered,
+          strategy: 'buffered',
+        })
+      }
+
+      const flushed = {
+        type: 'TEXT_MESSAGE_CONTENT',
+        messageId: lastMessageId,
+        delta: remaining,
+        content: filtered,
+        timestamp: Date.now(),
+      } as StreamChunk
+
+      resetState()
+      return flushed
+    }
+
+    resetState()
+    return null
+  }
+
+  return {
+    name: 'content-guard',
+
+    onStart() {
+      resetState()
+    },
+
+    onChunk(_ctx: ChatMiddlewareContext, chunk: StreamChunk) {
+      // Flush buffer on stream end events
+      if (chunk.type === 'TEXT_MESSAGE_END' || chunk.type === 'RUN_FINISHED') {
+        const flushed = flushBuffer()
+        if (flushed) return [flushed, chunk]
+        return // pass through end event
+      }
+
+      if (chunk.type !== 'TEXT_MESSAGE_CONTENT') return // pass through
+
+      // Flush buffer on message boundary change
+      const pending: Array<StreamChunk> = []
+      if (lastMessageId && chunk.messageId !== lastMessageId) {
+        const flushed = flushBuffer()
+        if (flushed) pending.push(flushed)
+      }
+
+      rawAccumulated += chunk.delta
+      lastMessageId = chunk.messageId
+
+      // Apply rules to full accumulated text, buffer in filtered space
+      const filtered = applyRules(rawAccumulated, rules)
+      const safeFilteredEnd = Math.max(0, filtered.length - bufferSize)
+
+      if (safeFilteredEnd <= emittedFilteredLength) {
+        return pending.length > 0 ? pending : null
+      }
+
+      if (blockOnMatch && filtered !== rawAccumulated) {
+        if (onFiltered) {
+          onFiltered({
+            messageId: chunk.messageId,
+            original: rawAccumulated,
+            filtered,
+            strategy: 'buffered',
+          })
+        }
+        return pending.length > 0 ? pending : null
+      }
+
+      const newDelta = filtered.slice(emittedFilteredLength, safeFilteredEnd)
+
+      if (filtered !== rawAccumulated && onFiltered) {
+        onFiltered({
+          messageId: chunk.messageId,
+          original: rawAccumulated,
+          filtered,
+          strategy: 'buffered',
+        })
+      }
+
+      emittedFilteredLength = safeFilteredEnd
+
+      const emitChunk = {
+        ...chunk,
+        delta: newDelta,
+        content: filtered.slice(0, safeFilteredEnd),
+      } as StreamChunk
+
+      pending.push(emitChunk)
+      return pending.length === 1 ? pending[0]! : pending
+    },
+  }
+}