@tanstack/ai 0.6.3 → 0.8.1

package/src/middlewares/content-guard.ts

@@ -0,0 +1,285 @@
+import type {
+  ChatMiddleware,
+  ChatMiddlewareContext,
+} from '../activities/chat/middleware/types'
+import type { StreamChunk } from '../types'
+
+/**
+ * A content guard rule — either a regex pattern with replacement, or a transform function.
+ */
+export type ContentGuardRule =
+  | { pattern: RegExp; replacement: string }
+  | { fn: (text: string) => string }
+
+/**
+ * Information passed to the onFiltered callback.
+ */
+export interface ContentFilteredInfo {
+  /** The message ID being filtered */
+  messageId: string
+  /** The original text before filtering */
+  original: string
+  /** The filtered text after rules applied */
+  filtered: string
+  /** Which strategy was used */
+  strategy: 'delta' | 'buffered'
+}
+
+/**
+ * Options for the content guard middleware.
+ */
+export interface ContentGuardMiddlewareOptions {
+  /**
+   * Rules to apply to text content. Each rule is either a regex pattern
+   * with a replacement string, or a custom transform function.
+   * Rules are applied in order. Each rule receives the output of the previous.
+   */
+  rules: Array<ContentGuardRule>
+
+  /**
+   * Matching strategy:
+   * - 'delta': Apply rules to each delta as it arrives. Fast, real-time,
+   *   but patterns spanning chunk boundaries may be missed.
+   * - 'buffered': Accumulate content and apply rules to settled portions,
+   *   holding back a look-behind buffer to catch cross-boundary patterns.
+   *
+   * @default 'buffered'
+   */
+  strategy?: 'delta' | 'buffered'
+
+  /**
+   * Number of characters to hold back before emitting (buffered strategy only).
+   * Should be at least as long as the longest pattern you expect to match.
+   * Buffer is flushed when the stream ends.
+   *
+   * @default 50
+   */
+  bufferSize?: number
+
+  /**
+   * If true, drop the entire chunk when any rule changes the content.
+   * @default false
+   */
+  blockOnMatch?: boolean
+
+  /**
+   * Callback when content is filtered by any rule.
+   */
+  onFiltered?: (info: ContentFilteredInfo) => void
+}
+
+/**
+ * Apply all rules to a string, returning the transformed result.
+ */
+function applyRules(text: string, rules: Array<ContentGuardRule>): string {
+  let result = text
+  for (const rule of rules) {
+    if ('pattern' in rule) {
+      result = result.replace(rule.pattern, rule.replacement)
+    } else {
+      result = rule.fn(result)
+    }
+  }
+  return result
+}
+
+/**
+ * Creates a middleware that filters or transforms streamed text content.
+ *
+ * @example
+ * ```ts
+ * import { contentGuardMiddleware } from '@tanstack/ai/middlewares'
+ *
+ * const guard = contentGuardMiddleware({
+ *   rules: [
+ *     { pattern: /\b\d{3}-\d{2}-\d{4}\b/g, replacement: '[SSN REDACTED]' },
+ *   ],
+ *   strategy: 'buffered',
+ * })
+ * ```
+ */
+export function contentGuardMiddleware(
+  options: ContentGuardMiddlewareOptions,
+): ChatMiddleware {
+  const {
+    rules,
+    strategy = 'buffered',
+    bufferSize = 50,
+    blockOnMatch = false,
+    onFiltered,
+  } = options
+
+  if (strategy === 'delta') {
+    return createDeltaStrategy(rules, blockOnMatch, onFiltered)
+  }
+  return createBufferedStrategy(rules, bufferSize, blockOnMatch, onFiltered)
+}
+
+function createDeltaStrategy(
+  rules: Array<ContentGuardRule>,
+  blockOnMatch: boolean,
+  onFiltered?: (info: ContentFilteredInfo) => void,
+): ChatMiddleware {
+  return {
+    name: 'content-guard',
+
+    onChunk(_ctx: ChatMiddlewareContext, chunk: StreamChunk) {
+      if (chunk.type !== 'TEXT_MESSAGE_CONTENT') return
+
+      const original = chunk.delta
+      const filtered = applyRules(original, rules)
+
+      if (filtered === original) return // unchanged, pass through
+
+      if (onFiltered) {
+        onFiltered({
+          messageId: chunk.messageId,
+          original,
+          filtered,
+          strategy: 'delta',
+        })
+      }
+
+      if (blockOnMatch) return null // drop chunk
+
+      return {
+        ...chunk,
+        delta: filtered,
+        content: undefined,
+      } as StreamChunk
+    },
+  }
+}
+
+function createBufferedStrategy(
+  rules: Array<ContentGuardRule>,
+  bufferSize: number,
+  blockOnMatch: boolean,
+  onFiltered?: (info: ContentFilteredInfo) => void,
+): ChatMiddleware {
+  let rawAccumulated = ''
+  let emittedFilteredLength = 0
+  let lastMessageId = ''
+
+  function resetState() {
+    rawAccumulated = ''
+    emittedFilteredLength = 0
+    lastMessageId = ''
+  }
+
+  function flushBuffer(): StreamChunk | null {
+    if (rawAccumulated.length === 0) return null
+
+    const filtered = applyRules(rawAccumulated, rules)
+
+    if (blockOnMatch && filtered !== rawAccumulated) {
+      if (onFiltered) {
+        onFiltered({
+          messageId: lastMessageId,
+          original: rawAccumulated,
+          filtered,
+          strategy: 'buffered',
+        })
+      }
+      resetState()
+      return null
+    }
+
+    const remaining = filtered.slice(emittedFilteredLength)
+    if (remaining.length > 0) {
+      if (filtered !== rawAccumulated && onFiltered) {
+        onFiltered({
+          messageId: lastMessageId,
+          original: rawAccumulated,
+          filtered,
+          strategy: 'buffered',
+        })
+      }
+
+      const flushed = {
+        type: 'TEXT_MESSAGE_CONTENT',
+        messageId: lastMessageId,
+        delta: remaining,
+        content: filtered,
+        timestamp: Date.now(),
+      } as StreamChunk
+
+      resetState()
+      return flushed
+    }
+
+    resetState()
+    return null
+  }
+
+  return {
+    name: 'content-guard',
+
+    onStart() {
+      resetState()
+    },
+
+    onChunk(_ctx: ChatMiddlewareContext, chunk: StreamChunk) {
+      // Flush buffer on stream end events
+      if (chunk.type === 'TEXT_MESSAGE_END' || chunk.type === 'RUN_FINISHED') {
+        const flushed = flushBuffer()
+        if (flushed) return [flushed, chunk]
+        return // pass through end event
+      }
+
+      if (chunk.type !== 'TEXT_MESSAGE_CONTENT') return // pass through
+
+      // Flush buffer on message boundary change
+      const pending: Array<StreamChunk> = []
+      if (lastMessageId && chunk.messageId !== lastMessageId) {
+        const flushed = flushBuffer()
+        if (flushed) pending.push(flushed)
+      }
+
+      rawAccumulated += chunk.delta
+      lastMessageId = chunk.messageId
+
+      // Apply rules to full accumulated text, buffer in filtered space
+      const filtered = applyRules(rawAccumulated, rules)
+      const safeFilteredEnd = Math.max(0, filtered.length - bufferSize)
+
+      if (safeFilteredEnd <= emittedFilteredLength) {
+        return pending.length > 0 ? pending : null
+      }
+
+      if (blockOnMatch && filtered !== rawAccumulated) {
+        if (onFiltered) {
+          onFiltered({
+            messageId: chunk.messageId,
+            original: rawAccumulated,
+            filtered,
+            strategy: 'buffered',
+          })
+        }
+        return pending.length > 0 ? pending : null
+      }
+
+      const newDelta = filtered.slice(emittedFilteredLength, safeFilteredEnd)
+
+      if (filtered !== rawAccumulated && onFiltered) {
+        onFiltered({
+          messageId: chunk.messageId,
+          original: rawAccumulated,
+          filtered,
+          strategy: 'buffered',
+        })
+      }
+
+      emittedFilteredLength = safeFilteredEnd
+
+      const emitChunk = {
+        ...chunk,
+        delta: newDelta,
+        content: filtered.slice(0, safeFilteredEnd),
+      } as StreamChunk
+
+      pending.push(emitChunk)
+      return pending.length === 1 ? pending[0]! : pending
+    },
+  }
+}

package/src/middlewares/index.ts

@@ -0,0 +1,13 @@
+export {
+  toolCacheMiddleware,
+  type ToolCacheMiddlewareOptions,
+  type ToolCacheStorage,
+  type ToolCacheEntry,
+} from './tool-cache'
+
+export {
+  contentGuardMiddleware,
+  type ContentGuardMiddlewareOptions,
+  type ContentGuardRule,
+  type ContentFilteredInfo,
+} from './content-guard'

package/src/middlewares/tool-cache.ts

@@ -0,0 +1,6 @@
+export {
+  toolCacheMiddleware,
+  type ToolCacheMiddlewareOptions,
+  type ToolCacheStorage,
+  type ToolCacheEntry,
+} from '../activities/chat/middleware/tool-cache-middleware'

package/src/realtime/index.ts

@@ -0,0 +1,38 @@
+import type { RealtimeToken, RealtimeTokenOptions } from './types'
+
+// Re-export all types
+export * from './types'
+
+/**
+ * Generate a realtime token using the provided adapter.
+ *
+ * This function is used on the server to generate ephemeral tokens
+ * that clients can use to establish realtime connections.
+ *
+ * @param options - Token generation options including the adapter
+ * @returns Promise resolving to a RealtimeToken
+ *
+ * @example
+ * ```typescript
+ * import { realtimeToken } from '@tanstack/ai'
+ * import { openaiRealtimeToken } from '@tanstack/ai-openai'
+ *
+ * // Server function (TanStack Start example)
+ * export const getRealtimeToken = createServerFn()
+ *   .handler(async () => {
+ *     return realtimeToken({
+ *       adapter: openaiRealtimeToken({
+ *         model: 'gpt-4o-realtime-preview',
+ *         voice: 'alloy',
+ *         instructions: 'You are a helpful assistant...',
+ *       }),
+ *     })
+ *   })
+ * ```
+ */
+export async function realtimeToken(
+  options: RealtimeTokenOptions,
+): Promise<RealtimeToken> {
+  const { adapter } = options
+  return adapter.generateToken()
+}

package/src/realtime/types.ts

@@ -0,0 +1,294 @@
+// ============================================================================
+// Token Types
+// ============================================================================
+
+/**
+ * Voice activity detection configuration
+ */
+export interface VADConfig {
+  /** Sensitivity threshold (0.0-1.0) */
+  threshold?: number
+  /** Audio to include before speech detection (ms) */
+  prefixPaddingMs?: number
+  /** Silence duration to end turn (ms) */
+  silenceDurationMs?: number
+}
+
+/**
+ * Serializable tool descriptor for realtime session configuration.
+ * Contains only the metadata needed by providers, not Zod schemas or execute functions.
+ */
+export interface RealtimeToolConfig {
+  name: string
+  description: string
+  inputSchema?: Record<string, any>
+}
+
+/**
+ * Configuration for a realtime session
+ */
+export interface RealtimeSessionConfig {
+  /** Model to use for the session */
+  model?: string
+  /** Voice to use for audio output */
+  voice?: string
+  /** System instructions for the assistant */
+  instructions?: string
+  /** Tools available in the session */
+  tools?: Array<RealtimeToolConfig>
+  /** VAD mode */
+  vadMode?: 'server' | 'semantic' | 'manual'
+  /** VAD configuration */
+  vadConfig?: VADConfig
+  /** Output modalities for responses (e.g., ['audio', 'text'], ['text']) */
+  outputModalities?: Array<'audio' | 'text'>
+  /** Temperature for generation (provider-specific range, e.g., 0.6-1.2 for OpenAI) */
+  temperature?: number
+  /** Maximum number of tokens in a response */
+  maxOutputTokens?: number | 'inf'
+  /** Eagerness level for semantic VAD ('low', 'medium', 'high') */
+  semanticEagerness?: 'low' | 'medium' | 'high'
+  /** Provider-specific options */
+  providerOptions?: Record<string, any>
+}
+
+/**
+ * Token returned by the server for client authentication
+ */
+export interface RealtimeToken {
+  /** Provider identifier */
+  provider: string
+  /** The ephemeral token value */
+  token: string
+  /** Token expiration timestamp (ms since epoch) */
+  expiresAt: number
+  /** Session configuration embedded in the token */
+  config: RealtimeSessionConfig
+}
+
+/**
+ * Adapter interface for generating provider-specific tokens
+ */
+export interface RealtimeTokenAdapter {
+  /** Provider identifier */
+  provider: string
+  /** Generate an ephemeral token for client use */
+  generateToken: () => Promise<RealtimeToken>
+}
+
+/**
+ * Options for the realtimeToken function
+ */
+export interface RealtimeTokenOptions {
+  /** The token adapter to use */
+  adapter: RealtimeTokenAdapter
+}
+
+// ============================================================================
+// Message Types
+// ============================================================================
+
+/**
+ * Text content part in a realtime message
+ */
+export interface RealtimeTextPart {
+  type: 'text'
+  content: string
+}
+
+/**
+ * Audio content part in a realtime message
+ */
+export interface RealtimeAudioPart {
+  type: 'audio'
+  /** Transcription of the audio */
+  transcript: string
+  /** Raw audio data (optional, if stored) */
+  audioData?: ArrayBuffer
+  /** Duration of the audio in milliseconds */
+  durationMs?: number
+}
+
+/**
+ * Tool call part in a realtime message
+ */
+export interface RealtimeToolCallPart {
+  type: 'tool-call'
+  id: string
+  name: string
+  arguments: string
+  input?: unknown
+  output?: unknown
+}
+
+/**
+ * Tool result part in a realtime message
+ */
+export interface RealtimeToolResultPart {
+  type: 'tool-result'
+  toolCallId: string
+  content: string
+}
+
+/**
+ * Image content part in a realtime message
+ */
+export interface RealtimeImagePart {
+  type: 'image'
+  /** Base64-encoded image data or a URL */
+  data: string
+  /** MIME type of the image (e.g., 'image/png', 'image/jpeg') */
+  mimeType: string
+}
+
+/**
+ * Union of all realtime message parts
+ */
+export type RealtimeMessagePart =
+  | RealtimeTextPart
+  | RealtimeAudioPart
+  | RealtimeToolCallPart
+  | RealtimeToolResultPart
+  | RealtimeImagePart
+
+/**
+ * A message in a realtime conversation
+ */
+export interface RealtimeMessage {
+  /** Unique message identifier */
+  id: string
+  /** Message role */
+  role: 'user' | 'assistant'
+  /** Timestamp when the message was created */
+  timestamp: number
+  /** Content parts of the message */
+  parts: Array<RealtimeMessagePart>
+  /** Whether this message was interrupted */
+  interrupted?: boolean
+  /** Reference to audio buffer if stored */
+  audioId?: string
+  /** Duration of the audio in milliseconds */
+  durationMs?: number
+}
+
+// ============================================================================
+// Status Types
+// ============================================================================
+
+/**
+ * Connection status of the realtime client
+ */
+export type RealtimeStatus =
+  | 'idle'
+  | 'connecting'
+  | 'connected'
+  | 'reconnecting'
+  | 'error'
+
+/**
+ * Current mode of the realtime session
+ */
+export type RealtimeMode = 'idle' | 'listening' | 'thinking' | 'speaking'
+
+// ============================================================================
+// Audio Visualization Types
+// ============================================================================
+
+/**
+ * Interface for accessing audio visualization data
+ */
+export interface AudioVisualization {
+  /** Input volume level (0-1 normalized) */
+  readonly inputLevel: number
+  /** Output volume level (0-1 normalized) */
+  readonly outputLevel: number
+
+  /** Get frequency data for input audio visualization */
+  getInputFrequencyData: () => Uint8Array
+  /** Get frequency data for output audio visualization */
+  getOutputFrequencyData: () => Uint8Array
+
+  /** Get time domain data for input waveform */
+  getInputTimeDomainData: () => Uint8Array
+  /** Get time domain data for output waveform */
+  getOutputTimeDomainData: () => Uint8Array
+
+  /** Input sample rate */
+  readonly inputSampleRate: number
+  /** Output sample rate */
+  readonly outputSampleRate: number
+
+  /** Subscribe to raw input audio samples */
+  onInputAudio?: (
+    callback: (samples: Float32Array, sampleRate: number) => void,
+  ) => () => void
+  /** Subscribe to raw output audio samples */
+  onOutputAudio?: (
+    callback: (samples: Float32Array, sampleRate: number) => void,
+  ) => () => void
+}
+
+// ============================================================================
+// Event Types
+// ============================================================================
+
+/**
+ * Events emitted by the realtime connection
+ */
+export type RealtimeEvent =
+  | 'status_change'
+  | 'mode_change'
+  | 'transcript'
+  | 'audio_chunk'
+  | 'tool_call'
+  | 'message_complete'
+  | 'interrupted'
+  | 'error'
+
+/**
+ * Event payloads for realtime events
+ */
+export interface RealtimeEventPayloads {
+  status_change: { status: RealtimeStatus }
+  mode_change: { mode: RealtimeMode }
+  transcript: {
+    role: 'user' | 'assistant'
+    transcript: string
+    isFinal: boolean
+  }
+  audio_chunk: { data: ArrayBuffer; sampleRate: number }
+  tool_call: { toolCallId: string; toolName: string; input: unknown }
+  message_complete: { message: RealtimeMessage }
+  interrupted: { messageId?: string }
+  error: { error: Error }
+}
+
+/**
+ * Handler type for realtime events
+ */
+export type RealtimeEventHandler<TEvent extends RealtimeEvent> = (
+  payload: RealtimeEventPayloads[TEvent],
+) => void
+
+// ============================================================================
+// Error Types
+// ============================================================================
+
+/**
+ * Error codes for realtime errors
+ */
+export type RealtimeErrorCode =
+  | 'TOKEN_EXPIRED'
+  | 'CONNECTION_FAILED'
+  | 'PERMISSION_DENIED'
+  | 'PROVIDER_ERROR'
+  | 'UNKNOWN'
+
+/**
+ * Extended error with realtime-specific information
+ */
+export interface RealtimeError extends Error {
+  code: RealtimeErrorCode
+  provider?: string
+  details?: unknown
+}

package/src/types.ts

@@ -730,6 +730,7 @@ export type AGUIEventType =
   | 'TOOL_CALL_END'
   | 'STEP_STARTED'
   | 'STEP_FINISHED'
+  | 'MESSAGES_SNAPSHOT'
   | 'STATE_SNAPSHOT'
   | 'STATE_DELTA'
   | 'CUSTOM'
@@ -806,8 +807,8 @@ export interface TextMessageStartEvent extends BaseAGUIEvent {
   type: 'TEXT_MESSAGE_START'
   /** Unique identifier for this message */
   messageId: string
-  /** Role is always assistant for generated messages */
-  role: 'assistant'
+  /** Role of the message sender */
+  role: 'user' | 'assistant' | 'system' | 'tool'
 }
 
 /**
@@ -841,6 +842,8 @@ export interface ToolCallStartEvent extends BaseAGUIEvent {
   toolCallId: string
   /** Name of the tool being called */
   toolName: string
+  /** ID of the parent message that initiated this tool call */
+  parentMessageId?: string
   /** Index for parallel tool calls */
   index?: number
 }
@@ -897,6 +900,19 @@ export interface StepFinishedEvent extends BaseAGUIEvent {
   content?: string
 }
 
+/**
+ * Emitted to provide a snapshot of all messages in a conversation.
+ *
+ * Unlike StateSnapshot (which carries arbitrary application state),
+ * MessagesSnapshot specifically delivers the conversation transcript.
+ * This is a first-class AG-UI event type.
+ */
+export interface MessagesSnapshotEvent extends BaseAGUIEvent {
+  type: 'MESSAGES_SNAPSHOT'
+  /** Complete array of messages in the conversation */
+  messages: Array<UIMessage>
+}
+
 /**
  * Emitted to provide a full state snapshot.
  */
@@ -941,6 +957,7 @@ export type AGUIEvent =
   | ToolCallEndEvent
   | StepStartedEvent
   | StepFinishedEvent
+  | MessagesSnapshotEvent
   | StateSnapshotEvent
   | StateDeltaEvent
   | CustomEvent