@tanstack/ai 0.53.0 → 0.55.0

package/src/activities/chat/index.ts

@@ -63,10 +63,12 @@ import {
 import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies'
 import { isCancelRequestedReason } from './cancel'
 import {
+  appendUiResourceToModelMessages,
   convertMessagesToModelMessages,
   generateMessageId,
   modelMessagesToUIMessages,
   safeJsonStringify,
+  uiResourcePartFromCustomValue,
 } from './messages'
 import { MiddlewareRunner } from './middleware/compose'
 import { getRunDetached } from './middleware/run-store'
@@ -1245,7 +1247,7 @@ class TextEngine<
                 ...this.deferredModelRunFinishedChunks,
               )
               this.deferredModelRunFinishedChunks = []
-            } else {
+            } else if (!this.finalStructuredOutput?.nativeCombined) {
               yield* this.flushDeferredModelRunFinishedChunks()
             }
           } else {
@@ -1279,6 +1281,9 @@ class TextEngine<
       ) {
         if (this.finalStructuredOutput.nativeCombined === true) {
           yield* this.harvestCombinedStructuredOutput()
+          if (!this.finalizationError && !this.isCancelled()) {
+            yield* this.flushDeferredModelRunFinishedChunks()
+          }
         } else {
           yield* this.runStructuredFinalization()
         }
@@ -2767,6 +2772,22 @@ class TextEngine<
     }
   }
 
+  /**
+   * Record a `ui-resource` CUSTOM chunk on the assistant ModelMessage owning
+   * its `toolCallId` so the resource survives later MESSAGES_SNAPSHOT chunks
+   * (e.g. the interrupt snapshot emitted when the run pauses on a client
+   * tool). Mirrors the anchor-preserving approach used for
+   * `toolCallMetadata` (#867). See #1397.
+   */
+  private recordEmittedUiResource(value: unknown): void {
+    const part = uiResourcePartFromCustomValue(value)
+    if (!part) return
+    const next = appendUiResourceToModelMessages(this.messages, part)
+    if (next === this.messages) return
+    this.messages = next
+    this.middlewareCtx.messages = this.messages
+  }
+
   private buildMessagesSnapshotChunk(): StreamChunk {
     const withIds = this.messages.map((message, index) => ({
       ...message,
@@ -3926,12 +3947,8 @@ class TextEngine<
     }
 
     // On success, emit the synthetic `structured-output.complete` carrying
-    // the parsed object + raw text. Pin the messageId so the client-side
-    // handler can target the right UIMessage even when the agent loop's
-    // terminal RUN_FINISHED has already cleared `activeMessageIds` (the
-    // complete event yields AFTER the loop ends, by which point
-    // `getActiveAssistantMessageId()` returns null and would otherwise drop
-    // the event silently).
+    // the parsed object + raw text before the deferred RUN_FINISHED. Pin
+    // the messageId so the client targets the schema-constrained turn.
     if (
       this.structuredOutputResult &&
       !this.finalizationError &&
@@ -4432,6 +4449,11 @@ class TextEngine<
           if (this.hasPublicRunStarted) continue
           this.hasPublicRunStarted = true
         }
+        // Persist MCP Apps ui-resource emissions onto the tool-call anchor
+        // message so interrupt MESSAGES_SNAPSHOT chunks keep them (#1397).
+        if (spec.type === EventType.CUSTOM && spec.name === 'ui-resource') {
+          this.recordEmittedUiResource((spec as CustomEvent).value)
+        }
         yield spec
         this.middlewareCtx.chunkIndex++
       }

package/src/activities/chat/messages.ts

@@ -387,6 +387,66 @@ function appendUiResources(
   return { ...ui, parts: [...ui.parts, ...extra] }
 }
 
+/**
+ * Build a UIResourcePart from the value of a CUSTOM `ui-resource` chunk
+ * emitted via `ctx.emitCustomEvent('ui-resource', ...)` (MCP Apps). The
+ * emission-side value carries `resource`/`serverId`/`toolName` plus the
+ * `toolCallId` stamped by the tool-call context wrapper — the `type`
+ * discriminator is added here. Returns undefined when the value does not
+ * match the ui-resource shape.
+ */
+export function uiResourcePartFromCustomValue(
+  value: unknown,
+): UIResourcePart | undefined {
+  if (!isRecord(value)) return undefined
+  const part: unknown = { type: 'ui-resource', ...value }
+  return isUiResourcePart(part) ? part : undefined
+}
+
+/**
+ * Store an emitted ui-resource part on the assistant ModelMessage that owns
+ * its `toolCallId` (the tool-call anchor), so it survives later
+ * MESSAGES_SNAPSHOT chunks — e.g. the interrupt snapshot emitted when the
+ * run pauses on a client tool (#1397). Mirrors how `toolCallMetadata` is
+ * preserved on the anchor (#867).
+ *
+ * Returns the SAME array reference when no anchor owns the tool call or the
+ * resource is already stored (idempotent).
+ */
+export function appendUiResourceToModelMessages(
+  messages: Array<ModelMessage>,
+  part: UIResourcePart,
+): Array<ModelMessage> {
+  for (let index = messages.length - 1; index >= 0; index--) {
+    const message = messages[index]
+    if (!message || message.role !== 'assistant') continue
+    const ownsToolCall = message.toolCalls?.some(
+      (toolCall) => toolCall.id === part.toolCallId,
+    )
+    if (!ownsToolCall) continue
+    const previous = tanstackMetadata(message)?.uiResources ?? []
+    if (
+      previous.some((stored) => uiResourceKey(stored) === uiResourceKey(part))
+    ) {
+      return messages
+    }
+    const nextMessage = {
+      ...message,
+      metadata: {
+        ...message.metadata,
+        tanstack: {
+          ...tanstackMetadata(message),
+          uiResources: [...previous, part],
+        },
+      },
+    }
+    const next = messages.slice()
+    next[index] = nextMessage
+    return next
+  }
+  return messages
+}
+
 function assistantMetadata(
   uiMessage: UIMessage,
 ): UIMessage['metadata'] | undefined {

package/src/activities/chat/stream/processor.ts

@@ -242,6 +242,37 @@ export class StreamProcessor {
     this.emitMessagesChange()
   }
 
+  /**
+   * Put older UI messages at the front of the conversation.
+   *
+   * Skip a message if its id is already in the list. Keep the existing message.
+   * Then emit the same messages-change event as `setMessages`.
+   *
+   * Use this for older history pages. The first hydrate window uses `setMessages`.
+   *
+   * @param messages Older UI messages in insertion order. The first item is the oldest.
+   *
+   * @example
+   * ```ts
+   * processor.setMessages([newest])
+   * processor.prependMessages([oldest])
+   * ```
+   */
+  prependMessages(messages: Array<UIMessage>) {
+    const existingIds = new Set(this.messages.map((message) => message.id))
+    const olderMessages: Array<UIMessage> = []
+    for (const message of messages) {
+      const isDuplicate = existingIds.has(message.id)
+      if (isDuplicate) {
+        continue
+      }
+      existingIds.add(message.id)
+      olderMessages.push(message)
+    }
+    this.messages = [...olderMessages, ...this.messages]
+    this.emitMessagesChange()
+  }
+
   /**
    * Add a user message to the conversation.
    * Supports both simple string content and multimodal content arrays.

package/src/activities/generateLiveVideo/adapter.ts

@@ -0,0 +1,99 @@
+import type {
+  LiveVideoGenerationOptions,
+  LiveVideoGenerationResult,
+} from '../../types'
+
+/**
+ * Configuration for live generation adapter instances.
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export interface LiveVideoAdapterConfig {
+  apiKey?: string
+  baseUrl?: string
+  timeout?: number
+  maxRetries?: number
+  headers?: Record<string, string>
+}
+
+/**
+ * Live adapter interface with pre-resolved generics.
+ *
+ * An adapter is created by a provider function: `provider('model')` → `adapter`.
+ * All type resolution happens at the provider call site, not in this interface.
+ *
+ * Generic parameters:
+ * - TModel: The specific model name (e.g. 'helios')
+ * - TProviderOptions: Provider-specific options (already resolved)
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export interface LiveVideoAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> {
+  /** Discriminator for adapter kind - used to determine API shape */
+  readonly kind: 'liveVideo'
+  /** Adapter name identifier */
+  readonly name: string
+  /** The model this adapter is configured for */
+  readonly model: TModel
+
+  /**
+   * @internal Type-only properties for inference. Not assigned at runtime.
+   */
+  '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  /**
+   * Open a live video session from a prompt.
+   *
+   * Server adapters typically mint a short-lived token and return it with the
+   * prompt so a browser can connect, set the prompt, and start streaming.
+   */
+  createLiveVideo: (
+    options: LiveVideoGenerationOptions<TProviderOptions>,
+  ) => Promise<LiveVideoGenerationResult>
+}
+
+/**
+ * A LiveVideoAdapter with any/unknown type parameters.
+ * Useful as a constraint in generic functions and interfaces.
+ */
+export type AnyLiveVideoAdapter = LiveVideoAdapter<any, any>
+
+/**
+ * Abstract base class for live generation adapters.
+ * Extend this class to implement a live adapter for a specific provider.
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export abstract class BaseLiveVideoAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> implements LiveVideoAdapter<TModel, TProviderOptions> {
+  readonly kind = 'liveVideo' as const
+  abstract readonly name: string
+  readonly model: TModel
+
+  // Type-only property - never assigned at runtime
+  declare '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  protected config: LiveVideoAdapterConfig
+
+  constructor(model: TModel, config: LiveVideoAdapterConfig = {}) {
+    this.config = config
+    this.model = model
+  }
+
+  abstract createLiveVideo(
+    options: LiveVideoGenerationOptions<TProviderOptions>,
+  ): Promise<LiveVideoGenerationResult>
+
+  protected generateId(): string {
+    return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
+  }
+}

package/src/activities/generateLiveVideo/index.ts

@@ -0,0 +1,339 @@
+/**
+ * Live Activity (Experimental)
+ *
+ * Mints a session token for a live, prompt-steerable video session. Unlike
+ * generateVideo (a job that finishes with a URL), the browser then connects
+ * with the token, sets the prompt, and streams until stop/close.
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
+import { resolveDebugOption } from '../../logger/resolve'
+import {
+  applyGenerationResultTransforms,
+  createGenerationContext,
+  runGenerationAbort,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware/run'
+import {
+  abortReasonMessage,
+  createActivityAbortControls,
+  isActivityAbortError,
+  raceWithAbort,
+} from '../../utilities/activity-abort'
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware/types'
+import type { LiveVideoAdapter } from './adapter'
+import type { StreamChunk, LiveVideoGenerationResult } from '../../types'
+
+// ===========================
+// Activity Kind
+// ===========================
+
+/** The adapter kind this activity handles */
+export const kind = 'liveVideo' as const
+
+// ===========================
+// Type Extraction Helpers
+// ===========================
+
+/**
+ * Extract provider options from a LiveVideoAdapter via ~types.
+ */
+export type LiveVideoProviderOptions<TAdapter> = TAdapter extends {
+  '~types': { providerOptions: infer P extends object }
+}
+  ? P
+  : object
+
+// ===========================
+// Activity Options Type
+// ===========================
+
+/**
+ * Options for the live generation activity.
+ * The model is extracted from the adapter's model property.
+ *
+ * @template TAdapter - The live adapter type
+ * @template TStream - Whether to stream the output
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export interface LiveVideoActivityOptions<
+  TAdapter extends LiveVideoAdapter<string, LiveVideoProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> {
+  /** The live adapter to use (must be created with a model) */
+  adapter: TAdapter & { kind: typeof kind }
+  /** Natural-language description of the shot or scene */
+  prompt: string
+  /** Provider-specific options for live generation */
+  modelOptions?: LiveVideoProviderOptions<TAdapter>
+  /**
+   * Whether to wrap the token result as StreamChunks for SSE transport.
+   * This is not the live video. When false or omitted, returns
+   * Promise<LiveVideoGenerationResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /**
+   * Enable debug logging. Pass `true` to enable all categories, `false` to
+   * silence everything including errors, or a `DebugConfig` object for granular
+   * control and/or a custom `Logger`.
+   */
+  debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
+  /** Stable conversation/thread id for correlating this run when persisted. */
+  threadId?: string
+  /** Stable run id for correlating this run when persisted. */
+  runId?: string
+  /**
+   * Maximum duration of the token mint in milliseconds.
+   * No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
+   */
+  timeout?: number
+  /**
+   * Caller cancellation signal (request disconnects, job/runtime cancellation).
+   * Composed with {@link timeout} into an effective signal forwarded to the
+   * adapter. Request-specific — not stored on global provider client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+// ===========================
+// Activity Result Type
+// ===========================
+
+/**
+ * Result type for the live generation activity.
+ * - If stream is true: AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<LiveVideoGenerationResult>
+ */
+export type LiveVideoActivityResult<TStream extends boolean = false> =
+  TStream extends true
+    ? AsyncIterable<StreamChunk>
+    : Promise<LiveVideoGenerationResult>
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+// ===========================
+// Activity Implementation
+// ===========================
+
+/**
+ * Live generation activity - opens a live, prompt-steerable video session.
+ *
+ * @example Mint a session token on the server
+ * ```ts
+ * import { generateLiveVideo } from '@tanstack/ai'
+ * import { reactorVideo } from '@tanstack/ai-reactor'
+ *
+ * const live = await generateLiveVideo({
+ *   adapter: reactorVideo('helios'),
+ *   prompt: 'A red sports car powerslides a mountain hairpin',
+ * })
+ *
+ * // Hand live.token, live.model, and live.prompt to the browser.
+ * ```
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export function generateLiveVideo<
+  TAdapter extends LiveVideoAdapter<string, LiveVideoProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: LiveVideoActivityOptions<TAdapter, TStream>,
+): LiveVideoActivityResult<TStream> {
+  if (options.stream) {
+    return streamGenerationResult(
+      (resolved) => runGenerateLiveVideo({ ...options, runId: resolved.runId }),
+      options,
+    ) as LiveVideoActivityResult<TStream>
+  }
+  return runGenerateLiveVideo(options) as LiveVideoActivityResult<TStream>
+}
+
+/**
+ * Run the core live generation logic (non-streaming).
+ */
+async function runGenerateLiveVideo<
+  TAdapter extends LiveVideoAdapter<string, LiveVideoProviderOptions<TAdapter>>,
+>(
+  options: LiveVideoActivityOptions<TAdapter, boolean>,
+): Promise<LiveVideoGenerationResult> {
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    threadId,
+    runId,
+    timeout,
+    abortSignal: callerAbortSignal,
+    ...rest
+  } = options
+  const model = adapter.model
+  const requestId = createId('liveVideo')
+  const startTime = Date.now()
+  const logger: InternalLogger = resolveDebugOption(options.debug)
+  const abortControls = createActivityAbortControls({
+    timeout,
+    abortSignal: callerAbortSignal,
+  })
+  const providerName =
+    (adapter as { name?: string; provider?: string }).provider ??
+    (adapter as { name?: string }).name ??
+    'unknown'
+
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'liveVideo',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    threadId,
+    runId,
+    artifactInputs: { prompt: rest.prompt },
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
+  aiEventClient.emit('liveVideo:request:started', {
+    requestId,
+    provider: adapter.name,
+    model,
+    prompt: rest.prompt,
+    timestamp: startTime,
+    ...(rest.modelOptions !== undefined && {
+      modelOptions: rest.modelOptions as Record<string, unknown>,
+    }),
+  })
+
+  logger.request(`activity=generateLiveVideo provider=${providerName}`, {
+    provider: providerName,
+    model,
+  })
+
+  try {
+    const rawResult = await raceWithAbort(
+      adapter.createLiveVideo({
+        ...rest,
+        model,
+        logger,
+        ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
+      }),
+      abortControls.signal,
+    )
+    abortControls.clear()
+    const result = await applyGenerationResultTransforms(mwCtx, rawResult)
+    const elapsedMs = Date.now() - startTime
+
+    aiEventClient.emit('liveVideo:request:completed', {
+      requestId,
+      provider: adapter.name,
+      model: result.model,
+      prompt: result.prompt,
+      status: result.status,
+      duration: elapsedMs,
+      timestamp: Date.now(),
+      ...(rest.modelOptions !== undefined && {
+        modelOptions: rest.modelOptions as Record<string, unknown>,
+      }),
+    })
+
+    if (result.usage) {
+      aiEventClient.emit('liveVideo:usage', {
+        requestId,
+        model: result.model,
+        usage: result.usage,
+        timestamp: Date.now(),
+        ...(rest.modelOptions !== undefined && {
+          modelOptions: rest.modelOptions as Record<string, unknown>,
+        }),
+      })
+    }
+
+    logger.output(`activity=generateLiveVideo provider=${providerName}`, {
+      model: result.model,
+      status: result.status,
+    })
+
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration: elapsedMs,
+      usage: result.usage,
+    })
+
+    return result
+  } catch (error) {
+    abortControls.clear()
+    const elapsedMs = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('liveVideo:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration: elapsedMs,
+      timestamp: Date.now(),
+      ...(rest.modelOptions !== undefined && {
+        modelOptions: rest.modelOptions as Record<string, unknown>,
+      }),
+    })
+    if (isActivityAbortError(error, abortControls.signal)) {
+      await runGenerationAbort(middleware, mwCtx, {
+        reason: abortReasonMessage(error, abortControls.signal),
+        duration: elapsedMs,
+      })
+    } else {
+      await runGenerationError(middleware, mwCtx, {
+        error,
+        duration: elapsedMs,
+      })
+    }
+    logger.errors('generateLiveVideo activity failed', {
+      error,
+      source: 'generateLiveVideo',
+    })
+    throw error
+  }
+}
+
+// ===========================
+// Options Factory
+// ===========================
+
+/**
+ * Create typed options for the generateLiveVideo() function without executing.
+ */
+export function createLiveVideoOptions<
+  TAdapter extends LiveVideoAdapter<string, LiveVideoProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: LiveVideoActivityOptions<TAdapter, TStream>,
+): LiveVideoActivityOptions<TAdapter, TStream> {
+  return options
+}
+
+// Re-export adapter types
+export type {
+  LiveVideoAdapter,
+  LiveVideoAdapterConfig,
+  AnyLiveVideoAdapter,
+} from './adapter'
+export { BaseLiveVideoAdapter } from './adapter'

package/src/activities/generateVideo/index.ts

@@ -1,7 +1,9 @@
 /**
  * Video Activity (Experimental)
  *
- * Generates videos from text prompts using a jobs/polling architecture.
+ * Generates videos from text prompts. Adapters use a jobs/polling
+ * architecture: create a job, poll for status, then fetch a download URL.
+ * For a live, prompt-steerable stream, use generateLiveVideo().
  * This is a self-contained module with implementation, types, and JSDoc.
  *
  * @experimental Video generation is an experimental feature and may change.
@@ -537,9 +539,6 @@ async function runCreateVideoJob<
 
   const mwCtx = contextFor(videoRunIdForJob(adapter.name, jobResult.jobId))
   await runGenerationStart(middleware, mwCtx)
-  // Transforms see the submission result (no url yet, so nothing to copy into a
-  // blob store) purely so the run record captures the jobId and any prompt
-  // inputs. No finish hook: the run is still running.
   return await applyGenerationResultTransforms(mwCtx, jobResult)
 }