@tanstack/ai 0.54.0 → 0.55.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai",
-  "version": "0.54.0",
+  "version": "0.55.0",
   "description": "Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.",
   "author": "Tanner Linsley",
   "license": "MIT",
@@ -88,8 +88,8 @@
     "@ag-ui/core": "0.1.1-canary.beta.0",
     "@standard-schema/spec": "^1.1.0",
     "partial-json": "^0.1.7",
-    "@tanstack/ai-event-client": "^0.11.3",
-    "@tanstack/ai-utils": "^0.4.0"
+    "@tanstack/ai-utils": "^0.4.0",
+    "@tanstack/ai-event-client": "^0.11.3"
   },
   "peerDependencies": {
     "@opentelemetry/api": ">=1.9.0"

package/skills/ai-core/adapter-configuration/SKILL.md

@@ -129,7 +129,7 @@ const adapterWithKey = createOpenaiChat('gpt-5.2', 'sk-...')
 
 - `bedrockText(model)` or `bedrockText(model, { api: 'converse' })` (the default) — Bedrock's native Converse API via `@aws-sdk/client-bedrock-runtime` (adapter name `bedrock-converse`). Reaches the broad catalog: Claude, Nova, Llama, Mistral, DeepSeek, and more.
 - `bedrockText(model, { api: 'chat' })` — OpenAI-compatible Chat Completions endpoint (adapter name `bedrock`). Open-weight models only (gpt-oss, DeepSeek V3.x, Gemma, Qwen, etc.). Does NOT reach Claude, Nova, or Llama.
-- `bedrockText(model, { api: 'responses' })` — OpenAI-compatible Responses API, mantle-only (adapter name `bedrock-responses`). Currently gpt-oss family.
+- `bedrockText(model, { api: 'responses' })` — OpenAI-compatible Responses API, mantle-only (adapter name `bedrock-responses`). Currently gpt-oss and Gemma 4.
 
 Use `createBedrockText(model, apiKey, config?)` to pass the key explicitly. Auth resolves from `BEDROCK_API_KEY` / `AWS_BEARER_TOKEN_BEDROCK`, or SigV4 via the standard AWS credential chain (no extra packages needed — handled by `@aws-sdk/client-bedrock-runtime`).
 

package/skills/ai-core/media-generation/SKILL.md

@@ -5,7 +5,7 @@ description: >
   activity-specific adapters: generateImage() with openaiImage/geminiImage/byteplusImage,
   generateAudio() with geminiAudio/falAudio, generateVideo() with async
   polling (openaiVideo/geminiVideo/grokVideo/falVideo/byteplusVideo/openRouterVideo,
-  per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech,
+  per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech/elevenlabsSpeech,
   generateTranscription() with openaiTranscription/byteplusTranscription. React hooks:
   useGenerateImage, useGenerateAudio,
   useGenerateSpeech, useTranscription, useGenerateVideo.
@@ -20,6 +20,7 @@ sources:
   - 'TanStack/ai:docs/media/audio-generation.md'
   - 'TanStack/ai:docs/media/video-generation.md'
   - 'TanStack/ai:docs/media/text-to-speech.md'
+  - 'TanStack/ai:docs/adapters/elevenlabs.md'
   - 'TanStack/ai:docs/media/transcription.md'
   - 'TanStack/ai:docs/advanced/debug-logging.md'
 ---
@@ -303,14 +304,14 @@ with `allowUrlFetch: true` on the adapter config
 
 **Provider support matrix:**
 
-| Provider   | `generateImage` image parts                                                                                                                                                                              | `generateVideo` image parts                                                                                                                                                                                                                                                                                  |
-| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| OpenAI     | gpt-image-2 / gpt-image-1 / -mini → `images.edit()` (up to 16). dall-e-2 → edit (1). dall-e-3 throws.                                                                                                    | Sora-2 / -pro → `input_reference` (single). Throws if >1.                                                                                                                                                                                                                                                    |
-| Gemini     | Native (gemini-\*-flash-image, "nano-banana") → multimodal `contents`. Imagen throws.                                                                                                                    | Veo → first un-roled / `'start_frame'` image is the input image; `'end_frame'` → `lastFrame`; `'reference'` / `'character'` → `referenceImages`. Omni Flash sends image/video parts as interaction content blocks (no role routing).                                                                         |
-| fal        | Per-endpoint field names from a generated map (`pnpm generate:fal-image-fields`). Defaults: 1 input → `image_url`; >1 → `image_urls`; roles → `mask_url` / `control_image_url` / `reference_image_urls`. | Per-endpoint map (e.g. Kling i2v start frame → `image_url`). Defaults: 1 input → `image_url`; `start_frame`/`end_frame` → `start_image_url`/`end_image_url`; `reference` → `reference_image_urls`.                                                                                                           |
-| Grok       | grok-imagine models → `/v1/images/edits` JSON endpoint (≤3 sources, addressed by xAI in request order; prompt sent verbatim; mask/control throw). grok-2-image-1212 throws.                              | Un-roled / `'start_frame'` image → starting frame; `'reference'` / `'character'` → `reference_images` (1.5). Starting frame and reference inputs cannot be combined. A `video` part + `modelOptions.mode: 'edit' \| 'extend'` routes to `/videos/edits` / `/videos/extensions` on `grok-imagine-video` only. |
-| OpenRouter | Prompt parts map 1:1 onto multimodal `text` / `image_url` content parts, preserving interleaved order.                                                                                                   | Dedicated async API (`openRouterVideo`): `start_frame`/`end_frame` → `frame_images[]` (`first_frame`/`last_frame`); `reference`/`character` → `input_references[]`; an unroled image defaults to the start frame. Frame roles validated against the model's `supported_frame_images` metadata.               |
-| Anthropic  | n/a (no image generation API).                                                                                                                                                                           | n/a                                                                                                                                                                                                                                                                                                          |
+| Provider   | `generateImage` image parts                                                                                                                                                                              | `generateVideo` image parts                                                                                                                                                                                                                                                                                                                   |
+| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| OpenAI     | gpt-image-2 / gpt-image-1 / -mini → `images.edit()` (up to 16). dall-e-2 → edit (1). dall-e-3 throws.                                                                                                    | Sora-2 / -pro → `input_reference` (single). Throws if >1.                                                                                                                                                                                                                                                                                     |
+| Gemini     | Native (gemini-\*-flash-image, "nano-banana") → multimodal `contents`. Imagen throws.                                                                                                                    | Veo → first un-roled / `'start_frame'` image is the input image; `'end_frame'` → `lastFrame`; `'reference'` / `'character'` → `referenceImages`. Omni Flash sends image/video parts as interaction content blocks (no role routing).                                                                                                          |
+| fal        | Per-endpoint field names from a generated map (`pnpm generate:fal-image-fields`). Defaults: 1 input → `image_url`; >1 → `image_urls`; roles → `mask_url` / `control_image_url` / `reference_image_urls`. | Per-endpoint map (e.g. Kling i2v start frame → `image_url`). Defaults: 1 input → `image_url`; `start_frame`/`end_frame` → `start_image_url`/`end_image_url`; `reference` → `reference_image_urls`.                                                                                                                                            |
+| Grok       | grok-imagine models → `/v1/images/edits` JSON endpoint (≤3 sources, addressed by xAI in request order; prompt sent verbatim; mask/control throw). grok-2-image-1212 throws.                              | Un-roled / `'start_frame'` image → starting frame; `'reference'` / `'character'` → `reference_images` (1.5). On 1.5 a starting frame can be combined with reference inputs (it pins the first frame). A `video` part + `modelOptions.mode: 'edit' \| 'extend'` routes to `/videos/edits` / `/videos/extensions` on `grok-imagine-video` only. |
+| OpenRouter | Prompt parts map 1:1 onto multimodal `text` / `image_url` content parts, preserving interleaved order.                                                                                                   | Dedicated async API (`openRouterVideo`): `start_frame`/`end_frame` → `frame_images[]` (`first_frame`/`last_frame`); `reference`/`character` → `input_references[]`; an unroled image defaults to the start frame. Frame roles validated against the model's `supported_frame_images` metadata.                                                |
+| Anthropic  | n/a (no image generation API).                                                                                                                                                                           | n/a                                                                                                                                                                                                                                                                                                                                           |
 
 Video and audio prompt parts follow the same `metadata.role` convention
 for video-to-video and lipsync flows on fal. Grok accepts one source
@@ -352,8 +353,14 @@ const { generate, result, isLoading } = useGenerateAudio({
 
 ### 3. Text-to-Speech
 
-Adapters: `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-audio-preview) and
-`byteplusSpeech` (`seed-audio-1.0`).
+Adapters include `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-audio-preview),
+`byteplusSpeech` (`seed-audio-1.0`), and `elevenlabsSpeech` (`eleven_v3`).
+
+`elevenlabsSpeech` accepts `format: 'mp3' | 'pcm' | 'opus' | 'wav'`.
+WAV output contains 44.1 kHz, 16-bit mono PCM with a RIFF header.
+AAC and FLAC requests throw before the API call.
+An explicit `modelOptions.outputFormat` overrides `format` and returns
+the selected provider format without WAV wrapping.
 
 > **BytePlus Seed Speech is a separate product from ModelArk** — it reads
 > **`BYTEPLUS_VOICE_API_KEY`**, not `ARK_API_KEY`, and an Ark key there fails
@@ -364,7 +371,10 @@ Adapters: `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-audio-preview) and
 > drops `voice`. Voice ids ending `_uranus_bigtts` are TTS 2.0,
 > `_mars_bigtts` / `_moon_bigtts` are TTS 1.0, and `*_emo_v2_*` are the 1.0
 > voices that accept emotion tags. Formats: `wav`, `mp3`, `pcm`, `ogg_opus`;
-> `watermark` is also available on `modelOptions`.
+> `modelOptions.watermark` takes an object here, not a boolean:
+> `{ aigc_watermark }` for an audible marker and `{ aigc_metadata: { enable } }`
+> for header provenance. `watermark: true` is shorthand for
+> `{ aigc_watermark: true }`.
 
 ```typescript
 import { generateSpeech } from '@tanstack/ai'

package/src/activities/chat/index.ts

@@ -63,10 +63,12 @@ import {
 import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies'
 import { isCancelRequestedReason } from './cancel'
 import {
+  appendUiResourceToModelMessages,
   convertMessagesToModelMessages,
   generateMessageId,
   modelMessagesToUIMessages,
   safeJsonStringify,
+  uiResourcePartFromCustomValue,
 } from './messages'
 import { MiddlewareRunner } from './middleware/compose'
 import { getRunDetached } from './middleware/run-store'
@@ -2770,6 +2772,22 @@ class TextEngine<
     }
   }
 
+  /**
+   * Record a `ui-resource` CUSTOM chunk on the assistant ModelMessage owning
+   * its `toolCallId` so the resource survives later MESSAGES_SNAPSHOT chunks
+   * (e.g. the interrupt snapshot emitted when the run pauses on a client
+   * tool). Mirrors the anchor-preserving approach used for
+   * `toolCallMetadata` (#867). See #1397.
+   */
+  private recordEmittedUiResource(value: unknown): void {
+    const part = uiResourcePartFromCustomValue(value)
+    if (!part) return
+    const next = appendUiResourceToModelMessages(this.messages, part)
+    if (next === this.messages) return
+    this.messages = next
+    this.middlewareCtx.messages = this.messages
+  }
+
   private buildMessagesSnapshotChunk(): StreamChunk {
     const withIds = this.messages.map((message, index) => ({
       ...message,
@@ -4431,6 +4449,11 @@ class TextEngine<
           if (this.hasPublicRunStarted) continue
           this.hasPublicRunStarted = true
         }
+        // Persist MCP Apps ui-resource emissions onto the tool-call anchor
+        // message so interrupt MESSAGES_SNAPSHOT chunks keep them (#1397).
+        if (spec.type === EventType.CUSTOM && spec.name === 'ui-resource') {
+          this.recordEmittedUiResource((spec as CustomEvent).value)
+        }
         yield spec
         this.middlewareCtx.chunkIndex++
       }

package/src/activities/chat/messages.ts

@@ -387,6 +387,66 @@ function appendUiResources(
   return { ...ui, parts: [...ui.parts, ...extra] }
 }
 
+/**
+ * Build a UIResourcePart from the value of a CUSTOM `ui-resource` chunk
+ * emitted via `ctx.emitCustomEvent('ui-resource', ...)` (MCP Apps). The
+ * emission-side value carries `resource`/`serverId`/`toolName` plus the
+ * `toolCallId` stamped by the tool-call context wrapper — the `type`
+ * discriminator is added here. Returns undefined when the value does not
+ * match the ui-resource shape.
+ */
+export function uiResourcePartFromCustomValue(
+  value: unknown,
+): UIResourcePart | undefined {
+  if (!isRecord(value)) return undefined
+  const part: unknown = { type: 'ui-resource', ...value }
+  return isUiResourcePart(part) ? part : undefined
+}
+
+/**
+ * Store an emitted ui-resource part on the assistant ModelMessage that owns
+ * its `toolCallId` (the tool-call anchor), so it survives later
+ * MESSAGES_SNAPSHOT chunks — e.g. the interrupt snapshot emitted when the
+ * run pauses on a client tool (#1397). Mirrors how `toolCallMetadata` is
+ * preserved on the anchor (#867).
+ *
+ * Returns the SAME array reference when no anchor owns the tool call or the
+ * resource is already stored (idempotent).
+ */
+export function appendUiResourceToModelMessages(
+  messages: Array<ModelMessage>,
+  part: UIResourcePart,
+): Array<ModelMessage> {
+  for (let index = messages.length - 1; index >= 0; index--) {
+    const message = messages[index]
+    if (!message || message.role !== 'assistant') continue
+    const ownsToolCall = message.toolCalls?.some(
+      (toolCall) => toolCall.id === part.toolCallId,
+    )
+    if (!ownsToolCall) continue
+    const previous = tanstackMetadata(message)?.uiResources ?? []
+    if (
+      previous.some((stored) => uiResourceKey(stored) === uiResourceKey(part))
+    ) {
+      return messages
+    }
+    const nextMessage = {
+      ...message,
+      metadata: {
+        ...message.metadata,
+        tanstack: {
+          ...tanstackMetadata(message),
+          uiResources: [...previous, part],
+        },
+      },
+    }
+    const next = messages.slice()
+    next[index] = nextMessage
+    return next
+  }
+  return messages
+}
+
 function assistantMetadata(
   uiMessage: UIMessage,
 ): UIMessage['metadata'] | undefined {

package/src/activities/chat/stream/processor.ts

@@ -242,6 +242,37 @@ export class StreamProcessor {
     this.emitMessagesChange()
   }
 
+  /**
+   * Put older UI messages at the front of the conversation.
+   *
+   * Skip a message if its id is already in the list. Keep the existing message.
+   * Then emit the same messages-change event as `setMessages`.
+   *
+   * Use this for older history pages. The first hydrate window uses `setMessages`.
+   *
+   * @param messages Older UI messages in insertion order. The first item is the oldest.
+   *
+   * @example
+   * ```ts
+   * processor.setMessages([newest])
+   * processor.prependMessages([oldest])
+   * ```
+   */
+  prependMessages(messages: Array<UIMessage>) {
+    const existingIds = new Set(this.messages.map((message) => message.id))
+    const olderMessages: Array<UIMessage> = []
+    for (const message of messages) {
+      const isDuplicate = existingIds.has(message.id)
+      if (isDuplicate) {
+        continue
+      }
+      existingIds.add(message.id)
+      olderMessages.push(message)
+    }
+    this.messages = [...olderMessages, ...this.messages]
+    this.emitMessagesChange()
+  }
+
   /**
    * Add a user message to the conversation.
    * Supports both simple string content and multimodal content arrays.