@tanstack/ai 0.61.0 → 0.64.0

package/src/activities/generateVideo/snap.ts

@@ -1,8 +1,38 @@
 import type { DurationOptions } from './adapter'
 
+/**
+ * `"6"`, `"6s"`, and `"6.5s"` are seconds. Anything else (`"auto"`) is a
+ * keyword the model must list exactly.
+ */
+const DURATION_TEMPLATE = /^(\d+(?:\.\d+)?)s?$/
+
+/**
+ * Seconds from a caller-supplied template, or `null` when `input` is a
+ * keyword (`"auto"`) rather than a length.
+ */
+function templateToSeconds(input: string): number | null {
+  const match = DURATION_TEMPLATE.exec(input)
+  const digits = match?.[1]
+  if (digits === undefined) return null
+  const seconds = Number(digits)
+  return Number.isFinite(seconds) ? seconds : null
+}
+
+/**
+ * Seconds from a duration a caller wrote: `6`, `"6"`, or `"6s"`.
+ * Returns `undefined` for keywords such as `"auto"` and for non-finite numbers.
+ */
+export function durationToSeconds(input: number | string): number | undefined {
+  if (typeof input === 'number') {
+    return Number.isFinite(input) ? input : undefined
+  }
+  const seconds = templateToSeconds(input)
+  return seconds === null ? undefined : seconds
+}
+
 /**
  * Extract a numeric seconds value from a `DurationOptions` entry. Returns
- * `null` for entries that don't parse as a number — e.g. `'auto'`.
+ * `null` for entries that don't parse as a number, for example `'auto'`.
  *
  * Handles the keyword-with-unit form FAL uses for Luma/Veo (`'8s'`, `'9s'`)
  * by stripping a trailing `s`. Pure-numeric strings (`'5'`, `'10'`) parse via
@@ -12,14 +42,16 @@ function entryToSeconds(entry: string | number): number | null {
   if (typeof entry === 'number') {
     return Number.isFinite(entry) ? entry : null
   }
-  const stripped = entry.endsWith('s') ? entry.slice(0, -1) : entry
-  const parsed = Number(stripped)
-  return Number.isFinite(parsed) ? parsed : null
+  return templateToSeconds(entry)
 }
 
 /**
- * Snap a raw seconds value to the closest valid duration for a model's
- * `DurationOptions`.
+ * Snap a caller duration to the closest valid option.
+ *
+ * `input` may be seconds (`7`), a numeric string (`"7"`), a template
+ * (`"6s"`), or a keyword the model lists (`"auto"`). A keyword that is not
+ * in the set returns `undefined`. Equal numeric distances keep the earlier
+ * option.
  *
  * - `none`            → `undefined`
  * - `discrete`        → closest numeric-parseable entry; if none parse,
@@ -30,6 +62,32 @@ function entryToSeconds(entry: string | number): number | null {
  * @experimental Video generation is an experimental feature and may change.
  */
 export function snapToDurationOption<T extends string | number | undefined>(
+  input: number | string,
+  options: DurationOptions<T>,
+): T | undefined {
+  // NaN is not a length. Infinity still clamps inside snapSeconds.
+  if (typeof input === 'number' && Number.isNaN(input)) return undefined
+
+  if (typeof input === 'string') {
+    const seconds = templateToSeconds(input)
+    if (seconds === null) return matchKeyword(input, options)
+    return snapSeconds(seconds, options)
+  }
+  return snapSeconds(input, options)
+}
+
+function matchKeyword<T extends string | number | undefined>(
+  keyword: string,
+  options: DurationOptions<T>,
+): T | undefined {
+  if (options.kind !== 'discrete' && options.kind !== 'mixed') return undefined
+  for (const value of options.values) {
+    if (value === keyword) return value
+  }
+  return undefined
+}
+
+function snapSeconds<T extends string | number | undefined>(
   seconds: number,
   options: DurationOptions<T>,
 ): T | undefined {

package/src/activities/generateVoice/index.ts

@@ -166,7 +166,7 @@ function createId(prefix: string): string {
  * if (!preview) throw new Error('No voice candidates returned')
  *
  * const speech = await generateSpeech({
- *   adapter: elevenlabsSpeech('eleven_v3'),
+ *   adapter: elevenlabsSpeech('eleven_v4'),
  *   text: 'Once upon a time...',
  *   voice: preview.voiceId,
  * })

package/src/activities/index.ts

@@ -209,9 +209,10 @@ export {
   type VideoAdapterConfig,
   type AnyVideoAdapter,
   type DurationOptions,
+  type VideoDurationSpell,
 } from './generateVideo/adapter'
 
-export { snapToDurationOption } from './generateVideo/snap'
+export { durationToSeconds, snapToDurationOption } from './generateVideo/snap'
 
 // ===========================
 // TTS Activity

package/src/adapter-internals.ts

@@ -59,3 +59,4 @@ export {
 } from './utilities/structured-output-events'
 export { tanstackMetadata } from './utilities/merge-metadata'
 export { isSpecTopLevelKey } from './utilities/spec-event-keys'
+export { REDACTED_THINKING_ID_PREFIX } from './utilities/reasoning-encrypted-value'

package/src/client.ts

@@ -371,6 +371,7 @@ export type {
   ThinkingPart,
   ToolCall,
   ToolCallPart,
+  ToolResultOutcome,
   ToolResultPart,
   UIMessage,
   UIResourcePart,

package/src/index.ts

@@ -535,6 +535,7 @@ export type { WireMessage } from './utilities/ag-ui-wire'
 export {
   isContentPart,
   isContentPartArray,
+  isToolResultOutcome,
   normalizeToolResult,
 } from './utilities/tool-result'
 export {

package/src/interrupt-resume.ts

@@ -14,6 +14,7 @@ import {
   isStandardSchema,
   validateWithStandardSchema,
 } from './activities/chat/tools/schema-converter'
+import { tanstackMetadata } from './utilities/merge-metadata'
 import type {
   InterruptBinding,
   InterruptSubmissionError,
@@ -95,6 +96,33 @@ function stringField(
   return typeof value[key] === 'string' ? value[key] : undefined
 }
 
+type ClientToolResumeResult =
+  | { state: 'output-available'; output: unknown }
+  | { state: 'output-error'; errorText: string }
+
+function clientToolResult(
+  entry: RunAgentResumeItem,
+): ClientToolResumeResult | null {
+  if (tanstackMetadata(entry)?.state === 'output-error') {
+    const result = objectValue(entry.payload)
+    if (
+      !result ||
+      Object.keys(result).length !== 1 ||
+      typeof result.error !== 'string'
+    ) {
+      return null
+    }
+    return {
+      state: 'output-error',
+      errorText: result.error,
+    }
+  }
+  return {
+    state: 'output-available',
+    output: entry.payload,
+  }
+}
+
 function normalizeIssuePath(
   path: ReadonlyArray<unknown> | undefined,
 ): ReadonlyArray<string | number> | undefined {
@@ -528,7 +556,19 @@ export async function validateInterruptResumeBatch(
     if (schemaDrifted) continue
 
     if (binding.kind === 'client-tool-execution') {
-      if (responseSchema !== undefined) {
+      const result = clientToolResult(entry)
+      if (!result) {
+        errors.push(
+          interruptItemError(
+            input,
+            record.interruptId,
+            'invalid-tool-output',
+            `Tool ${binding.toolName} result is invalid.`,
+          ),
+        )
+        continue
+      }
+      if (result.state === 'output-available' && responseSchema !== undefined) {
         await pushSchemaIssues({
           request: input,
           errors,
@@ -539,13 +579,16 @@ export async function validateInterruptResumeBatch(
           label: `Tool ${binding.toolName} output is invalid`,
         })
       }
-      if (tool.outputSchema !== undefined) {
+      if (
+        result.state === 'output-available' &&
+        tool.outputSchema !== undefined
+      ) {
         await pushSchemaIssues({
           request: input,
           errors,
           interruptId: record.interruptId,
           schema: tool.outputSchema,
-          value: entry.payload,
+          value: result.output,
           code: 'invalid-tool-output',
           label: `Tool ${binding.toolName} output is invalid`,
         })
@@ -682,6 +725,7 @@ export async function validateInterruptResumeBatch(
   const canonical = canonicalizeInterruptResolutions(input.resume ?? [])
   const approvals = new Map<string, ToolApprovalResolution>()
   const clientToolResults = new Map<string, unknown>()
+  const clientToolErrors = new Map<string, string>()
   const genericInterrupts = new Map<
     string,
     | { interruptId: string; status: 'resolved'; payload: unknown }
@@ -738,7 +782,13 @@ export async function validateInterruptResumeBatch(
       continue
     }
     if (binding.kind === 'client-tool-execution') {
-      clientToolResults.set(binding.toolCallId, entry.payload)
+      const result = clientToolResult(entry)
+      if (!result) continue
+      if (result.state === 'output-error') {
+        clientToolErrors.set(binding.toolCallId, result.errorText)
+      } else {
+        clientToolResults.set(binding.toolCallId, result.output)
+      }
       continue
     }
     const envelope = objectValue(entry.payload)
@@ -781,6 +831,7 @@ export async function validateInterruptResumeBatch(
     resumeToolState: {
       approvals,
       clientToolResults,
+      clientToolErrors,
       genericInterrupts,
       deniedToolResults,
       cancelledToolCallIds,

package/src/middlewares/otel.ts

@@ -108,7 +108,9 @@ export interface OtelMiddlewareOptions {
   meter?: Meter
   /**
    * When `true`, prompt and completion content is attached to iteration spans
-   * as `gen_ai.*.message` / `gen_ai.choice` events. Defaults to `false` so
+   * as `gen_ai.*.message` / `gen_ai.choice` events. Media spans get the
+   * prompt, input media and output (URLs or transcript text) as
+   * `gen_ai.input.messages` / `gen_ai.output.messages`. Defaults to `false` so
    * that PII never lands on a span by accident.
    */
   captureContent?: boolean
@@ -121,7 +123,8 @@ export interface OtelMiddlewareOptions {
   redact?: (text: string) => string
   /**
    * Maximum characters kept in the per-iteration assistant text buffer used
-   * to emit `gen_ai.choice` events. Extra characters are truncated with a
+   * to emit `gen_ai.choice` events, and in each text part of a media span's
+   * input and output. Extra characters are truncated with a
    * trailing `"…"` marker. Defaults to 100 000. Set to `0` to disable the
    * cap. Exporters typically truncate long attribute values anyway.
    */
@@ -221,6 +224,111 @@ function serializeContent(content: unknown): string {
   return parts.join(' ')
 }
 
+type InputPart =
+  | { type: 'text'; content: string }
+  | { type: 'uri'; modality: string; uri: string; mime_type?: string }
+  | { type: 'file'; modality: string; file_id: string; mime_type?: string }
+
+/**
+ * Structured form of `ContentPart[]` for `gen_ai.input.messages`, using the
+ * OTel GenAI semconv part shapes. URL and file-handle media keep their
+ * reference; inline bytes (and `data:` URLs) stay a `[type]` placeholder so
+ * they never blow attribute size limits. `redact` runs on text parts only.
+ */
+function serializeParts(
+  content: Array<unknown>,
+  redact: (text: string) => string,
+): Array<InputPart> {
+  const parts: Array<InputPart> = []
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue
+    const p = part as {
+      type?: string
+      text?: string
+      content?: string
+      source?: { type?: string; value?: string; mimeType?: string }
+    }
+    if (p.type === 'text') {
+      parts.push({
+        type: 'text',
+        content: redact((p.text ?? p.content ?? '').toString()),
+      })
+      continue
+    }
+    const modality = p.type ?? 'unknown'
+    const { type, value, mimeType } = p.source ?? {}
+    const mime = mimeType ? { mime_type: mimeType } : {}
+    if (type === 'url' && value && !value.startsWith('data:')) {
+      parts.push({ type: 'uri', modality, uri: value, ...mime })
+    } else if (type === 'file' && value) {
+      parts.push({ type: 'file', modality, file_id: value, ...mime })
+    } else {
+      parts.push({ type: 'text', content: `[${modality}]` })
+    }
+  }
+  return parts
+}
+
+/** A media reference as a content part, so `serializeParts` can map it. */
+function mediaPart(modality: string, url: unknown): unknown {
+  return {
+    type: modality,
+    source: typeof url === 'string' ? { type: 'url', value: url } : undefined,
+  }
+}
+
+/**
+ * Content parts for a media call's inputs, read from `artifactInputs`: the
+ * prompt (a string or `MediaPrompt` parts), TTS `text`, and transcription
+ * `audio`. Audio is a reference only when it is an http(s) URL; a base64
+ * string, `File` or `Blob` becomes a placeholder.
+ */
+function mediaInputParts(inputs: unknown): Array<unknown> {
+  if (!inputs || typeof inputs !== 'object') return []
+  const { prompt, text, audio } = inputs as Record<string, unknown>
+  const parts: Array<unknown> = []
+  for (const value of [prompt, text]) {
+    if (typeof value === 'string') parts.push({ type: 'text', content: value })
+    else if (Array.isArray(value)) parts.push(...value)
+  }
+  if (audio !== undefined) {
+    const url =
+      typeof audio === 'string' && /^https?:\/\//.test(audio) ? audio : null
+    parts.push(mediaPart('audio', url))
+  }
+  return parts
+}
+
+/**
+ * Content parts for a media call's result: generated image, audio or video
+ * URLs, and transcript text. Base64 output becomes a placeholder.
+ */
+function mediaOutputParts(
+  activity: GenerationActivity,
+  result: unknown,
+): Array<unknown> {
+  if (!result || typeof result !== 'object') return []
+  const r = result as Record<string, unknown>
+  const parts: Array<unknown> = []
+  if (Array.isArray(r.images)) {
+    for (const image of r.images) {
+      parts.push(mediaPart('image', (image as { url?: unknown } | null)?.url))
+    }
+  }
+  // `TTSResult.audio` is a base64 string; `AudioGenerationResult.audio` is a
+  // `{ url } | { b64Json }` source.
+  if (r.audio !== undefined) {
+    parts.push(mediaPart('audio', (r.audio as { url?: unknown } | null)?.url))
+  }
+  // Only video puts its asset on a top-level `url`. A world `url` is a viewer
+  // page, not media.
+  if (activity === 'video' && typeof r.url === 'string') {
+    parts.push(mediaPart('video', r.url))
+  }
+  if (typeof r.text === 'string') parts.push({ type: 'text', content: r.text })
+  return parts
+}
+
 function messageEventName(role: string): string {
   switch (role) {
     case 'user':
@@ -340,6 +448,29 @@ export function otelMiddleware(
     })
   }
 
+  // Media prompts and transcripts are single strings, so cap each text part
+  // with `maxContentLength` the same way the chat completion buffer is capped.
+  const redactMediaText = (text: string): string =>
+    redactContent(
+      maxContentLength > 0 && text.length > maxContentLength
+        ? text.slice(0, maxContentLength) + '…'
+        : text,
+    )
+
+  const setMediaMessages = (
+    span: Span,
+    direction: 'input' | 'output',
+    parts: Array<unknown>,
+  ): void => {
+    const content = serializeParts(parts, redactMediaText)
+    if (content.length === 0) return
+    const json = JSON.stringify([
+      { role: direction === 'input' ? 'user' : 'assistant', content },
+    ])
+    span.setAttribute(`gen_ai.${direction}.messages`, json)
+    span.setAttribute(`langfuse.observation.${direction}`, json)
+  }
+
   const startMediaSpan = (ctx: GenerationMiddlewareContext): void => {
     safeCall('otel.onStart', () => {
       const operationName = OPERATION_NAME[ctx.activity]
@@ -365,6 +496,22 @@ export function otelMiddleware(
       )
       if (enriched) span.setAttributes(enriched)
       mediaSpans.set(ctx, span)
+
+      if (captureContent) {
+        setMediaMessages(span, 'input', mediaInputParts(ctx.artifactInputs))
+        // Observe the result through a transform: the terminal hooks never
+        // see it. Returns `undefined`, so the result is left unchanged.
+        ctx.resultTransforms.push((result) => {
+          safeCall('otel.captureOutput', () =>
+            setMediaMessages(
+              span,
+              'output',
+              mediaOutputParts(ctx.activity, result),
+            ),
+          )
+          return undefined
+        })
+      }
     })
   }
 
@@ -581,7 +728,12 @@ export function otelMiddleware(
           // Also emit the current GenAI-semconv attribute form
           // (`gen_ai.input.messages`) — backends like PostHog read prompt
           // content from this attribute, not from span events.
-          const inputMessages: Array<{ role: string; content: string }> = []
+          // Multimodal messages keep their parts structured so image / audio /
+          // video / document references survive into the trace (#1525).
+          const inputMessages: Array<{
+            role: string
+            content: string | Array<InputPart>
+          }> = []
           for (const sys of systemPromptContents) {
             inputMessages.push({
               role: 'system',
@@ -589,6 +741,12 @@ export function otelMiddleware(
             })
           }
           for (const m of config.messages) {
+            if (Array.isArray(m.content)) {
+              const parts = serializeParts(m.content, redactContent)
+              if (parts.length === 0) continue
+              inputMessages.push({ role: m.role, content: parts })
+              continue
+            }
             const body = serializeContent(m.content)
             if (body.length === 0) continue
             inputMessages.push({

package/src/types.ts

@@ -100,6 +100,9 @@ export type ToolResultState =
   | 'complete' // Result is complete
   | 'error' // Error occurred
 
+/** Why a tool result ended without executing successfully. */
+export type ToolResultOutcome = 'cancelled' | 'denied'
+
 export type ToolOutputState = 'output-available' | 'output-error'
 
 /**
@@ -196,6 +199,13 @@ export interface ToolCall<TMetadata = unknown> extends Omit<
   metadata?: TMetadata
 }
 
+/** One source link from a provider-executed web search. */
+export interface ProviderExecutedToolSource {
+  url: string
+  title?: string
+  pageAge?: string
+}
+
 /**
  * Convention for tool-call `metadata` that marks a call as **provider-executed**
  * — run by the provider's own infrastructure (e.g. Anthropic `web_search` /
@@ -209,10 +219,12 @@ export interface ToolCall<TMetadata = unknown> extends Omit<
  *
  * Provider-specific payloads live under a namespaced key (e.g. `anthropic`),
  * keeping this convention opaque to the framework core. The index signature
- * preserves those per-adapter fields.
+ * preserves those per-adapter fields. `sources` is the normalized list of
+ * links a web search used, shared across providers.
  */
 export interface ProviderExecutedToolMetadata {
   providerExecuted?: boolean
+  sources?: Array<ProviderExecutedToolSource>
   [key: string]: unknown
 }
 
@@ -369,7 +381,12 @@ export interface ModelMessage<
   name?: string
   toolCalls?: Array<ToolCall>
   toolCallId?: string
-  thinking?: Array<{ content: string; signature?: string }>
+  /**
+   * Signed thinking to send back to the provider. `redacted: true` marks a
+   * block the provider encrypted: `content` is empty and `signature` holds its
+   * opaque data. See `ThinkingPart.signature` for the planned rename.
+   */
+  thinking?: Array<{ content: string; signature?: string; redacted?: boolean }>
   /** Error reported by an AG-UI tool message. */
   error?: string
   /** Optional AG-UI message metadata. TanStack-owned fields live under `tanstack`. */
@@ -442,6 +459,8 @@ export interface ToolResultPart {
   toolCallId: string
   content: string | Array<ContentPart>
   state: ToolResultState
+  /** Set when the user or middleware cancelled or denied the tool call; state remains `error`. */
+  outcome?: ToolResultOutcome
   error?: string // Error message if state is "error"
   metadata?: Record<string, unknown>
   createdAt?: Date
@@ -451,7 +470,20 @@ export interface ThinkingPart {
   type: 'thinking'
   content: string
   stepId?: string
+  /**
+   * The provider's opaque reasoning artefact, sent back unchanged: an
+   * Anthropic signature, Anthropic redacted data, or OpenAI encrypted content.
+   * TODO(#1581): rename to `encryptedValue` to match AG-UI's `ReasoningMessage`.
+   * Renaming breaks stored messages, so it needs a read shim for `signature`.
+   */
   signature?: string
+  /**
+   * The provider encrypted this thinking block (Anthropic `redacted_thinking`).
+   * `content` is empty, and `signature` holds the opaque data that goes back
+   * to the provider unchanged. On the AG-UI wire, the reasoning message id
+   * starts with `redacted_thinking-` instead.
+   */
+  redacted?: boolean
 }
 
 /**
@@ -560,6 +592,12 @@ export interface TanStackMessageMetadata {
   model?: string
   /** Parent chat run that produced this assistant message. */
   runId?: string
+  /**
+   * The chat run that produced this assistant message. `withPersistence` sets
+   * `id`. `reconstructChat` with `includeRuns: true` adds the finished run's
+   * timings, in epoch ms.
+   */
+  run?: { id: string; startedAt?: number; finishedAt?: number }
   /** Card data on a child wire message. See `uiMessagesToWire`. */
   subagent?: SubagentWireInfo
   /** Thinking signature for a `role: 'reasoning'` fan-out message. */
@@ -571,6 +609,8 @@ export interface TanStackMessageMetadata {
     createdAt?: string
     content?: Array<ContentPart>
   }
+  /** Outcome of a cancelled or denied tool result; when present, the UI state is `error`. */
+  toolResultOutcome?: ToolResultOutcome
   structuredOutput?: {
     status?: 'streaming' | 'complete' | 'error'
     partial?: unknown
@@ -679,6 +719,15 @@ export interface EmitCustomEventOptions {
   batch?: boolean
 }
 
+/**
+ * The user's answer to an `mcp_input` interrupt.
+ * `resolved` carries the `payload` from `resolveInterrupt`.
+ * `cancelled` means the user called `cancel()`.
+ */
+export type ToolInputResponse =
+  | { status: 'resolved'; payload: unknown }
+  | { status: 'cancelled' }
+
 /**
  * Context passed to tool execute functions, providing capabilities like
  * emitting custom events during execution.
@@ -693,6 +742,12 @@ export type ToolExecutionContext<TContext = unknown> =
      * e.g. MCP `callTool` — should forward this to cancel in-flight work.
      */
     abortSignal?: AbortSignal
+    /**
+     * The answer to the input request that this tool call raised in the
+     * previous run. It is set only when the run resumes an `mcp_input`
+     * interrupt for this tool call.
+     */
+    inputResponse?: ToolInputResponse
     /**
      * Emit a custom event during tool execution.
      * Events are streamed to the client in real-time as AG-UI CUSTOM events.
@@ -1051,6 +1106,11 @@ export interface TextOptions<
    */
   systemPrompts?: Array<SystemPrompt>
   agentLoopStrategy?: AgentLoopStrategy
+  /**
+   * How the server tools of one model turn run. `'parallel'` (the default)
+   * starts them together, and `'sequential'` runs them one at a time.
+   */
+  toolExecution?: 'parallel' | 'sequential'
   /**
    * Optional configuration for lazy-tool discovery (tools marked `lazy: true`).
    * Tunes how much of each lazy tool's description appears in the discovery
@@ -2240,9 +2300,10 @@ export interface VideoGenerationOptions<
   /** Video size — format depends on the provider (e.g., "16:9", "1280x720") */
   size?: TSize
   /**
-   * Video duration in seconds. Adapters that declare a per-model duration
-   * map narrow this to the model's valid union; use
-   * `adapter.snapDuration(seconds)` to coerce raw seconds to a valid value.
+   * Video duration. Adapters that declare a per-model duration map narrow
+   * this to that model's union (a number, `"8"`, or `"8s"`). Use
+   * `adapter.snapDuration(input)` to coerce a raw value. `input` may be
+   * seconds, a `"6s"` template, or `"auto"` when the model lists it.
    */
   duration?: TDuration
   /** Model-specific options for video generation */