@tanstack/ai 0.59.0 → 0.63.0

package/src/activities/generateVideo/adapter.ts

@@ -74,6 +74,13 @@ export interface VideoAdapter<
   readonly kind: 'video'
   /** Adapter name identifier */
   readonly name: string
+  /**
+   * Declares that this adapter can consume `{ type: 'file' }` content
+   * sources (provider Files API references). The activity dispatcher rejects
+   * file sources in preflight for adapters that don't declare this, so
+   * adapters written before the file arm existed fail closed.
+   */
+  readonly supportsFileSources?: boolean
   /** The model this adapter is configured for */
   readonly model: TModel
 
@@ -161,6 +168,7 @@ export abstract class BaseVideoAdapter<
 > {
   readonly kind = 'video' as const
   abstract readonly name: string
+  readonly supportsFileSources: boolean = false
   readonly model: TModel
 
   // Type-only property - never assigned at runtime

package/src/activities/generateVideo/index.ts

@@ -12,6 +12,7 @@
 import { aiEventClient } from '@tanstack/ai-event-client'
 import { toRunErrorPayload } from '../error-payload'
 import { resolveDebugOption } from '../../logger/resolve'
+import { assertPromptFileSourceSupport } from '../../utilities/content-source'
 import {
   applyGenerationResultTransforms,
   createGenerationContext,
@@ -452,6 +453,9 @@ async function runCreateVideoJob<
     timeout,
     abortSignal: callerAbortSignal,
   } = options
+  // Fail closed on `{ type: 'file' }` sources for adapters that haven't
+  // declared support (see assertPromptFileSourceSupport).
+  assertPromptFileSourceSupport(adapter, prompt)
   const model = adapter.model
   const requestId = createId('video')
   const startTime = Date.now()
@@ -580,6 +584,9 @@ async function* runStreamingVideoGeneration<
     timeout,
     abortSignal: callerAbortSignal,
   } = options
+  // Fail closed on `{ type: 'file' }` sources for adapters that haven't
+  // declared support (see assertPromptFileSourceSupport).
+  assertPromptFileSourceSupport(adapter, prompt)
   const model = adapter.model
   const runId = options.runId ?? createId('run')
   const requestId = createId('video')

package/src/activities/generateWorld/adapter.ts

@@ -44,10 +44,12 @@ export interface WorldAdapter<
   }
 
   /**
-   * Open a world session from a prompt.
+   * Create a world from a prompt.
    *
-   * Server adapters typically mint a short-lived token and return it with the
+   * Live session adapters mint a short-lived token and return it with the
    * prompt so a browser can connect, set the prompt, and start streaming.
+   * Job adapters start generation and return a world URL (or an operation
+   * id while the job is still running).
    */
   createWorld: (
     options: WorldGenerationOptions<TProviderOptions>,

package/src/activities/generateWorld/index.ts

@@ -1,9 +1,9 @@
 /**
  * World Activity (Experimental)
  *
- * Mints a session token for a live, prompt-steerable world. Unlike
- * generateVideo (a job that finishes with a URL), the browser then connects
- * with the token, sets the prompt, and streams until pause/reset/close.
+ * Live adapters mint a session token for a prompt-steerable world. Job
+ * adapters start generation and return a viewer URL, or an operation id
+ * while the job is still running.
  *
  * @experimental World generation is an experimental feature and may change.
  */
@@ -76,7 +76,7 @@ export interface WorldActivityOptions<
   /** Provider-specific options for world generation */
   modelOptions?: WorldProviderOptions<TAdapter>
   /**
-   * Whether to wrap the token result as StreamChunks for SSE transport.
+   * Whether to wrap the result as StreamChunks for SSE transport.
    * This is not the live video. When false or omitted, returns
    * Promise<WorldGenerationResult>.
    *
@@ -100,7 +100,7 @@ export interface WorldActivityOptions<
   /** Stable run id for correlating this run when persisted. */
   runId?: string
   /**
-   * Maximum duration of the token mint in milliseconds.
+   * Maximum wait for the mint or job poll, in milliseconds.
    * No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
    */
   timeout?: number
@@ -135,7 +135,8 @@ function createId(prefix: string): string {
 // ===========================
 
 /**
- * World generation activity - opens a live, prompt-steerable world session.
+ * World generation activity. Live adapters mint a session token. Job
+ * adapters return a viewer URL or an in-progress operation id.
  *
  * @example Mint a session token on the server
  * ```ts

package/src/activities/index.ts

@@ -27,6 +27,7 @@ import type { AnyRerankAdapter } from './rerank/adapter'
 import type { AnyEvaluateAdapter } from './evaluate/adapter'
 import type { AnyWorldAdapter } from './generateWorld/adapter'
 import type { AnyLiveVideoAdapter } from './generateLiveVideo/adapter'
+import type { AnyFilesAdapter } from './files/adapter'
 
 // ===========================
 // Chat Activity
@@ -330,11 +331,32 @@ export {
   type AnyLiveVideoAdapter,
 } from './generateLiveVideo/adapter'
 
+// ===========================
+// Files Activity
+// ===========================
+
+export {
+  kind as filesKind,
+  uploadFile,
+  getFile,
+  deleteFile,
+  fileSourceFromHandle,
+} from './files/index'
+
+export {
+  BaseFilesAdapter,
+  normalizeFileUploadInput,
+  type FilesAdapter,
+  type AnyFilesAdapter,
+  type FileHandle,
+  type FileUploadInput,
+} from './files/adapter'
+
 // ===========================
 // Adapter Union Types
 // ===========================
 
-/** Union of all adapter types that can be passed to chat() */
+/** Union of all adapter types across every activity kind */
 export type AIAdapter =
   | AnyTextAdapter
   | AnySummarizeAdapter
@@ -349,6 +371,7 @@ export type AIAdapter =
   | AnyEvaluateAdapter
   | AnyWorldAdapter
   | AnyLiveVideoAdapter
+  | AnyFilesAdapter
 
 /** Union type of all adapter kinds */
 export type AdapterKind =
@@ -365,3 +388,4 @@ export type AdapterKind =
   | 'evaluate'
   | 'world'
   | 'liveVideo'
+  | 'files'

package/src/activities/summarize/chat-stream-summarize.ts

@@ -65,6 +65,8 @@ function throwRunError(
  * `SummarizationOptions<TProviderOptions>` on the wrapper itself.
  */
 export interface ChatStreamCapable {
+  /** Native token-limit option for adapters whose provider name is user-defined. */
+  readonly maxTokensKey?: string
   chatStream: (options: TextOptions<any>) => AsyncIterable<AdapterYieldChunk>
 }
 
@@ -151,8 +153,8 @@ function applyDefaultTemperature(
 
 /**
  * Resolve `maxLength` to the provider-native max-output-tokens key for the
- * given summarize-adapter `name` (this wrapper's OWN `name`, not the wrapped
- * text adapter's) and merge it into a working copy of the caller's
+ * wrapped text adapter's explicit key, falling back to this wrapper's `name`,
+ * and merge it into a working copy of the caller's
  * `modelOptions`. The caller always wins: if they already set any recognised
  * token-limit key (flat or, for Ollama, nested `options.num_predict`), the
  * default is left untouched. Unknown/unrecognised adapter names fall back to
@@ -172,10 +174,11 @@ function applyMaxLength(
   adapterName: string,
   maxLength: number,
   modelOptions: Record<string, unknown>,
+  maxTokensKey?: string,
 ): Record<string, unknown> {
   const merged: Record<string, unknown> = { ...modelOptions }
 
-  if (adapterName === 'ollama') {
+  if (adapterName === 'ollama' && maxTokensKey === undefined) {
     // Honor a caller-set limit in either shape: a recognised flat key (e.g.
     // left over from a migration) or the nested `options.num_predict`.
     const callerSetFlatLimit = KNOWN_MAX_TOKENS_KEYS.some(
@@ -195,12 +198,12 @@ function applyMaxLength(
     return merged
   }
 
-  const key = MAX_TOKENS_KEY_BY_ADAPTER[adapterName]
+  const key = maxTokensKey ?? MAX_TOKENS_KEY_BY_ADAPTER[adapterName]
   if (key === undefined) return merged
 
-  const callerSetLimit = KNOWN_MAX_TOKENS_KEYS.some(
-    (k) => typeof merged[k] === 'number',
-  )
+  const callerSetLimit =
+    typeof merged[key] === 'number' ||
+    KNOWN_MAX_TOKENS_KEYS.some((k) => typeof merged[k] === 'number')
   if (callerSetLimit) return merged
 
   merged[key] = maxLength
@@ -372,17 +375,24 @@ export class ChatStreamSummarizeAdapter<
     working = applyDefaultTemperature(this.name, 0.3, working)
     // `maxLength` must reach the wire under the provider-native token key (it
     // differs per provider, and no adapter reads a generic `maxTokens`).
-    // Resolve it from this summarize adapter's `name` (the constructor arg,
-    // not the wrapped text adapter's name), never overriding a caller-supplied
-    // token limit.
+    // Prefer the wrapped adapter's explicit key, then this wrapper's name,
+    // never overriding a caller-supplied token limit.
     if (options.maxLength !== undefined) {
-      if (!isKnownMaxTokensAdapter(this.name)) {
+      if (
+        this.textAdapter.maxTokensKey === undefined &&
+        !isKnownMaxTokensAdapter(this.name)
+      ) {
         options.logger.warn(
           `summarize: maxLength=${options.maxLength} could not be mapped to a provider token key for adapter name "${this.name}" — it was dropped from modelOptions (the prompt still asks the model to stay under it). Construct ChatStreamSummarizeAdapter with a recognised provider name to forward the cap.`,
           { provider: this.name },
         )
       }
-      working = applyMaxLength(this.name, options.maxLength, working)
+      working = applyMaxLength(
+        this.name,
+        options.maxLength,
+        working,
+        this.textAdapter.maxTokensKey,
+      )
     }
     const modelOptions = working as TProviderOptions
 

package/src/client.ts

@@ -329,6 +329,13 @@ export type { AdapterYieldChunk } from './utilities/adapter-yield-chunk'
 export { getChunkRunId, getChunkThreadId } from './utilities/chunk-ids'
 export type { WireMessage } from './utilities/ag-ui-wire'
 
+// A browser client that received an uploaded handle from its server can build
+// the `{ type: 'file' }` content source itself — `fileSourceFromHandle` is a
+// pure object builder, so exporting it here keeps the documented client flow
+// from pulling in the server entry.
+export { fileSourceFromHandle } from './activities/files/index'
+export type { FileHandle } from './activities/files/adapter'
+
 export type {
   AudioPart,
   ContentPart,
@@ -364,6 +371,7 @@ export type {
   ThinkingPart,
   ToolCall,
   ToolCallPart,
+  ToolResultOutcome,
   ToolResultPart,
   UIMessage,
   UIResourcePart,

package/src/index.ts

@@ -18,6 +18,10 @@ export {
   embed,
   generateWorld,
   generateLiveVideo,
+  uploadFile,
+  getFile,
+  deleteFile,
+  fileSourceFromHandle,
 } from './activities/index'
 
 // Create options functions - for pre-defining typed configurations
@@ -70,6 +74,10 @@ export type {
   WorldAdapter,
   AnyLiveVideoAdapter,
   LiveVideoAdapter,
+  FilesAdapter,
+  AnyFilesAdapter,
+  FileHandle,
+  FileUploadInput,
 } from './activities/index'
 
 // Rerank adapter base + types
@@ -527,8 +535,17 @@ export type { WireMessage } from './utilities/ag-ui-wire'
 export {
   isContentPart,
   isContentPartArray,
+  isToolResultOutcome,
   normalizeToolResult,
 } from './utilities/tool-result'
+export {
+  assertMessagesFileSourceSupport,
+  assertPromptFileSourceSupport,
+  fileReferenceFor,
+  isFileSource,
+  unsupportedFileSourceError,
+  type FileSourceCapable,
+} from './utilities/content-source'
 
 export {
   getProviderExecutedMetadata,

package/src/interrupt-resume.ts

@@ -14,6 +14,7 @@ import {
   isStandardSchema,
   validateWithStandardSchema,
 } from './activities/chat/tools/schema-converter'
+import { tanstackMetadata } from './utilities/merge-metadata'
 import type {
   InterruptBinding,
   InterruptSubmissionError,
@@ -95,6 +96,33 @@ function stringField(
   return typeof value[key] === 'string' ? value[key] : undefined
 }
 
+type ClientToolResumeResult =
+  | { state: 'output-available'; output: unknown }
+  | { state: 'output-error'; errorText: string }
+
+function clientToolResult(
+  entry: RunAgentResumeItem,
+): ClientToolResumeResult | null {
+  if (tanstackMetadata(entry)?.state === 'output-error') {
+    const result = objectValue(entry.payload)
+    if (
+      !result ||
+      Object.keys(result).length !== 1 ||
+      typeof result.error !== 'string'
+    ) {
+      return null
+    }
+    return {
+      state: 'output-error',
+      errorText: result.error,
+    }
+  }
+  return {
+    state: 'output-available',
+    output: entry.payload,
+  }
+}
+
 function normalizeIssuePath(
   path: ReadonlyArray<unknown> | undefined,
 ): ReadonlyArray<string | number> | undefined {
@@ -528,7 +556,19 @@ export async function validateInterruptResumeBatch(
     if (schemaDrifted) continue
 
     if (binding.kind === 'client-tool-execution') {
-      if (responseSchema !== undefined) {
+      const result = clientToolResult(entry)
+      if (!result) {
+        errors.push(
+          interruptItemError(
+            input,
+            record.interruptId,
+            'invalid-tool-output',
+            `Tool ${binding.toolName} result is invalid.`,
+          ),
+        )
+        continue
+      }
+      if (result.state === 'output-available' && responseSchema !== undefined) {
         await pushSchemaIssues({
           request: input,
           errors,
@@ -539,13 +579,16 @@ export async function validateInterruptResumeBatch(
           label: `Tool ${binding.toolName} output is invalid`,
         })
       }
-      if (tool.outputSchema !== undefined) {
+      if (
+        result.state === 'output-available' &&
+        tool.outputSchema !== undefined
+      ) {
         await pushSchemaIssues({
           request: input,
           errors,
           interruptId: record.interruptId,
           schema: tool.outputSchema,
-          value: entry.payload,
+          value: result.output,
           code: 'invalid-tool-output',
           label: `Tool ${binding.toolName} output is invalid`,
         })
@@ -682,6 +725,7 @@ export async function validateInterruptResumeBatch(
   const canonical = canonicalizeInterruptResolutions(input.resume ?? [])
   const approvals = new Map<string, ToolApprovalResolution>()
   const clientToolResults = new Map<string, unknown>()
+  const clientToolErrors = new Map<string, string>()
   const genericInterrupts = new Map<
     string,
     | { interruptId: string; status: 'resolved'; payload: unknown }
@@ -738,7 +782,13 @@ export async function validateInterruptResumeBatch(
       continue
     }
     if (binding.kind === 'client-tool-execution') {
-      clientToolResults.set(binding.toolCallId, entry.payload)
+      const result = clientToolResult(entry)
+      if (!result) continue
+      if (result.state === 'output-error') {
+        clientToolErrors.set(binding.toolCallId, result.errorText)
+      } else {
+        clientToolResults.set(binding.toolCallId, result.output)
+      }
       continue
     }
     const envelope = objectValue(entry.payload)
@@ -781,6 +831,7 @@ export async function validateInterruptResumeBatch(
     resumeToolState: {
       approvals,
       clientToolResults,
+      clientToolErrors,
       genericInterrupts,
       deniedToolResults,
       cancelledToolCallIds,

package/src/middlewares/otel.ts

@@ -108,7 +108,9 @@ export interface OtelMiddlewareOptions {
   meter?: Meter
   /**
    * When `true`, prompt and completion content is attached to iteration spans
-   * as `gen_ai.*.message` / `gen_ai.choice` events. Defaults to `false` so
+   * as `gen_ai.*.message` / `gen_ai.choice` events. Media spans get the
+   * prompt, input media and output (URLs or transcript text) as
+   * `gen_ai.input.messages` / `gen_ai.output.messages`. Defaults to `false` so
    * that PII never lands on a span by accident.
    */
   captureContent?: boolean
@@ -121,7 +123,8 @@ export interface OtelMiddlewareOptions {
   redact?: (text: string) => string
   /**
    * Maximum characters kept in the per-iteration assistant text buffer used
-   * to emit `gen_ai.choice` events. Extra characters are truncated with a
+   * to emit `gen_ai.choice` events, and in each text part of a media span's
+   * input and output. Extra characters are truncated with a
    * trailing `"…"` marker. Defaults to 100 000. Set to `0` to disable the
    * cap. Exporters typically truncate long attribute values anyway.
    */
@@ -221,6 +224,111 @@ function serializeContent(content: unknown): string {
   return parts.join(' ')
 }
 
+type InputPart =
+  | { type: 'text'; content: string }
+  | { type: 'uri'; modality: string; uri: string; mime_type?: string }
+  | { type: 'file'; modality: string; file_id: string; mime_type?: string }
+
+/**
+ * Structured form of `ContentPart[]` for `gen_ai.input.messages`, using the
+ * OTel GenAI semconv part shapes. URL and file-handle media keep their
+ * reference; inline bytes (and `data:` URLs) stay a `[type]` placeholder so
+ * they never blow attribute size limits. `redact` runs on text parts only.
+ */
+function serializeParts(
+  content: Array<unknown>,
+  redact: (text: string) => string,
+): Array<InputPart> {
+  const parts: Array<InputPart> = []
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue
+    const p = part as {
+      type?: string
+      text?: string
+      content?: string
+      source?: { type?: string; value?: string; mimeType?: string }
+    }
+    if (p.type === 'text') {
+      parts.push({
+        type: 'text',
+        content: redact((p.text ?? p.content ?? '').toString()),
+      })
+      continue
+    }
+    const modality = p.type ?? 'unknown'
+    const { type, value, mimeType } = p.source ?? {}
+    const mime = mimeType ? { mime_type: mimeType } : {}
+    if (type === 'url' && value && !value.startsWith('data:')) {
+      parts.push({ type: 'uri', modality, uri: value, ...mime })
+    } else if (type === 'file' && value) {
+      parts.push({ type: 'file', modality, file_id: value, ...mime })
+    } else {
+      parts.push({ type: 'text', content: `[${modality}]` })
+    }
+  }
+  return parts
+}
+
+/** A media reference as a content part, so `serializeParts` can map it. */
+function mediaPart(modality: string, url: unknown): unknown {
+  return {
+    type: modality,
+    source: typeof url === 'string' ? { type: 'url', value: url } : undefined,
+  }
+}
+
+/**
+ * Content parts for a media call's inputs, read from `artifactInputs`: the
+ * prompt (a string or `MediaPrompt` parts), TTS `text`, and transcription
+ * `audio`. Audio is a reference only when it is an http(s) URL; a base64
+ * string, `File` or `Blob` becomes a placeholder.
+ */
+function mediaInputParts(inputs: unknown): Array<unknown> {
+  if (!inputs || typeof inputs !== 'object') return []
+  const { prompt, text, audio } = inputs as Record<string, unknown>
+  const parts: Array<unknown> = []
+  for (const value of [prompt, text]) {
+    if (typeof value === 'string') parts.push({ type: 'text', content: value })
+    else if (Array.isArray(value)) parts.push(...value)
+  }
+  if (audio !== undefined) {
+    const url =
+      typeof audio === 'string' && /^https?:\/\//.test(audio) ? audio : null
+    parts.push(mediaPart('audio', url))
+  }
+  return parts
+}
+
+/**
+ * Content parts for a media call's result: generated image, audio or video
+ * URLs, and transcript text. Base64 output becomes a placeholder.
+ */
+function mediaOutputParts(
+  activity: GenerationActivity,
+  result: unknown,
+): Array<unknown> {
+  if (!result || typeof result !== 'object') return []
+  const r = result as Record<string, unknown>
+  const parts: Array<unknown> = []
+  if (Array.isArray(r.images)) {
+    for (const image of r.images) {
+      parts.push(mediaPart('image', (image as { url?: unknown } | null)?.url))
+    }
+  }
+  // `TTSResult.audio` is a base64 string; `AudioGenerationResult.audio` is a
+  // `{ url } | { b64Json }` source.
+  if (r.audio !== undefined) {
+    parts.push(mediaPart('audio', (r.audio as { url?: unknown } | null)?.url))
+  }
+  // Only video puts its asset on a top-level `url`. A world `url` is a viewer
+  // page, not media.
+  if (activity === 'video' && typeof r.url === 'string') {
+    parts.push(mediaPart('video', r.url))
+  }
+  if (typeof r.text === 'string') parts.push({ type: 'text', content: r.text })
+  return parts
+}
+
 function messageEventName(role: string): string {
   switch (role) {
     case 'user':
@@ -340,6 +448,29 @@ export function otelMiddleware(
     })
   }
 
+  // Media prompts and transcripts are single strings, so cap each text part
+  // with `maxContentLength` the same way the chat completion buffer is capped.
+  const redactMediaText = (text: string): string =>
+    redactContent(
+      maxContentLength > 0 && text.length > maxContentLength
+        ? text.slice(0, maxContentLength) + '…'
+        : text,
+    )
+
+  const setMediaMessages = (
+    span: Span,
+    direction: 'input' | 'output',
+    parts: Array<unknown>,
+  ): void => {
+    const content = serializeParts(parts, redactMediaText)
+    if (content.length === 0) return
+    const json = JSON.stringify([
+      { role: direction === 'input' ? 'user' : 'assistant', content },
+    ])
+    span.setAttribute(`gen_ai.${direction}.messages`, json)
+    span.setAttribute(`langfuse.observation.${direction}`, json)
+  }
+
   const startMediaSpan = (ctx: GenerationMiddlewareContext): void => {
     safeCall('otel.onStart', () => {
       const operationName = OPERATION_NAME[ctx.activity]
@@ -365,6 +496,22 @@ export function otelMiddleware(
       )
       if (enriched) span.setAttributes(enriched)
       mediaSpans.set(ctx, span)
+
+      if (captureContent) {
+        setMediaMessages(span, 'input', mediaInputParts(ctx.artifactInputs))
+        // Observe the result through a transform: the terminal hooks never
+        // see it. Returns `undefined`, so the result is left unchanged.
+        ctx.resultTransforms.push((result) => {
+          safeCall('otel.captureOutput', () =>
+            setMediaMessages(
+              span,
+              'output',
+              mediaOutputParts(ctx.activity, result),
+            ),
+          )
+          return undefined
+        })
+      }
     })
   }
 
@@ -581,7 +728,12 @@ export function otelMiddleware(
           // Also emit the current GenAI-semconv attribute form
           // (`gen_ai.input.messages`) — backends like PostHog read prompt
           // content from this attribute, not from span events.
-          const inputMessages: Array<{ role: string; content: string }> = []
+          // Multimodal messages keep their parts structured so image / audio /
+          // video / document references survive into the trace (#1525).
+          const inputMessages: Array<{
+            role: string
+            content: string | Array<InputPart>
+          }> = []
           for (const sys of systemPromptContents) {
             inputMessages.push({
               role: 'system',
@@ -589,6 +741,12 @@ export function otelMiddleware(
             })
           }
           for (const m of config.messages) {
+            if (Array.isArray(m.content)) {
+              const parts = serializeParts(m.content, redactContent)
+              if (parts.length === 0) continue
+              inputMessages.push({ role: m.role, content: parts })
+              continue
+            }
             const body = serializeContent(m.content)
             if (body.length === 0) continue
             inputMessages.push({