@tanstack/ai 0.59.0 → 0.61.0

package/src/client.ts

@@ -329,6 +329,13 @@ export type { AdapterYieldChunk } from './utilities/adapter-yield-chunk'
 export { getChunkRunId, getChunkThreadId } from './utilities/chunk-ids'
 export type { WireMessage } from './utilities/ag-ui-wire'
 
+// A browser client that received an uploaded handle from its server can build
+// the `{ type: 'file' }` content source itself — `fileSourceFromHandle` is a
+// pure object builder, so exporting it here keeps the documented client flow
+// from pulling in the server entry.
+export { fileSourceFromHandle } from './activities/files/index'
+export type { FileHandle } from './activities/files/adapter'
+
 export type {
   AudioPart,
   ContentPart,

package/src/index.ts

@@ -18,6 +18,10 @@ export {
   embed,
   generateWorld,
   generateLiveVideo,
+  uploadFile,
+  getFile,
+  deleteFile,
+  fileSourceFromHandle,
 } from './activities/index'
 
 // Create options functions - for pre-defining typed configurations
@@ -70,6 +74,10 @@ export type {
   WorldAdapter,
   AnyLiveVideoAdapter,
   LiveVideoAdapter,
+  FilesAdapter,
+  AnyFilesAdapter,
+  FileHandle,
+  FileUploadInput,
 } from './activities/index'
 
 // Rerank adapter base + types
@@ -529,6 +537,14 @@ export {
   isContentPartArray,
   normalizeToolResult,
 } from './utilities/tool-result'
+export {
+  assertMessagesFileSourceSupport,
+  assertPromptFileSourceSupport,
+  fileReferenceFor,
+  isFileSource,
+  unsupportedFileSourceError,
+  type FileSourceCapable,
+} from './utilities/content-source'
 
 export {
   getProviderExecutedMetadata,

package/src/types.ts

@@ -239,8 +239,24 @@ export interface ContentPartUrlSource extends AGUIUrlSource {}
 /**
  * A provider-issued file handle (Files API). AG-UI `FileSource`: the handle
  * is opaque, do not fetch or parse it.
+ *
+ * The media is uploaded once via a `files` adapter (`openaiFiles()`,
+ * `anthropicFiles()`, `geminiFiles()`, `grokFiles()`, `falFiles()`) and
+ * referenced here by the returned handle instead of re-sending base64 or a
+ * public URL on each request. Only the provider that minted a handle can
+ * resolve it. Adapters that cannot consume file handles at all are rejected
+ * by the activity-layer preflight before mapping starts.
  */
-export interface ContentPartFileSource extends AGUIFileSource {}
+export interface ContentPartFileSource<
+  TProvider extends string = string,
+> extends AGUIFileSource {
+  /**
+   * The adapter name of the provider that issued the handle (`'openai'`,
+   * `'gemini'`, ...), the same id TanStack reports as the usage provider.
+   * When present, an adapter rejects a handle another provider issued.
+   */
+  provider?: TProvider
+}
 
 /**
  * Where a media part's bytes come from: inline data, a URL, or a provider
@@ -2305,7 +2321,7 @@ export interface VideoUrlResult {
 // ============================================================================
 
 /**
- * Options for world generation (live, prompt-steerable sessions).
+ * Options for world generation (live session or finished job).
  *
  * @experimental World generation is an experimental feature and may change.
  */
@@ -2317,8 +2333,10 @@ export interface WorldGenerationOptions<
   /** Natural-language description of the world or scene */
   prompt: string
   /**
-   * Provider mint options. Reactor resolution/seed/audio are browser
-   * `sendCommand` fields, not token-mint fields.
+   * Provider-specific options. Live adapters (Reactor) use mint fields here.
+   * Job adapters (World Labs) use image/video inputs, `wait`, and poll.
+   * Reactor resolution/seed/audio are browser `sendCommand` fields, not
+   * token-mint fields.
    */
   modelOptions?: TProviderOptions
   /**
@@ -2335,28 +2353,73 @@ export interface WorldGenerationOptions<
   abortSignal?: AbortSignal
 }
 
+/**
+ * Assets from a finished world job. Live session adapters omit this.
+ * URLs are often signed CDN links. They can expire and may need a proxy
+ * to fetch from a browser.
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldGenerationAssets {
+  /** Auto-generated scene description */
+  caption?: string
+  /** Preview image URL */
+  thumbnailUrl?: string
+  splats?: {
+    /** Quality-key map of splat URLs (`100k`, `500k`, `full_res`, …) */
+    spzUrls?: Record<string, string>
+    metricScaleFactor?: number
+    groundPlaneOffset?: number
+  }
+  mesh?: {
+    colliderMeshUrl?: string
+    hqMeshUrl?: string
+    fullResMeshUrl?: string
+  }
+  imagery?: {
+    panoUrl?: string
+  }
+}
+
 /**
  * Result of world generation. JSON-serializable so a server route can return
- * it to a browser. The browser uses `token` + `model` to open the live
- * session (set the prompt, start streaming, steer mid-run).
+ * it to a browser.
+ *
+ * Live adapters (Reactor): `status: 'ready'` with `token` and token
+ * `expiresAt`. The browser uses `token` + `model` to open the session.
+ *
+ * Job adapters (World Labs): `status: 'ready'` with viewer `url` and
+ * `worldId`, or `status: 'waiting'` with `operationId` and no `url`.
+ * `expiresAt` on a job is operation expiry, not a session token.
  *
  * @experimental World generation is an experimental feature and may change.
  */
 export interface WorldGenerationResult {
   /** Unique identifier for this generation */
   id: string
-  /** Model used for generation (provider connect slug) */
+  /** Model used for generation (provider connect slug or model id) */
   model: string
-  /** Short-lived session token for the client connection */
-  token: string
-  /** Token expiry as milliseconds since epoch */
-  expiresAt: number
-  /** Prompt the client should send when it starts the session */
+  /** Short-lived session token for a live client connection */
+  token?: string
+  /**
+   * Expiry as milliseconds since epoch. Live adapters: session token.
+   * Job adapters: operation expiry when the provider sends it.
+   */
+  expiresAt?: number
+  /** Prompt used to generate the world, or the prompt the client should send */
   prompt: string
-  /** Session status after the server half finishes */
+  /** Status after the server half finishes */
   status: 'ready' | 'waiting'
   /** Provider session id, when the adapter created one */
   sessionId?: string
+  /** Viewer URL for a finished world job (not an asset download URL) */
+  url?: string
+  /** Provider world id for a finished or in-progress job */
+  worldId?: string
+  /** Provider operation id for a long-running world job */
+  operationId?: string
+  /** Assets when a world job has finished and the provider returned them */
+  assets?: WorldGenerationAssets
   /** Token usage / billing, when the adapter can report it */
   usage?: TokenUsage
 }

package/src/utilities/ag-ui-wire.ts

@@ -19,6 +19,7 @@ import type {
 } from '../types'
 import type { MetadataRecord } from './merge-metadata'
 import { tanstackMetadata } from './merge-metadata'
+import { isProviderExecutedToolCall } from './provider-executed'
 import { normalizeToolResult } from './tool-result'
 import { wireSubagentInfo, wireSubagentRunId } from './subagent-wire'
 import type { SubagentWireInfo } from './subagent-wire'
@@ -171,9 +172,45 @@ export function uiMessagesToWire(
       continue
     }
 
-    // assistant: emit reasoning fan-outs first, then anchor, then tool fan-outs
+    // assistant: reasoning fan-outs, then anchor, then tool fan-outs.
+    //
+    // Provider-executed tools (Anthropic web_search / web_fetch) run inside one
+    // provider response, and the provider signs every thinking block against
+    // the blocks before it. Emitting all reasoning first and a single anchor
+    // with the joined text and every tool call turns
+    // "thinking, tool, thinking, text, tool" into
+    // "thinking, thinking, text, tool, tool", and the provider rejects the next
+    // turn ("thinking blocks in the latest assistant message cannot be
+    // modified"). Split the message into ordered segments at each thinking
+    // part that follows a provider-executed tool call, the same rule
+    // buildAssistantMessages applies, so the wire keeps the signed order.
+    // Segments after the first get derived anchor ids; strict AG-UI consumers
+    // still see plain anchors.
+    type Segment = {
+      thinking: Array<Extract<MessagePart, { type: 'thinking' }>>
+      parts: Array<MessagePart>
+    }
+    let current: Segment = { thinking: [], parts: [] }
+    const segments: Array<Segment> = [current]
     for (const part of parts) {
       if (part.type === 'thinking') {
+        if (
+          current.parts.some(
+            (p) => p.type === 'tool-call' && isProviderExecutedToolCall(p),
+          )
+        ) {
+          current = { thinking: [part], parts: [] }
+          segments.push(current)
+        } else {
+          current.thinking.push(part)
+        }
+      } else {
+        current.parts.push(part)
+      }
+    }
+
+    segments.forEach((segment, index) => {
+      for (const part of segment.thinking) {
         const reasoning: WireReasoningMessage = {
           role: 'reasoning',
           id: uniqueWireId(deriveReasoningId(uiMessage.id, part), usedWireIds),
@@ -184,22 +221,29 @@ export function uiMessagesToWire(
         }
         wire.push(reasoning)
       }
-    }
 
-    const text = collectText(parts)
-    const toolCalls = collectToolCalls(parts)
-    wire.push(
-      toAnchor(
-        uiMessage,
-        'assistant',
-        {
-          ...(text !== '' && { content: text }),
-          ...(toolCalls && { toolCalls }),
-        },
-        parts,
-        includeSnapshotStructuredOutput,
-      ),
-    )
+      const text = collectText(segment.parts)
+      const toolCalls = collectToolCalls(segment.parts)
+      const anchorMessage: UIMessage =
+        index === 0
+          ? uiMessage
+          : {
+              ...uiMessage,
+              id: uniqueWireId(`${uiMessage.id}-segment-${index}`, usedWireIds),
+            }
+      wire.push(
+        toAnchor(
+          anchorMessage,
+          'assistant',
+          {
+            ...(text !== '' && { content: text }),
+            ...(toolCalls && { toolCalls }),
+          },
+          segment.parts,
+          includeSnapshotStructuredOutput,
+        ),
+      )
+    })
 
     const explicitToolResults = new Set(
       parts.flatMap((part) =>

package/src/utilities/content-source.ts

@@ -0,0 +1,138 @@
+import type { ContentPartFileSource, ContentPartSource } from '../types'
+
+/**
+ * Narrow a {@link ContentPartSource} to the provider-file-reference arm.
+ *
+ * Issuer adapters use this to route a file source to their native wire field;
+ * everyone else is protected by the core preflight (see
+ * {@link assertMessagesFileSourceSupport}) plus a defensive throw at their own
+ * mapping site.
+ */
+export function isFileSource(
+  source: ContentPartSource,
+): source is ContentPartFileSource {
+  return source.type === 'file'
+}
+
+/**
+ * Resolve the handle `providerName` should send for a file source.
+ *
+ * A file source carries one opaque handle (`value`) and, optionally, the
+ * provider that issued it. An adapter always knows which provider it talks
+ * to, so a source that names no provider is taken as-is.
+ *
+ * @throws when the source names a different issuing provider. A handle only
+ * resolves at the provider that minted it.
+ */
+export function fileReferenceFor(
+  source: ContentPartFileSource,
+  providerName: string,
+): string {
+  if (source.provider !== undefined && source.provider !== providerName) {
+    throw new Error(
+      `${providerName}: file source was issued by ${source.provider}. ` +
+        `A provider file handle only works with the provider that issued ` +
+        `it. Upload the file with ${providerName}Files(), or pass a data or ` +
+        `url source instead.`,
+    )
+  }
+  return source.value
+}
+
+/**
+ * Build the standard error a non-issuer adapter throws when it encounters a
+ * `{ type: 'file' }` source it can't consume — either because the provider has
+ * no file-reference input surface, or because the endpoint requires raw bytes
+ * (image edits, Veo) rather than a reference.
+ *
+ * @param detail Optional context appended to the message (e.g. a modality or
+ * endpoint name, or a pointer to the adapter that does support references).
+ * When provided it replaces the generic remediation tail, so a site-specific
+ * hint ("pass inline bytes") is never contradicted by generic advice.
+ */
+export function unsupportedFileSourceError(
+  providerName: string,
+  detail?: string,
+): Error {
+  return new Error(
+    `${providerName} does not support provider file-handle sources ` +
+      `({ type: 'file' })` +
+      (detail
+        ? ` ${detail}.`
+        : `. Pass a data or url source, or upload via the provider's files ` +
+          `adapter where supported.`),
+  )
+}
+
+/**
+ * The slice of an adapter the file-source preflight reads. Adapters that can
+ * consume `{ type: 'file' }` sources declare `supportsFileSources: true`;
+ * everything else — including adapters written before this arm existed —
+ * fails closed at the activity layer instead of falling through to a
+ * URL/data branch and silently mis-mapping the reference.
+ */
+export interface FileSourceCapable {
+  name: string
+  supportsFileSources?: boolean
+}
+
+/** True when a content-part-like value carries a `{ type: 'file' }` source. */
+function partHasFileSource(part: unknown): boolean {
+  if (typeof part !== 'object' || part === null) return false
+  const source = (part as { source?: unknown }).source
+  return (
+    typeof source === 'object' &&
+    source !== null &&
+    (source as { type?: unknown }).type === 'file'
+  )
+}
+
+/**
+ * True when `value` is a content part with a file source, or an array
+ * (possibly nested — fused embedding items) that contains one.
+ */
+function inputHasFileSource(value: unknown): boolean {
+  if (Array.isArray(value)) return value.some(inputHasFileSource)
+  return partHasFileSource(value)
+}
+
+/**
+ * Fail-closed preflight for media prompts and embedding inputs
+ * (`generateImage` / `generateVideo` / `embed`): throws when the input
+ * carries a `{ type: 'file' }` source and the adapter hasn't declared
+ * `supportsFileSources`. Runs in the activity dispatcher — the same layer
+ * that validates modality — so an adapter that predates the file arm can
+ * never receive one. Walks a single part, an array of parts, and nested
+ * arrays (fused embedding items).
+ */
+export function assertPromptFileSourceSupport(
+  adapter: FileSourceCapable,
+  prompt: unknown,
+): void {
+  if (adapter.supportsFileSources === true) return
+  if (inputHasFileSource(prompt)) {
+    throw unsupportedFileSourceError(adapter.name)
+  }
+}
+
+/**
+ * Fail-closed preflight for chat messages: throws when any message content
+ * part carries a `{ type: 'file' }` source and the adapter hasn't declared
+ * `supportsFileSources`. See {@link assertPromptFileSourceSupport}.
+ */
+export function assertMessagesFileSourceSupport(
+  adapter: FileSourceCapable,
+  messages: ReadonlyArray<unknown>,
+): void {
+  if (adapter.supportsFileSources === true) return
+  for (const message of messages) {
+    if (typeof message !== 'object' || message === null) continue
+    const content = (message as { content?: unknown }).content
+    if (!Array.isArray(content)) continue
+    for (const part of content) {
+      if (partHasFileSource(part)) {
+        throw unsupportedFileSourceError(adapter.name)
+      }
+    }
+  }
+}

package/src/utilities/provider-executed.ts

@@ -30,3 +30,16 @@ export function isProviderExecutedToolCall(
 ): boolean {
   return getProviderExecutedMetadata(toolCall) !== null
 }
+
+/**
+ * True when `id` is a `${parentId}-segment-${n}` message. The wire and the
+ * run loop split one provider turn into segments at thinking that follows a
+ * provider-executed tool call, to keep signed thinking order. Readers fold a
+ * segment back into its parent so the UI still shows one message.
+ */
+export function isAssistantSegmentOf(
+  id: string | undefined,
+  parentId: string,
+): boolean {
+  return id?.startsWith(`${parentId}-segment-`) === true
+}

package/src/utilities/tool-result.ts

@@ -10,8 +10,9 @@ const CONTENT_PART_TYPES = new Set([
 
 /**
  * Structural check for a single `ContentPart`. A text part must carry a string
- * `content`; every other modality must carry a `source` with `type` of
- * `'url' | 'data'` and a string `value`.
+ * `content`. Every other part carries a source with a string `value`; a file
+ * source's `value` is a non-empty opaque handle, and its optional `provider`
+ * is a string.
  */
 export function isContentPart(value: unknown): value is ContentPart {
   if (typeof value !== 'object' || value === null) return false
@@ -26,6 +27,14 @@ export function isContentPart(value: unknown): value is ContentPart {
   if (typeof source !== 'object' || source === null) return false
   const src = source as Record<string, unknown>
   if (typeof src.value !== 'string') return false
+  // `file` sources carry an opaque handle in `value`; `provider`, when set,
+  // names the issuer.
+  if (src.type === 'file') {
+    return (
+      src.value.length > 0 &&
+      (src.provider === undefined || typeof src.provider === 'string')
+    )
+  }
   // `data` sources require a mimeType (matches ContentPartDataSource); `url`
   // sources don't. Requiring it here keeps the runtime guard consistent with
   // the type and avoids emitting `data:undefined;base64,...` downstream.
@@ -45,6 +54,33 @@ export function isContentPartArray(
   return Array.isArray(value) && value.length > 0 && value.every(isContentPart)
 }
 
+/**
+ * Error text for a failed tool result: `output.error` when it is a string,
+ * else the output itself when it is a string, else a generic message.
+ * `StreamProcessor` and `chat()` history share it, so a reload shows the
+ * same text as the live stream.
+ */
+export function toolResultErrorText(output: unknown): string {
+  if (
+    output &&
+    typeof output === 'object' &&
+    'error' in output &&
+    typeof output.error === 'string'
+  ) {
+    return output.error
+  }
+  return typeof output === 'string' ? output : 'Tool execution failed'
+}
+
+/** Parse tool result content as JSON. Plain text stays a string. */
+export function parseToolOutput(content: string): unknown {
+  try {
+    return JSON.parse(content)
+  } catch {
+    return content
+  }
+}
+
 /**
  * Normalize a tool's return value for transport:
  * - string            → unchanged