@tanstack/ai 0.61.0 → 0.63.0

package/src/middlewares/otel.ts

@@ -108,7 +108,9 @@ export interface OtelMiddlewareOptions {
   meter?: Meter
   /**
    * When `true`, prompt and completion content is attached to iteration spans
-   * as `gen_ai.*.message` / `gen_ai.choice` events. Defaults to `false` so
+   * as `gen_ai.*.message` / `gen_ai.choice` events. Media spans get the
+   * prompt, input media and output (URLs or transcript text) as
+   * `gen_ai.input.messages` / `gen_ai.output.messages`. Defaults to `false` so
    * that PII never lands on a span by accident.
    */
   captureContent?: boolean
@@ -121,7 +123,8 @@ export interface OtelMiddlewareOptions {
   redact?: (text: string) => string
   /**
    * Maximum characters kept in the per-iteration assistant text buffer used
-   * to emit `gen_ai.choice` events. Extra characters are truncated with a
+   * to emit `gen_ai.choice` events, and in each text part of a media span's
+   * input and output. Extra characters are truncated with a
    * trailing `"…"` marker. Defaults to 100 000. Set to `0` to disable the
    * cap. Exporters typically truncate long attribute values anyway.
    */
@@ -221,6 +224,111 @@ function serializeContent(content: unknown): string {
   return parts.join(' ')
 }
 
+type InputPart =
+  | { type: 'text'; content: string }
+  | { type: 'uri'; modality: string; uri: string; mime_type?: string }
+  | { type: 'file'; modality: string; file_id: string; mime_type?: string }
+
+/**
+ * Structured form of `ContentPart[]` for `gen_ai.input.messages`, using the
+ * OTel GenAI semconv part shapes. URL and file-handle media keep their
+ * reference; inline bytes (and `data:` URLs) stay a `[type]` placeholder so
+ * they never blow attribute size limits. `redact` runs on text parts only.
+ */
+function serializeParts(
+  content: Array<unknown>,
+  redact: (text: string) => string,
+): Array<InputPart> {
+  const parts: Array<InputPart> = []
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue
+    const p = part as {
+      type?: string
+      text?: string
+      content?: string
+      source?: { type?: string; value?: string; mimeType?: string }
+    }
+    if (p.type === 'text') {
+      parts.push({
+        type: 'text',
+        content: redact((p.text ?? p.content ?? '').toString()),
+      })
+      continue
+    }
+    const modality = p.type ?? 'unknown'
+    const { type, value, mimeType } = p.source ?? {}
+    const mime = mimeType ? { mime_type: mimeType } : {}
+    if (type === 'url' && value && !value.startsWith('data:')) {
+      parts.push({ type: 'uri', modality, uri: value, ...mime })
+    } else if (type === 'file' && value) {
+      parts.push({ type: 'file', modality, file_id: value, ...mime })
+    } else {
+      parts.push({ type: 'text', content: `[${modality}]` })
+    }
+  }
+  return parts
+}
+
+/** A media reference as a content part, so `serializeParts` can map it. */
+function mediaPart(modality: string, url: unknown): unknown {
+  return {
+    type: modality,
+    source: typeof url === 'string' ? { type: 'url', value: url } : undefined,
+  }
+}
+
+/**
+ * Content parts for a media call's inputs, read from `artifactInputs`: the
+ * prompt (a string or `MediaPrompt` parts), TTS `text`, and transcription
+ * `audio`. Audio is a reference only when it is an http(s) URL; a base64
+ * string, `File` or `Blob` becomes a placeholder.
+ */
+function mediaInputParts(inputs: unknown): Array<unknown> {
+  if (!inputs || typeof inputs !== 'object') return []
+  const { prompt, text, audio } = inputs as Record<string, unknown>
+  const parts: Array<unknown> = []
+  for (const value of [prompt, text]) {
+    if (typeof value === 'string') parts.push({ type: 'text', content: value })
+    else if (Array.isArray(value)) parts.push(...value)
+  }
+  if (audio !== undefined) {
+    const url =
+      typeof audio === 'string' && /^https?:\/\//.test(audio) ? audio : null
+    parts.push(mediaPart('audio', url))
+  }
+  return parts
+}
+
+/**
+ * Content parts for a media call's result: generated image, audio or video
+ * URLs, and transcript text. Base64 output becomes a placeholder.
+ */
+function mediaOutputParts(
+  activity: GenerationActivity,
+  result: unknown,
+): Array<unknown> {
+  if (!result || typeof result !== 'object') return []
+  const r = result as Record<string, unknown>
+  const parts: Array<unknown> = []
+  if (Array.isArray(r.images)) {
+    for (const image of r.images) {
+      parts.push(mediaPart('image', (image as { url?: unknown } | null)?.url))
+    }
+  }
+  // `TTSResult.audio` is a base64 string; `AudioGenerationResult.audio` is a
+  // `{ url } | { b64Json }` source.
+  if (r.audio !== undefined) {
+    parts.push(mediaPart('audio', (r.audio as { url?: unknown } | null)?.url))
+  }
+  // Only video puts its asset on a top-level `url`. A world `url` is a viewer
+  // page, not media.
+  if (activity === 'video' && typeof r.url === 'string') {
+    parts.push(mediaPart('video', r.url))
+  }
+  if (typeof r.text === 'string') parts.push({ type: 'text', content: r.text })
+  return parts
+}
+
 function messageEventName(role: string): string {
   switch (role) {
     case 'user':
@@ -340,6 +448,29 @@ export function otelMiddleware(
     })
   }
 
+  // Media prompts and transcripts are single strings, so cap each text part
+  // with `maxContentLength` the same way the chat completion buffer is capped.
+  const redactMediaText = (text: string): string =>
+    redactContent(
+      maxContentLength > 0 && text.length > maxContentLength
+        ? text.slice(0, maxContentLength) + '…'
+        : text,
+    )
+
+  const setMediaMessages = (
+    span: Span,
+    direction: 'input' | 'output',
+    parts: Array<unknown>,
+  ): void => {
+    const content = serializeParts(parts, redactMediaText)
+    if (content.length === 0) return
+    const json = JSON.stringify([
+      { role: direction === 'input' ? 'user' : 'assistant', content },
+    ])
+    span.setAttribute(`gen_ai.${direction}.messages`, json)
+    span.setAttribute(`langfuse.observation.${direction}`, json)
+  }
+
   const startMediaSpan = (ctx: GenerationMiddlewareContext): void => {
     safeCall('otel.onStart', () => {
       const operationName = OPERATION_NAME[ctx.activity]
@@ -365,6 +496,22 @@ export function otelMiddleware(
       )
       if (enriched) span.setAttributes(enriched)
       mediaSpans.set(ctx, span)
+
+      if (captureContent) {
+        setMediaMessages(span, 'input', mediaInputParts(ctx.artifactInputs))
+        // Observe the result through a transform: the terminal hooks never
+        // see it. Returns `undefined`, so the result is left unchanged.
+        ctx.resultTransforms.push((result) => {
+          safeCall('otel.captureOutput', () =>
+            setMediaMessages(
+              span,
+              'output',
+              mediaOutputParts(ctx.activity, result),
+            ),
+          )
+          return undefined
+        })
+      }
     })
   }
 
@@ -581,7 +728,12 @@ export function otelMiddleware(
           // Also emit the current GenAI-semconv attribute form
           // (`gen_ai.input.messages`) — backends like PostHog read prompt
           // content from this attribute, not from span events.
-          const inputMessages: Array<{ role: string; content: string }> = []
+          // Multimodal messages keep their parts structured so image / audio /
+          // video / document references survive into the trace (#1525).
+          const inputMessages: Array<{
+            role: string
+            content: string | Array<InputPart>
+          }> = []
           for (const sys of systemPromptContents) {
             inputMessages.push({
               role: 'system',
@@ -589,6 +741,12 @@ export function otelMiddleware(
             })
           }
           for (const m of config.messages) {
+            if (Array.isArray(m.content)) {
+              const parts = serializeParts(m.content, redactContent)
+              if (parts.length === 0) continue
+              inputMessages.push({ role: m.role, content: parts })
+              continue
+            }
             const body = serializeContent(m.content)
             if (body.length === 0) continue
             inputMessages.push({

package/src/types.ts

@@ -100,6 +100,9 @@ export type ToolResultState =
   | 'complete' // Result is complete
   | 'error' // Error occurred
 
+/** Why a tool result ended without executing successfully. */
+export type ToolResultOutcome = 'cancelled' | 'denied'
+
 export type ToolOutputState = 'output-available' | 'output-error'
 
 /**
@@ -196,6 +199,13 @@ export interface ToolCall<TMetadata = unknown> extends Omit<
   metadata?: TMetadata
 }
 
+/** One source link from a provider-executed web search. */
+export interface ProviderExecutedToolSource {
+  url: string
+  title?: string
+  pageAge?: string
+}
+
 /**
  * Convention for tool-call `metadata` that marks a call as **provider-executed**
  * — run by the provider's own infrastructure (e.g. Anthropic `web_search` /
@@ -209,10 +219,12 @@ export interface ToolCall<TMetadata = unknown> extends Omit<
  *
  * Provider-specific payloads live under a namespaced key (e.g. `anthropic`),
  * keeping this convention opaque to the framework core. The index signature
- * preserves those per-adapter fields.
+ * preserves those per-adapter fields. `sources` is the normalized list of
+ * links a web search used, shared across providers.
  */
 export interface ProviderExecutedToolMetadata {
   providerExecuted?: boolean
+  sources?: Array<ProviderExecutedToolSource>
   [key: string]: unknown
 }
 
@@ -442,6 +454,8 @@ export interface ToolResultPart {
   toolCallId: string
   content: string | Array<ContentPart>
   state: ToolResultState
+  /** Set when the user or middleware cancelled or denied the tool call; state remains `error`. */
+  outcome?: ToolResultOutcome
   error?: string // Error message if state is "error"
   metadata?: Record<string, unknown>
   createdAt?: Date
@@ -560,6 +574,12 @@ export interface TanStackMessageMetadata {
   model?: string
   /** Parent chat run that produced this assistant message. */
   runId?: string
+  /**
+   * The chat run that produced this assistant message. `withPersistence` sets
+   * `id`. `reconstructChat` with `includeRuns: true` adds the finished run's
+   * timings, in epoch ms.
+   */
+  run?: { id: string; startedAt?: number; finishedAt?: number }
   /** Card data on a child wire message. See `uiMessagesToWire`. */
   subagent?: SubagentWireInfo
   /** Thinking signature for a `role: 'reasoning'` fan-out message. */
@@ -571,6 +591,8 @@ export interface TanStackMessageMetadata {
     createdAt?: string
     content?: Array<ContentPart>
   }
+  /** Outcome of a cancelled or denied tool result; when present, the UI state is `error`. */
+  toolResultOutcome?: ToolResultOutcome
   structuredOutput?: {
     status?: 'streaming' | 'complete' | 'error'
     partial?: unknown
@@ -679,6 +701,15 @@ export interface EmitCustomEventOptions {
   batch?: boolean
 }
 
+/**
+ * The user's answer to an `mcp_input` interrupt.
+ * `resolved` carries the `payload` from `resolveInterrupt`.
+ * `cancelled` means the user called `cancel()`.
+ */
+export type ToolInputResponse =
+  | { status: 'resolved'; payload: unknown }
+  | { status: 'cancelled' }
+
 /**
  * Context passed to tool execute functions, providing capabilities like
  * emitting custom events during execution.
@@ -693,6 +724,12 @@ export type ToolExecutionContext<TContext = unknown> =
      * e.g. MCP `callTool` — should forward this to cancel in-flight work.
      */
     abortSignal?: AbortSignal
+    /**
+     * The answer to the input request that this tool call raised in the
+     * previous run. It is set only when the run resumes an `mcp_input`
+     * interrupt for this tool call.
+     */
+    inputResponse?: ToolInputResponse
     /**
      * Emit a custom event during tool execution.
      * Events are streamed to the client in real-time as AG-UI CUSTOM events.

package/src/utilities/ag-ui-wire.ts

@@ -14,6 +14,7 @@ import type {
   StructuredOutputPart,
   SubagentPart,
   TanStackMessageMetadata,
+  ToolResultOutcome,
   UIMessage,
   UIResourcePart,
 } from '../types'
@@ -49,6 +50,7 @@ function rebuiltToolMetadata(
   id: string | undefined,
   content: string | null | Array<ContentPart>,
   anchorOwnsUiResources = false,
+  outcome?: ToolResultOutcome,
 ): MetadataRecord | undefined {
   const source: MetadataRecord = isRecord(metadata) ? metadata : {}
   const tanstack = isRecord(source.tanstack) ? { ...source.tanstack } : {}
@@ -61,7 +63,11 @@ function rebuiltToolMetadata(
   }
   const result = {
     ...source,
-    tanstack: { ...tanstack, toolResult },
+    tanstack: {
+      ...tanstack,
+      ...(outcome !== undefined && { toolResultOutcome: outcome }),
+      toolResult,
+    },
   }
   return Object.keys(result).length ? result : undefined
 }
@@ -262,6 +268,7 @@ export function uiMessagesToWire(
           part.id,
           part.content,
           true,
+          part.outcome,
         )
         wire.push({
           role: 'tool',
@@ -306,6 +313,8 @@ export function uiMessagesToWire(
             undefined,
             undefined,
             result,
+            false,
+            part.approval?.approved === false ? 'denied' : undefined,
           ),
         })
       }
@@ -385,6 +394,8 @@ function messageMetadata(
     tanstack.model = previousTanstack.model
   if (previousTanstack?.runId !== undefined)
     tanstack.runId = previousTanstack.runId
+  if (previousTanstack?.run?.id !== undefined)
+    tanstack.run = { id: previousTanstack.run.id }
   if (previousTanstack?.signature !== undefined)
     tanstack.signature = previousTanstack.signature
   const createdAt = coerceCreatedAt(msg.createdAt)

package/src/utilities/tool-result.ts

@@ -1,4 +1,4 @@
-import type { ContentPart } from '../types'
+import type { ContentPart, ToolResultOutcome } from '../types'
 
 const CONTENT_PART_TYPES = new Set([
   'text',
@@ -8,6 +8,12 @@ const CONTENT_PART_TYPES = new Set([
   'document',
 ])
 
+export function isToolResultOutcome(
+  value: unknown,
+): value is ToolResultOutcome {
+  return value === 'cancelled' || value === 'denied'
+}
+
 /**
  * Structural check for a single `ContentPart`. A text part must carry a string
  * `content`. Every other part carries a source with a string `value`; a file