@tanstack/ai 0.37.0 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/esm/activities/chat/index.js +28 -0
  2. package/dist/esm/activities/chat/index.js.map +1 -1
  3. package/dist/esm/activities/chat/mcp/manager.js +10 -1
  4. package/dist/esm/activities/chat/mcp/manager.js.map +1 -1
  5. package/dist/esm/activities/chat/mcp/types.d.ts +22 -0
  6. package/dist/esm/activities/chat/messages.js.map +1 -1
  7. package/dist/esm/activities/chat/middleware/compose.d.ts +7 -1
  8. package/dist/esm/activities/chat/middleware/compose.js +27 -0
  9. package/dist/esm/activities/chat/middleware/compose.js.map +1 -1
  10. package/dist/esm/activities/chat/middleware/index.d.ts +1 -1
  11. package/dist/esm/activities/chat/middleware/sandbox-runtime.d.ts +8 -0
  12. package/dist/esm/activities/chat/middleware/sandbox-runtime.js +9 -0
  13. package/dist/esm/activities/chat/middleware/sandbox-runtime.js.map +1 -0
  14. package/dist/esm/activities/chat/middleware/types.d.ts +22 -0
  15. package/dist/esm/activities/chat/stream/processor.js +23 -0
  16. package/dist/esm/activities/chat/stream/processor.js.map +1 -1
  17. package/dist/esm/activities/chat/tools/tool-calls.d.ts +6 -1
  18. package/dist/esm/activities/chat/tools/tool-calls.js +36 -0
  19. package/dist/esm/activities/chat/tools/tool-calls.js.map +1 -1
  20. package/dist/esm/adapter-internals.d.ts +2 -0
  21. package/dist/esm/adapter-internals.js +4 -0
  22. package/dist/esm/adapter-internals.js.map +1 -1
  23. package/dist/esm/client.d.ts +1 -1
  24. package/dist/esm/client.js.map +1 -1
  25. package/dist/esm/index.d.ts +2 -2
  26. package/dist/esm/logger/internal-logger.d.ts +2 -0
  27. package/dist/esm/logger/internal-logger.js +6 -1
  28. package/dist/esm/logger/internal-logger.js.map +1 -1
  29. package/dist/esm/logger/resolve.js +6 -3
  30. package/dist/esm/logger/resolve.js.map +1 -1
  31. package/dist/esm/logger/types.d.ts +5 -0
  32. package/dist/esm/types.d.ts +52 -1
  33. package/package.json +1 -1
  34. package/skills/ai-core/adapter-configuration/SKILL.md +25 -12
  35. package/skills/ai-core/chat-experience/SKILL.md +30 -2
  36. package/skills/ai-core/media-generation/SKILL.md +13 -0
  37. package/skills/ai-core/tool-calling/SKILL.md +10 -3
  38. package/src/activities/chat/index.ts +39 -0
  39. package/src/activities/chat/mcp/manager.ts +31 -1
  40. package/src/activities/chat/mcp/types.ts +23 -0
  41. package/src/activities/chat/messages.ts +5 -0
  42. package/src/activities/chat/middleware/compose.ts +34 -0
  43. package/src/activities/chat/middleware/index.ts +2 -0
  44. package/src/activities/chat/middleware/sandbox-runtime.ts +21 -0
  45. package/src/activities/chat/middleware/types.ts +37 -0
  46. package/src/activities/chat/stream/processor.ts +42 -0
  47. package/src/activities/chat/tools/tool-calls.ts +96 -2
  48. package/src/adapter-internals.ts +6 -0
  49. package/src/client.ts +1 -0
  50. package/src/index.ts +4 -0
  51. package/src/logger/internal-logger.ts +6 -0
  52. package/src/logger/resolve.ts +3 -0
  53. package/src/logger/types.ts +5 -0
  54. package/src/types.ts +51 -0
@@ -278,7 +278,35 @@ if (part.type === 'image') {
278
278
  }
279
279
  ```
280
280
 
281
- ### 4. HTTP Stream Format (Alternative to SSE)
281
+ ### 4. Sending Audio Messages (Browser Recording)
282
+
283
+ Use `useAudioRecorder` from `@tanstack/ai-react` (or `createAudioRecorder` in Svelte) to capture audio in the browser. The resolved `AudioRecording` includes a ready-to-use `part` that slots directly into `sendMessage`.
284
+
285
+ ```typescript
286
+ import {
287
+ useAudioRecorder,
288
+ useChat,
289
+ fetchServerSentEvents,
290
+ } from '@tanstack/ai-react'
291
+
292
+ const { isRecording, isSupported, start, stop } = useAudioRecorder()
293
+ const { sendMessage } = useChat({
294
+ connection: fetchServerSentEvents('/api/chat'),
295
+ })
296
+
297
+ async function toggle() {
298
+ if (!isRecording) {
299
+ await start()
300
+ return
301
+ }
302
+ const recording = await stop()
303
+ await sendMessage({ content: [recording.part] })
304
+ }
305
+ ```
306
+
307
+ `recording.part` is `{ type: 'audio', source: { type: 'data', value: base64, mimeType } }`. Returns the recorder's native format (`audio/webm` or `audio/mp4`) with no transcoding.
308
+
309
+ ### 5. HTTP Stream Format (Alternative to SSE)
282
310
 
283
311
  Use `toHttpResponse` + `fetchHttpStream` for newline-delimited JSON instead of SSE.
284
312
 
@@ -310,7 +338,7 @@ const { messages, sendMessage } = useChat({
310
338
  The only difference is swapping `toServerSentEventsResponse` / `fetchServerSentEvents`
311
339
  for `toHttpResponse` / `fetchHttpStream`. Everything else stays identical.
312
340
 
313
- ### 5. MCP Tool Discovery via `chat({ mcp })`
341
+ ### 6. MCP Tool Discovery via `chat({ mcp })`
314
342
 
315
343
  Pass `mcp` to let `chat()` own discovery **and** lifecycle for one or more MCP
316
344
  clients. Useful when you want minimal boilerplate and don't need to reuse the
@@ -359,6 +359,19 @@ const { generate, result, isLoading } = useGenerateSpeech({
359
359
  Adapter: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
360
360
  gpt-4o-mini-transcribe).
361
361
 
362
+ > **Capturing audio in the browser:** Use `useAudioRecorder` from `@tanstack/ai-react` to record directly in the browser, then pass the recording as the `audio` input to `generate()`, or use `recording.part` as a prompt part in chat/generation calls. No transcoding or extra dependencies required — the recorder returns the native browser format (`audio/webm` or `audio/mp4`). For transcription, wrap it as a `data:` URL so the provider gets the real content type; passing raw `recording.base64` makes the adapter assume `audio/mpeg` and mislabel the webm/mp4 bytes.
363
+ >
364
+ > ```typescript
365
+ > const { isRecording, start, stop } = useAudioRecorder()
366
+ > const { generate } = useTranscription({
367
+ > connection: fetchServerSentEvents('/api/transcribe'),
368
+ > })
369
+ > // ...
370
+ > const recording = await stop()
371
+ > const mimeType = recording.mimeType.split(';')[0] // strip ;codecs=...
372
+ > await generate({ audio: `data:${mimeType};base64,${recording.base64}` })
373
+ > ```
374
+
362
375
  ```typescript
363
376
  import { generateTranscription } from '@tanstack/ai'
364
377
  import { openaiTranscription } from '@tanstack/ai-openai'
@@ -75,7 +75,7 @@ import { updateCartUIDef } from '@/tools/definitions'
75
75
  export async function POST(request: Request) {
76
76
  const { messages } = await request.json()
77
77
  const stream = chat({
78
- adapter: openaiText('gpt-4o'),
78
+ adapter: openaiText('gpt-5.5'),
79
79
  messages,
80
80
  tools: [getProducts, updateCartUIDef], // server tool + client definition
81
81
  })
@@ -158,7 +158,7 @@ const getUserData = getUserDataDef.server(async ({ userId }) => {
158
158
 
159
159
  // In your route handler:
160
160
  const stream = chat({
161
- adapter: openaiText('gpt-4o'),
161
+ adapter: openaiText('gpt-5.5'),
162
162
  messages,
163
163
  tools: [getUserData],
164
164
  })
@@ -188,7 +188,7 @@ Server -- pass definition only (no execute function):
188
188
 
189
189
  ```typescript
190
190
  const stream = chat({
191
- adapter: openaiText('gpt-4o'),
191
+ adapter: openaiText('gpt-5.5'),
192
192
  messages,
193
193
  tools: [showNotificationDef],
194
194
  })
@@ -405,6 +405,13 @@ The post-discovery payload always returns the full description and schema regard
405
405
  `@tanstack/ai-mcp` lets a server-side `chat()` call discover and invoke tools
406
406
  hosted on any MCP server (Streamable HTTP, SSE, or stdio).
407
407
 
408
+ **MCP tools and UI resources:** When an MCP tool result carries a `ui://`
409
+ resource URI (via `_meta.ui.resourceUri`), TanStack AI surfaces it as a
410
+ `UIResourcePart` on the assistant `UIMessage` in the client message list.
411
+ `UIResourcePart` is a presentational-only part — it never enters model input.
412
+ See the `@tanstack/ai-mcp` skill for the full MCP Apps API
413
+ (`createMcpAppCallHandler`, `createMcpAppBridge`, `MCPAppResource`).
414
+
408
415
  ### Basic usage — auto-discovery
409
416
 
410
417
  ```typescript
@@ -27,6 +27,7 @@ import {
27
27
  import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies'
28
28
  import { convertMessagesToModelMessages, generateMessageId } from './messages'
29
29
  import { MiddlewareRunner } from './middleware/compose'
30
+ import { provideSandboxRuntime } from './middleware/sandbox-runtime'
30
31
  import { CapabilityRegistry } from './middleware/capabilities'
31
32
  import { validateCapabilities } from './middleware/validate'
32
33
  import { MCPManager } from './mcp/manager'
@@ -67,6 +68,7 @@ import type {
67
68
  ChatMiddleware,
68
69
  ChatMiddlewareConfig,
69
70
  ChatMiddlewareContext,
71
+ SandboxFileEvent,
70
72
  StructuredOutputMiddlewareConfig,
71
73
  } from './middleware/types'
72
74
  import type { CheckCoverage } from './middleware/builder'
@@ -542,6 +544,7 @@ class TextEngine<
542
544
  // Middleware support
543
545
  private readonly middlewareRunner: MiddlewareRunner<TContext>
544
546
  private readonly middlewareCtx: ChatMiddlewareContext<TContext>
547
+ private readonly sandboxFileQueue: Array<StreamChunk> = []
545
548
  private readonly deferredPromises: Array<Promise<unknown>> = []
546
549
  private abortReason?: string
547
550
  private readonly middlewareAbortController?: AbortController
@@ -701,6 +704,21 @@ class TextEngine<
701
704
  capability[0](this.middlewareCtx, { optional: true }),
702
705
  provide: (capability, value) => capability[1](this.middlewareCtx, value),
703
706
  }
707
+
708
+ // Provide the internal SandboxRuntime capability so harness adapters and
709
+ // sandbox middleware can emit file events. The sink logs, fans the event
710
+ // out through the middleware `onFile*` hooks (fire-and-forget), and queues
711
+ // a `sandbox.file` custom chunk to be drained into the public stream.
712
+ provideSandboxRuntime(this.middlewareCtx, {
713
+ logger: this.logger,
714
+ emit: (event: SandboxFileEvent) => {
715
+ this.logger.sandbox(`file ${event.type} ${event.path}`, { event })
716
+ void this.middlewareRunner.runSandboxFile(this.middlewareCtx, event)
717
+ this.sandboxFileQueue.push(
718
+ this.createCustomEventChunk('sandbox.file', { ...event }),
719
+ )
720
+ },
721
+ })
704
722
  }
705
723
 
706
724
  /** Get the accumulated content after the chat loop completes */
@@ -1021,6 +1039,10 @@ class TextEngine<
1021
1039
  threadId: this.threadId,
1022
1040
  runId: this.runIdOverride,
1023
1041
  parentRunId: this.parentRunIdOverride,
1042
+ // Expose provided capabilities (e.g. sandbox) to harness adapters.
1043
+ capabilities: this.middlewareCtx,
1044
+ // Client approval decisions, for harness interactive-approval resolution.
1045
+ approvals: this.initialApprovals,
1024
1046
  ...(combinedSchema ? { outputSchema: combinedSchema } : {}),
1025
1047
  })) {
1026
1048
  if (this.isCancelled()) {
@@ -1105,10 +1127,16 @@ class TextEngine<
1105
1127
  await this.middlewareRunner.runOnUsage(this.middlewareCtx, chunk.usage)
1106
1128
  }
1107
1129
 
1130
+ // Drain any sandbox.file events emitted while processing this chunk.
1131
+ yield* this.drainSandboxFileQueue()
1132
+
1108
1133
  if (this.earlyTermination) {
1109
1134
  break
1110
1135
  }
1111
1136
  }
1137
+
1138
+ // Drain any remaining sandbox.file events emitted after the stream ended.
1139
+ yield* this.drainSandboxFileQueue()
1112
1140
  }
1113
1141
 
1114
1142
  private handleStreamChunk(chunk: StreamChunk): void {
@@ -2502,6 +2530,17 @@ class TextEngine<
2502
2530
  }
2503
2531
  }
2504
2532
 
2533
+ /**
2534
+ * Drain queued `sandbox.file` chunks (emitted via the SandboxRuntime sink)
2535
+ * through the middleware pipeline and into the public stream.
2536
+ */
2537
+ private async *drainSandboxFileQueue(): AsyncGenerator<StreamChunk> {
2538
+ while (this.sandboxFileQueue.length > 0) {
2539
+ const chunk = this.sandboxFileQueue.shift()
2540
+ if (chunk) yield* this.pipeThroughMiddleware(chunk)
2541
+ }
2542
+ }
2543
+
2505
2544
  /**
2506
2545
  * Drain an executeToolCalls async generator, yielding any CustomEvent chunks
2507
2546
  * through the middleware pipeline and returning the final ExecuteToolCallsResult.
@@ -1,6 +1,33 @@
1
1
  import type { ServerTool } from '../tools/tool-definition'
2
2
  import type { ChatMCPOptions, MCPToolSource } from './types'
3
3
 
4
+ /**
5
+ * Bind the source's `readResource` onto a ui-linked tool's `metadata.mcp` so it
6
+ * travels with the tool to the server-tool execution/emit site (`tool-calls.ts`).
7
+ *
8
+ * `discover()` is the single place in `@tanstack/ai` that has both a tool and
9
+ * its originating source, and `@tanstack/ai` must not import `@tanstack/ai-mcp`,
10
+ * so this is where the source handle is threaded onto the tool. Only tools that
11
+ * actually link a `ui://` resource (their discovery stamped
12
+ * `metadata.mcp.uiResourceUri`) and whose source can read resources get bound;
13
+ * everything else is left untouched.
14
+ *
15
+ * This mutates the discovered tool's `metadata.mcp` in place. That is safe
16
+ * because discovery returns fresh tool objects per `discover()` call: the bound
17
+ * `readResource` closes over `source`, whose connection stays live until the run
18
+ * drains. If discovery results were ever cached and reused across runs, this
19
+ * would bind a closure over an already-closed source — bind onto a copy then.
20
+ */
21
+ function bindReadResource(tool: ServerTool, source: MCPToolSource): void {
22
+ if (!source.readResource) return
23
+ const meta = (
24
+ tool.metadata as { mcp?: { uiResourceUri?: string } } | undefined
25
+ )?.mcp
26
+ if (!meta?.uiResourceUri) return
27
+ ;(meta as { readResource?: MCPToolSource['readResource'] }).readResource =
28
+ source.readResource.bind(source)
29
+ }
30
+
4
31
  export class MCPDuplicateToolNameError extends Error {
5
32
  constructor(public readonly toolName: string) {
6
33
  super(
@@ -57,7 +84,10 @@ export class MCPManager {
57
84
  for (const [source, result] of zipped) {
58
85
  if (result === undefined) continue
59
86
  if (result.status === 'fulfilled') {
60
- tools.push(...result.value)
87
+ for (const t of result.value) {
88
+ bindReadResource(t, source)
89
+ tools.push(t)
90
+ }
61
91
  } else if (this.#onDiscoveryError) {
62
92
  // throw/reject inside handler ⇒ propagate (fail-fast); return ⇒ skip
63
93
  await this.#onDiscoveryError(result.reason, source)
@@ -1,5 +1,20 @@
1
1
  import type { ServerTool } from '../tools/tool-definition'
2
2
 
3
+ /**
4
+ * The shape `readResource` resolves to — a structural subset of MCP's
5
+ * `ReadResourceResult`. Single source of truth shared by
6
+ * `MCPToolSource.readResource` (this file) and the tool-bound
7
+ * `McpToolAppMeta.readResource` (tool-calls.ts) so the two copies cannot drift.
8
+ */
9
+ export interface McpResourceReadResult {
10
+ contents: Array<{
11
+ uri: string
12
+ mimeType?: string
13
+ text?: string
14
+ blob?: string
15
+ }>
16
+ }
17
+
3
18
  /**
4
19
  * Minimal structural shape that `chat({ mcp })` needs from an MCP client.
5
20
  *
@@ -13,6 +28,14 @@ export interface MCPToolSource {
13
28
  // forwards what is declared here.
14
29
  tools: (options?: { lazy?: boolean }) => Promise<Array<ServerTool>>
15
30
  close: () => Promise<void>
31
+ /**
32
+ * Reads an MCP resource by URI. Used by the chat manager to eagerly fetch
33
+ * `ui://` resource widgets (MCP Apps) after a tool result resolves.
34
+ *
35
+ * Optional — sources that do not serve `ui://` resources need not implement
36
+ * this method. `ai-mcp`'s `MCPClient` satisfies this structurally.
37
+ */
38
+ readResource?: (uri: string) => Promise<McpResourceReadResult>
16
39
  }
17
40
 
18
41
  /**
@@ -322,6 +322,11 @@ function buildAssistantMessages(uiMessage: UIMessage): Array<ModelMessage> {
322
322
  }
323
323
  break
324
324
 
325
+ case 'ui-resource':
326
+ // MCP Apps widget — rendered client-side only. It must never enter
327
+ // model input, so it is intentionally dropped from the model message.
328
+ break
329
+
325
330
  default:
326
331
  break
327
332
  }
@@ -11,6 +11,7 @@ import type {
11
11
  ErrorInfo,
12
12
  FinishInfo,
13
13
  IterationInfo,
14
+ SandboxFileEvent,
14
15
  StructuredOutputMiddlewareConfig,
15
16
  ToolCallHookContext,
16
17
  ToolPhaseCompleteInfo,
@@ -344,6 +345,39 @@ export class MiddlewareRunner<TContext = unknown> {
344
345
  return chunks
345
346
  }
346
347
 
348
+ /**
349
+ * Dispatch a sandbox file event to every middleware's `sandbox` hooks, in
350
+ * array order: the catch-all `onFile` then the type-specific hook. Errors are
351
+ * logged and swallowed so one bad hook can't break the run.
352
+ */
353
+ async runSandboxFile(
354
+ ctx: ChatMiddlewareContext<TContext>,
355
+ event: SandboxFileEvent,
356
+ ): Promise<void> {
357
+ const typed = (
358
+ {
359
+ create: 'onFileCreate',
360
+ change: 'onFileChange',
361
+ delete: 'onFileDelete',
362
+ } as const
363
+ )[event.type]
364
+ for (const mw of this.middlewares) {
365
+ const hooks = mw.sandbox
366
+ if (!hooks) continue
367
+ for (const fn of [hooks.onFile, hooks[typed]]) {
368
+ if (!fn) continue
369
+ try {
370
+ await fn(ctx, event)
371
+ } catch (error) {
372
+ this.logger.sandbox(
373
+ `hook=${typed} middleware=${mw.name ?? 'unnamed'} threw`,
374
+ { middleware: mw.name ?? 'unnamed', error },
375
+ )
376
+ }
377
+ }
378
+ }
379
+ }
380
+
347
381
  /**
348
382
  * Run onBeforeToolCall through middleware in order.
349
383
  * Returns the first non-void decision, or undefined to continue normally.
@@ -13,6 +13,8 @@ export type {
13
13
  FinishInfo,
14
14
  AbortInfo,
15
15
  ErrorInfo,
16
+ SandboxFileEvent,
17
+ ChatSandboxHooks,
16
18
  } from './types'
17
19
 
18
20
  export { MiddlewareRunner } from './compose'
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Internal runtime seam the chat engine PROVIDES so the sandbox middleware can
3
+ * surface file events without a public ctx method. `emit` runs every
4
+ * middleware's `sandbox` hooks AND emits a CUSTOM `sandbox.file` chunk into the
5
+ * stream; `logger` lets the sandbox layer log under the `sandbox` debug
6
+ * category. Consumed (optionally) by `withSandbox` in `@tanstack/ai-sandbox`.
7
+ */
8
+ import { createCapability } from './capabilities'
9
+ import type { InternalLogger } from '../../../logger/internal-logger'
10
+ import type { SandboxFileEvent } from './types'
11
+
12
+ export interface SandboxRuntime {
13
+ emit: (event: SandboxFileEvent) => void
14
+ logger: InternalLogger
15
+ }
16
+
17
+ export const SandboxRuntimeCapability =
18
+ createCapability<SandboxRuntime>()('sandbox-runtime')
19
+
20
+ export const [getSandboxRuntime, provideSandboxRuntime] =
21
+ SandboxRuntimeCapability
@@ -13,6 +13,37 @@ import type {
13
13
  CapabilityRegistry,
14
14
  } from './capabilities'
15
15
 
16
+ /** A file change observed inside a sandbox during a chat run. */
17
+ export interface SandboxFileEvent {
18
+ type: 'create' | 'change' | 'delete'
19
+ /** Absolute path inside the sandbox (under the workspace root). */
20
+ path: string
21
+ timestamp: number
22
+ }
23
+
24
+ /**
25
+ * Sandbox file-event hooks a chat middleware can declare. Fire server-side for
26
+ * every file create/change/delete observed in the sandbox during the run.
27
+ */
28
+ export interface ChatSandboxHooks<TContext = unknown> {
29
+ onFile?: (
30
+ ctx: ChatMiddlewareContext<TContext>,
31
+ e: SandboxFileEvent,
32
+ ) => void | Promise<void>
33
+ onFileCreate?: (
34
+ ctx: ChatMiddlewareContext<TContext>,
35
+ e: SandboxFileEvent,
36
+ ) => void | Promise<void>
37
+ onFileChange?: (
38
+ ctx: ChatMiddlewareContext<TContext>,
39
+ e: SandboxFileEvent,
40
+ ) => void | Promise<void>
41
+ onFileDelete?: (
42
+ ctx: ChatMiddlewareContext<TContext>,
43
+ e: SandboxFileEvent,
44
+ ) => void | Promise<void>
45
+ }
46
+
16
47
  // ===========================
17
48
  // Middleware Context
18
49
  // ===========================
@@ -539,6 +570,12 @@ export interface ChatMiddleware<TContext = unknown> {
539
570
  ctx: ChatMiddlewareContext<TContext>,
540
571
  info: ErrorInfo,
541
572
  ) => void | Promise<void>
573
+
574
+ /**
575
+ * Sandbox file-event hooks. Fire when a sandbox provided by `withSandbox` is
576
+ * active during the run and a file is created/changed/deleted. Server-side.
577
+ */
578
+ sandbox?: ChatSandboxHooks<TContext>
542
579
  }
543
580
 
544
581
  /** A `ChatMiddleware` with a permissive context — for use as a constraint. */
@@ -56,6 +56,8 @@ import type {
56
56
  ToolCallPart,
57
57
  ToolResultPart,
58
58
  UIMessage,
59
+ UIResourceEvent,
60
+ UIResourcePart,
59
61
  } from '../../../types'
60
62
 
61
63
  /**
@@ -1624,6 +1626,46 @@ export class StreamProcessor {
1624
1626
  return
1625
1627
  }
1626
1628
 
1629
+ // Handle MCP Apps ui-resource events — materialize a UIResourcePart on the
1630
+ // active assistant message. Never falls through to onCustomEvent because
1631
+ // ui-resource is a system event, not a user-defined custom event.
1632
+ if (chunk.name === 'ui-resource' && chunk.value) {
1633
+ const v: UIResourceEvent['value'] = chunk.value
1634
+ // Resolve the target assistant message. When a toolCallId is present, the
1635
+ // tool call's OWNER message is authoritative, so prefer it first; fall
1636
+ // back to the active assistant id only if the tool call isn't mapped.
1637
+ // This avoids misattaching the widget to a different active message in a
1638
+ // multi-message session.
1639
+ const resolvedMessageId =
1640
+ this.toolCallToMessage.get(v.toolCallId) ?? messageId
1641
+ if (resolvedMessageId) {
1642
+ const part: UIResourcePart = {
1643
+ type: 'ui-resource',
1644
+ resource: v.resource,
1645
+ toolCallId: v.toolCallId,
1646
+ toolName: v.toolName,
1647
+ ...(v.serverId !== undefined && { serverId: v.serverId }),
1648
+ ...(v.meta !== undefined && { meta: v.meta }),
1649
+ }
1650
+ this.messages = this.messages.map((msg) =>
1651
+ msg.id === resolvedMessageId
1652
+ ? { ...msg, parts: [...msg.parts, part] }
1653
+ : msg,
1654
+ )
1655
+ this.emitMessagesChange()
1656
+ } else {
1657
+ // No owner message and no active assistant id — the server read and
1658
+ // streamed a widget that has nowhere to attach (e.g. a toolCallId never
1659
+ // registered, or the event arrived after the run cleared its active
1660
+ // ids). Drop fail-soft, but warn: a vanished widget is otherwise
1661
+ // undebuggable from the client.
1662
+ console.warn(
1663
+ `[mcp-apps] dropped ui-resource: no target message for toolCallId "${v.toolCallId}" (toolName "${v.toolName}")`,
1664
+ )
1665
+ }
1666
+ return
1667
+ }
1668
+
1627
1669
  // Forward non-system custom events to onCustomEvent callback
1628
1670
  if (this.events.onCustomEvent) {
1629
1671
  const toolCallId =
@@ -18,6 +18,7 @@ import type {
18
18
  AfterToolCallInfo,
19
19
  BeforeToolCallDecision,
20
20
  } from '../middleware/types'
21
+ import type { McpResourceReadResult } from '../mcp/types'
21
22
  import type {
22
23
  ContextFromTool,
23
24
  DefinedContext,
@@ -33,6 +34,92 @@ function safeJsonParse(value: string): unknown {
33
34
  }
34
35
  }
35
36
 
37
+ /**
38
+ * MCP Apps metadata attached to a server tool at discovery (see
39
+ * `@tanstack/ai-mcp` discovery + `MCPManager.discover()`).
40
+ *
41
+ * - `uiResourceUri` / `serverId` are stamped by ai-mcp at tool discovery.
42
+ * - `readResource` is bound by `MCPManager.discover()` (the one site that has
43
+ * both the tool and its originating source) so the resource can be eagerly
44
+ * read at the emit site. Under `chat()`-managed MCP lifecycle
45
+ * (`connection:'close'`), the MCP source is not disposed until the run
46
+ * drains, so `readResource` is still live at this emit point. Note: a caller
47
+ * who closes the MCP source early (outside `chat()`'s managed lifecycle)
48
+ * degrades fail-soft — `readResource` may reject, the widget is absent, but
49
+ * the tool result still flows to the model.
50
+ * `@tanstack/ai` never imports `@tanstack/ai-mcp`; this travels structurally
51
+ * on the tool.
52
+ */
53
+ interface McpToolAppMeta {
54
+ uiResourceUri?: string
55
+ serverId?: string
56
+ /** Server-native (unprefixed) MCP tool name — used as the renderer's toolName. */
57
+ serverToolName?: string
58
+ readResource?: (uri: string) => Promise<McpResourceReadResult>
59
+ }
60
+
61
+ function readMcpAppMeta(tool: AnyTool): McpToolAppMeta | undefined {
62
+ const meta = (tool.metadata as { mcp?: McpToolAppMeta } | undefined)?.mcp
63
+ return meta
64
+ }
65
+
66
+ /**
67
+ * Eagerly read a tool's linked `ui://` resource (MCP Apps) and emit a
68
+ * `ui-resource` CUSTOM event so the client can render the widget. The model
69
+ * still receives the normal text tool-result; the widget rides alongside and
70
+ * never enters model input.
71
+ *
72
+ * Fail-soft: any read error logs a warning and emits nothing — it never throws,
73
+ * so the normal tool-result still flows and a broken widget cannot break the run.
74
+ */
75
+ async function emitUiResourceIfLinked<TContext>(
76
+ tool: AnyTool,
77
+ context: ToolExecutionContext<TContext>,
78
+ ): Promise<void> {
79
+ const mcp = readMcpAppMeta(tool)
80
+ const uiUri = mcp?.uiResourceUri
81
+ if (!uiUri || !mcp.readResource) return
82
+
83
+ // The try covers ONLY the fallible read — keep `emitCustomEvent` out of it so
84
+ // an exception from the emit path can't be mislabeled as a read failure.
85
+ let matched: McpResourceReadResult['contents'][number] | undefined
86
+ try {
87
+ const res = await mcp.readResource(uiUri)
88
+ // Emit ONLY the content whose uri matches the requested `uiUri`. A source
89
+ // can return unrelated contents; falling back to `contents[0]` would risk
90
+ // rendering a widget that doesn't correspond to the linked resource. This
91
+ // is a display widget — a mismatched resource is worse than none, so if no
92
+ // content matches we fail-soft (warn + return) rather than emit.
93
+ matched = res.contents.find((c) => c.uri === uiUri)
94
+ } catch (err) {
95
+ // fail-soft — the text tool-result already flows; a broken widget must
96
+ // not break the run.
97
+ console.warn(`[mcp-apps] failed to read ui resource ${uiUri}:`, err)
98
+ return
99
+ }
100
+ if (!matched) {
101
+ console.warn(
102
+ `[mcp-apps] ui resource ${uiUri} returned no content matching that uri; not emitting`,
103
+ )
104
+ return
105
+ }
106
+ // NOTE: `toolCallId` is intentionally NOT set here — it is stamped onto
107
+ // every emitted event by the `executeToolCalls` context wrapper, so the
108
+ // UIResourceEvent.value.toolCallId / UIResourcePart.toolCallId contract is
109
+ // still satisfied downstream.
110
+ context.emitCustomEvent('ui-resource', {
111
+ resource: {
112
+ uri: matched.uri,
113
+ mimeType: matched.mimeType ?? 'text/html',
114
+ text: matched.text,
115
+ blob: matched.blob,
116
+ },
117
+ serverId: mcp.serverId,
118
+ toolName: mcp.serverToolName ?? tool.name,
119
+ meta: undefined,
120
+ })
121
+ }
122
+
36
123
  /**
37
124
  * Optional middleware hooks for tool execution.
38
125
  * When provided, these callbacks are invoked before/after each tool execution.
@@ -456,7 +543,7 @@ async function applyBeforeToolCallDecision(
456
543
  * Execute a server-side tool with event polling, output validation, and middleware hooks.
457
544
  * Yields CustomEvent chunks during execution and pushes the result to the results array.
458
545
  */
459
- async function* executeServerTool<TContext = unknown>(
546
+ export async function* executeServerTool<TContext = unknown>(
460
547
  toolCall: ToolCall,
461
548
  tool: AnyTool,
462
549
  toolName: string,
@@ -475,7 +562,14 @@ async function* executeServerTool<TContext = unknown>(
475
562
  let result = yield* executeWithEventPolling(executionPromise, pendingEvents)
476
563
  const duration = Date.now() - startTime
477
564
 
478
- // Flush remaining events
565
+ // MCP Apps: if this tool links a ui:// resource, eagerly read it and queue
566
+ // a `ui-resource` CUSTOM event. The MCP source stays live until the run
567
+ // drains (MCPManager's `connection:'close'` policy disposes on completion),
568
+ // so `readResource` is callable here. Fail-soft: a read error warns and
569
+ // emits nothing — the text result still flows.
570
+ await emitUiResourceIfLinked(tool, context)
571
+
572
+ // Flush remaining events (including any queued ui-resource event)
479
573
  let pendingEvent: CustomEvent | undefined
480
574
  while ((pendingEvent = pendingEvents.shift()) !== undefined) {
481
575
  yield pendingEvent
@@ -10,3 +10,9 @@ export {
10
10
  toRunErrorPayload,
11
11
  toRunErrorRawEvent,
12
12
  } from './activities/error-payload'
13
+ export {
14
+ getSandboxRuntime,
15
+ provideSandboxRuntime,
16
+ SandboxRuntimeCapability,
17
+ } from './activities/chat/middleware/sandbox-runtime'
18
+ export type { SandboxRuntime } from './activities/chat/middleware/sandbox-runtime'
package/src/client.ts CHANGED
@@ -114,6 +114,7 @@ export type {
114
114
  ToolCallPart,
115
115
  ToolResultPart,
116
116
  UIMessage,
117
+ UIResourcePart,
117
118
  VideoPart,
118
119
  InferSchemaType,
119
120
  } from './types'
package/src/index.ts CHANGED
@@ -120,6 +120,8 @@ export type {
120
120
  FinishInfo,
121
121
  AbortInfo,
122
122
  ErrorInfo,
123
+ SandboxFileEvent,
124
+ ChatSandboxHooks,
123
125
  } from './activities/chat/middleware/index'
124
126
 
125
127
  // Base, activity-agnostic middleware. The observe-only superset that media
@@ -149,6 +151,8 @@ export type {
149
151
  CapabilityContext,
150
152
  CapabilityGetter,
151
153
  CapabilityProvider,
154
+ DefinedChatMiddleware,
155
+ AnyChatMiddleware,
152
156
  } from './activities/chat/middleware/index'
153
157
 
154
158
  // All types
@@ -31,6 +31,7 @@ const CATEGORY_EMOJI: Record<keyof ResolvedCategories, string> = {
31
31
  agentLoop: '🔁',
32
32
  config: '⚙️',
33
33
  errors: '❌',
34
+ sandbox: '📦',
34
35
  }
35
36
 
36
37
  export class InternalLogger {
@@ -82,6 +83,11 @@ export class InternalLogger {
82
83
  this.emit('debug', 'tools', message, meta)
83
84
  }
84
85
 
86
+ /** Log sandbox internals (watcher, file events, hook dispatch). Chat-only. */
87
+ sandbox(message: string, meta?: Record<string, unknown>): void {
88
+ this.emit('debug', 'sandbox', message, meta)
89
+ }
90
+
85
91
  /** Log an agent-loop iteration marker or phase transition. Chat-only. */
86
92
  agentLoop(message: string, meta?: Record<string, unknown>): void {
87
93
  this.emit('debug', 'agentLoop', message, meta)