@tanstack/ai 0.37.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/activities/chat/index.js +28 -0
- package/dist/esm/activities/chat/index.js.map +1 -1
- package/dist/esm/activities/chat/mcp/manager.js +10 -1
- package/dist/esm/activities/chat/mcp/manager.js.map +1 -1
- package/dist/esm/activities/chat/mcp/types.d.ts +22 -0
- package/dist/esm/activities/chat/messages.js.map +1 -1
- package/dist/esm/activities/chat/middleware/compose.d.ts +7 -1
- package/dist/esm/activities/chat/middleware/compose.js +27 -0
- package/dist/esm/activities/chat/middleware/compose.js.map +1 -1
- package/dist/esm/activities/chat/middleware/index.d.ts +1 -1
- package/dist/esm/activities/chat/middleware/sandbox-runtime.d.ts +8 -0
- package/dist/esm/activities/chat/middleware/sandbox-runtime.js +9 -0
- package/dist/esm/activities/chat/middleware/sandbox-runtime.js.map +1 -0
- package/dist/esm/activities/chat/middleware/types.d.ts +22 -0
- package/dist/esm/activities/chat/stream/processor.js +23 -0
- package/dist/esm/activities/chat/stream/processor.js.map +1 -1
- package/dist/esm/activities/chat/tools/tool-calls.d.ts +6 -1
- package/dist/esm/activities/chat/tools/tool-calls.js +36 -0
- package/dist/esm/activities/chat/tools/tool-calls.js.map +1 -1
- package/dist/esm/adapter-internals.d.ts +2 -0
- package/dist/esm/adapter-internals.js +4 -0
- package/dist/esm/adapter-internals.js.map +1 -1
- package/dist/esm/client.d.ts +1 -1
- package/dist/esm/client.js.map +1 -1
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/logger/internal-logger.d.ts +2 -0
- package/dist/esm/logger/internal-logger.js +6 -1
- package/dist/esm/logger/internal-logger.js.map +1 -1
- package/dist/esm/logger/resolve.js +6 -3
- package/dist/esm/logger/resolve.js.map +1 -1
- package/dist/esm/logger/types.d.ts +5 -0
- package/dist/esm/types.d.ts +52 -1
- package/package.json +1 -1
- package/skills/ai-core/adapter-configuration/SKILL.md +25 -12
- package/skills/ai-core/chat-experience/SKILL.md +30 -2
- package/skills/ai-core/media-generation/SKILL.md +13 -0
- package/skills/ai-core/tool-calling/SKILL.md +10 -3
- package/src/activities/chat/index.ts +39 -0
- package/src/activities/chat/mcp/manager.ts +31 -1
- package/src/activities/chat/mcp/types.ts +23 -0
- package/src/activities/chat/messages.ts +5 -0
- package/src/activities/chat/middleware/compose.ts +34 -0
- package/src/activities/chat/middleware/index.ts +2 -0
- package/src/activities/chat/middleware/sandbox-runtime.ts +21 -0
- package/src/activities/chat/middleware/types.ts +37 -0
- package/src/activities/chat/stream/processor.ts +42 -0
- package/src/activities/chat/tools/tool-calls.ts +96 -2
- package/src/adapter-internals.ts +6 -0
- package/src/client.ts +1 -0
- package/src/index.ts +4 -0
- package/src/logger/internal-logger.ts +6 -0
- package/src/logger/resolve.ts +3 -0
- package/src/logger/types.ts +5 -0
- package/src/types.ts +51 -0
|
@@ -278,7 +278,35 @@ if (part.type === 'image') {
|
|
|
278
278
|
}
|
|
279
279
|
```
|
|
280
280
|
|
|
281
|
-
### 4.
|
|
281
|
+
### 4. Sending Audio Messages (Browser Recording)
|
|
282
|
+
|
|
283
|
+
Use `useAudioRecorder` from `@tanstack/ai-react` (or `createAudioRecorder` in Svelte) to capture audio in the browser. The resolved `AudioRecording` includes a ready-to-use `part` that slots directly into `sendMessage`.
|
|
284
|
+
|
|
285
|
+
```typescript
|
|
286
|
+
import {
|
|
287
|
+
useAudioRecorder,
|
|
288
|
+
useChat,
|
|
289
|
+
fetchServerSentEvents,
|
|
290
|
+
} from '@tanstack/ai-react'
|
|
291
|
+
|
|
292
|
+
const { isRecording, isSupported, start, stop } = useAudioRecorder()
|
|
293
|
+
const { sendMessage } = useChat({
|
|
294
|
+
connection: fetchServerSentEvents('/api/chat'),
|
|
295
|
+
})
|
|
296
|
+
|
|
297
|
+
async function toggle() {
|
|
298
|
+
if (!isRecording) {
|
|
299
|
+
await start()
|
|
300
|
+
return
|
|
301
|
+
}
|
|
302
|
+
const recording = await stop()
|
|
303
|
+
await sendMessage({ content: [recording.part] })
|
|
304
|
+
}
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
`recording.part` is `{ type: 'audio', source: { type: 'data', value: base64, mimeType } }`. Returns the recorder's native format (`audio/webm` or `audio/mp4`) with no transcoding.
|
|
308
|
+
|
|
309
|
+
### 5. HTTP Stream Format (Alternative to SSE)
|
|
282
310
|
|
|
283
311
|
Use `toHttpResponse` + `fetchHttpStream` for newline-delimited JSON instead of SSE.
|
|
284
312
|
|
|
@@ -310,7 +338,7 @@ const { messages, sendMessage } = useChat({
|
|
|
310
338
|
The only difference is swapping `toServerSentEventsResponse` / `fetchServerSentEvents`
|
|
311
339
|
for `toHttpResponse` / `fetchHttpStream`. Everything else stays identical.
|
|
312
340
|
|
|
313
|
-
###
|
|
341
|
+
### 6. MCP Tool Discovery via `chat({ mcp })`
|
|
314
342
|
|
|
315
343
|
Pass `mcp` to let `chat()` own discovery **and** lifecycle for one or more MCP
|
|
316
344
|
clients. Useful when you want minimal boilerplate and don't need to reuse the
|
|
@@ -359,6 +359,19 @@ const { generate, result, isLoading } = useGenerateSpeech({
|
|
|
359
359
|
Adapter: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
|
|
360
360
|
gpt-4o-mini-transcribe).
|
|
361
361
|
|
|
362
|
+
> **Capturing audio in the browser:** Use `useAudioRecorder` from `@tanstack/ai-react` to record directly in the browser, then pass the recording as the `audio` input to `generate()`, or use `recording.part` as a prompt part in chat/generation calls. No transcoding or extra dependencies required — the recorder returns the native browser format (`audio/webm` or `audio/mp4`). For transcription, wrap it as a `data:` URL so the provider gets the real content type; passing raw `recording.base64` makes the adapter assume `audio/mpeg` and mislabel the webm/mp4 bytes.
|
|
363
|
+
>
|
|
364
|
+
> ```typescript
|
|
365
|
+
> const { isRecording, start, stop } = useAudioRecorder()
|
|
366
|
+
> const { generate } = useTranscription({
|
|
367
|
+
> connection: fetchServerSentEvents('/api/transcribe'),
|
|
368
|
+
> })
|
|
369
|
+
> // ...
|
|
370
|
+
> const recording = await stop()
|
|
371
|
+
> const mimeType = recording.mimeType.split(';')[0] // strip ;codecs=...
|
|
372
|
+
> await generate({ audio: `data:${mimeType};base64,${recording.base64}` })
|
|
373
|
+
> ```
|
|
374
|
+
|
|
362
375
|
```typescript
|
|
363
376
|
import { generateTranscription } from '@tanstack/ai'
|
|
364
377
|
import { openaiTranscription } from '@tanstack/ai-openai'
|
|
@@ -75,7 +75,7 @@ import { updateCartUIDef } from '@/tools/definitions'
|
|
|
75
75
|
export async function POST(request: Request) {
|
|
76
76
|
const { messages } = await request.json()
|
|
77
77
|
const stream = chat({
|
|
78
|
-
adapter: openaiText('gpt-
|
|
78
|
+
adapter: openaiText('gpt-5.5'),
|
|
79
79
|
messages,
|
|
80
80
|
tools: [getProducts, updateCartUIDef], // server tool + client definition
|
|
81
81
|
})
|
|
@@ -158,7 +158,7 @@ const getUserData = getUserDataDef.server(async ({ userId }) => {
|
|
|
158
158
|
|
|
159
159
|
// In your route handler:
|
|
160
160
|
const stream = chat({
|
|
161
|
-
adapter: openaiText('gpt-
|
|
161
|
+
adapter: openaiText('gpt-5.5'),
|
|
162
162
|
messages,
|
|
163
163
|
tools: [getUserData],
|
|
164
164
|
})
|
|
@@ -188,7 +188,7 @@ Server -- pass definition only (no execute function):
|
|
|
188
188
|
|
|
189
189
|
```typescript
|
|
190
190
|
const stream = chat({
|
|
191
|
-
adapter: openaiText('gpt-
|
|
191
|
+
adapter: openaiText('gpt-5.5'),
|
|
192
192
|
messages,
|
|
193
193
|
tools: [showNotificationDef],
|
|
194
194
|
})
|
|
@@ -405,6 +405,13 @@ The post-discovery payload always returns the full description and schema regard
|
|
|
405
405
|
`@tanstack/ai-mcp` lets a server-side `chat()` call discover and invoke tools
|
|
406
406
|
hosted on any MCP server (Streamable HTTP, SSE, or stdio).
|
|
407
407
|
|
|
408
|
+
**MCP tools and UI resources:** When an MCP tool result carries a `ui://`
|
|
409
|
+
resource URI (via `_meta.ui.resourceUri`), TanStack AI surfaces it as a
|
|
410
|
+
`UIResourcePart` on the assistant `UIMessage` in the client message list.
|
|
411
|
+
`UIResourcePart` is a presentational-only part — it never enters model input.
|
|
412
|
+
See the `@tanstack/ai-mcp` skill for the full MCP Apps API
|
|
413
|
+
(`createMcpAppCallHandler`, `createMcpAppBridge`, `MCPAppResource`).
|
|
414
|
+
|
|
408
415
|
### Basic usage — auto-discovery
|
|
409
416
|
|
|
410
417
|
```typescript
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies'
|
|
28
28
|
import { convertMessagesToModelMessages, generateMessageId } from './messages'
|
|
29
29
|
import { MiddlewareRunner } from './middleware/compose'
|
|
30
|
+
import { provideSandboxRuntime } from './middleware/sandbox-runtime'
|
|
30
31
|
import { CapabilityRegistry } from './middleware/capabilities'
|
|
31
32
|
import { validateCapabilities } from './middleware/validate'
|
|
32
33
|
import { MCPManager } from './mcp/manager'
|
|
@@ -67,6 +68,7 @@ import type {
|
|
|
67
68
|
ChatMiddleware,
|
|
68
69
|
ChatMiddlewareConfig,
|
|
69
70
|
ChatMiddlewareContext,
|
|
71
|
+
SandboxFileEvent,
|
|
70
72
|
StructuredOutputMiddlewareConfig,
|
|
71
73
|
} from './middleware/types'
|
|
72
74
|
import type { CheckCoverage } from './middleware/builder'
|
|
@@ -542,6 +544,7 @@ class TextEngine<
|
|
|
542
544
|
// Middleware support
|
|
543
545
|
private readonly middlewareRunner: MiddlewareRunner<TContext>
|
|
544
546
|
private readonly middlewareCtx: ChatMiddlewareContext<TContext>
|
|
547
|
+
private readonly sandboxFileQueue: Array<StreamChunk> = []
|
|
545
548
|
private readonly deferredPromises: Array<Promise<unknown>> = []
|
|
546
549
|
private abortReason?: string
|
|
547
550
|
private readonly middlewareAbortController?: AbortController
|
|
@@ -701,6 +704,21 @@ class TextEngine<
|
|
|
701
704
|
capability[0](this.middlewareCtx, { optional: true }),
|
|
702
705
|
provide: (capability, value) => capability[1](this.middlewareCtx, value),
|
|
703
706
|
}
|
|
707
|
+
|
|
708
|
+
// Provide the internal SandboxRuntime capability so harness adapters and
|
|
709
|
+
// sandbox middleware can emit file events. The sink logs, fans the event
|
|
710
|
+
// out through the middleware `onFile*` hooks (fire-and-forget), and queues
|
|
711
|
+
// a `sandbox.file` custom chunk to be drained into the public stream.
|
|
712
|
+
provideSandboxRuntime(this.middlewareCtx, {
|
|
713
|
+
logger: this.logger,
|
|
714
|
+
emit: (event: SandboxFileEvent) => {
|
|
715
|
+
this.logger.sandbox(`file ${event.type} ${event.path}`, { event })
|
|
716
|
+
void this.middlewareRunner.runSandboxFile(this.middlewareCtx, event)
|
|
717
|
+
this.sandboxFileQueue.push(
|
|
718
|
+
this.createCustomEventChunk('sandbox.file', { ...event }),
|
|
719
|
+
)
|
|
720
|
+
},
|
|
721
|
+
})
|
|
704
722
|
}
|
|
705
723
|
|
|
706
724
|
/** Get the accumulated content after the chat loop completes */
|
|
@@ -1021,6 +1039,10 @@ class TextEngine<
|
|
|
1021
1039
|
threadId: this.threadId,
|
|
1022
1040
|
runId: this.runIdOverride,
|
|
1023
1041
|
parentRunId: this.parentRunIdOverride,
|
|
1042
|
+
// Expose provided capabilities (e.g. sandbox) to harness adapters.
|
|
1043
|
+
capabilities: this.middlewareCtx,
|
|
1044
|
+
// Client approval decisions, for harness interactive-approval resolution.
|
|
1045
|
+
approvals: this.initialApprovals,
|
|
1024
1046
|
...(combinedSchema ? { outputSchema: combinedSchema } : {}),
|
|
1025
1047
|
})) {
|
|
1026
1048
|
if (this.isCancelled()) {
|
|
@@ -1105,10 +1127,16 @@ class TextEngine<
|
|
|
1105
1127
|
await this.middlewareRunner.runOnUsage(this.middlewareCtx, chunk.usage)
|
|
1106
1128
|
}
|
|
1107
1129
|
|
|
1130
|
+
// Drain any sandbox.file events emitted while processing this chunk.
|
|
1131
|
+
yield* this.drainSandboxFileQueue()
|
|
1132
|
+
|
|
1108
1133
|
if (this.earlyTermination) {
|
|
1109
1134
|
break
|
|
1110
1135
|
}
|
|
1111
1136
|
}
|
|
1137
|
+
|
|
1138
|
+
// Drain any remaining sandbox.file events emitted after the stream ended.
|
|
1139
|
+
yield* this.drainSandboxFileQueue()
|
|
1112
1140
|
}
|
|
1113
1141
|
|
|
1114
1142
|
private handleStreamChunk(chunk: StreamChunk): void {
|
|
@@ -2502,6 +2530,17 @@ class TextEngine<
|
|
|
2502
2530
|
}
|
|
2503
2531
|
}
|
|
2504
2532
|
|
|
2533
|
+
/**
|
|
2534
|
+
* Drain queued `sandbox.file` chunks (emitted via the SandboxRuntime sink)
|
|
2535
|
+
* through the middleware pipeline and into the public stream.
|
|
2536
|
+
*/
|
|
2537
|
+
private async *drainSandboxFileQueue(): AsyncGenerator<StreamChunk> {
|
|
2538
|
+
while (this.sandboxFileQueue.length > 0) {
|
|
2539
|
+
const chunk = this.sandboxFileQueue.shift()
|
|
2540
|
+
if (chunk) yield* this.pipeThroughMiddleware(chunk)
|
|
2541
|
+
}
|
|
2542
|
+
}
|
|
2543
|
+
|
|
2505
2544
|
/**
|
|
2506
2545
|
* Drain an executeToolCalls async generator, yielding any CustomEvent chunks
|
|
2507
2546
|
* through the middleware pipeline and returning the final ExecuteToolCallsResult.
|
|
@@ -1,6 +1,33 @@
|
|
|
1
1
|
import type { ServerTool } from '../tools/tool-definition'
|
|
2
2
|
import type { ChatMCPOptions, MCPToolSource } from './types'
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* Bind the source's `readResource` onto a ui-linked tool's `metadata.mcp` so it
|
|
6
|
+
* travels with the tool to the server-tool execution/emit site (`tool-calls.ts`).
|
|
7
|
+
*
|
|
8
|
+
* `discover()` is the single place in `@tanstack/ai` that has both a tool and
|
|
9
|
+
* its originating source, and `@tanstack/ai` must not import `@tanstack/ai-mcp`,
|
|
10
|
+
* so this is where the source handle is threaded onto the tool. Only tools that
|
|
11
|
+
* actually link a `ui://` resource (their discovery stamped
|
|
12
|
+
* `metadata.mcp.uiResourceUri`) and whose source can read resources get bound;
|
|
13
|
+
* everything else is left untouched.
|
|
14
|
+
*
|
|
15
|
+
* This mutates the discovered tool's `metadata.mcp` in place. That is safe
|
|
16
|
+
* because discovery returns fresh tool objects per `discover()` call: the bound
|
|
17
|
+
* `readResource` closes over `source`, whose connection stays live until the run
|
|
18
|
+
* drains. If discovery results were ever cached and reused across runs, this
|
|
19
|
+
* would bind a closure over an already-closed source — bind onto a copy then.
|
|
20
|
+
*/
|
|
21
|
+
function bindReadResource(tool: ServerTool, source: MCPToolSource): void {
|
|
22
|
+
if (!source.readResource) return
|
|
23
|
+
const meta = (
|
|
24
|
+
tool.metadata as { mcp?: { uiResourceUri?: string } } | undefined
|
|
25
|
+
)?.mcp
|
|
26
|
+
if (!meta?.uiResourceUri) return
|
|
27
|
+
;(meta as { readResource?: MCPToolSource['readResource'] }).readResource =
|
|
28
|
+
source.readResource.bind(source)
|
|
29
|
+
}
|
|
30
|
+
|
|
4
31
|
export class MCPDuplicateToolNameError extends Error {
|
|
5
32
|
constructor(public readonly toolName: string) {
|
|
6
33
|
super(
|
|
@@ -57,7 +84,10 @@ export class MCPManager {
|
|
|
57
84
|
for (const [source, result] of zipped) {
|
|
58
85
|
if (result === undefined) continue
|
|
59
86
|
if (result.status === 'fulfilled') {
|
|
60
|
-
|
|
87
|
+
for (const t of result.value) {
|
|
88
|
+
bindReadResource(t, source)
|
|
89
|
+
tools.push(t)
|
|
90
|
+
}
|
|
61
91
|
} else if (this.#onDiscoveryError) {
|
|
62
92
|
// throw/reject inside handler ⇒ propagate (fail-fast); return ⇒ skip
|
|
63
93
|
await this.#onDiscoveryError(result.reason, source)
|
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
import type { ServerTool } from '../tools/tool-definition'
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* The shape `readResource` resolves to — a structural subset of MCP's
|
|
5
|
+
* `ReadResourceResult`. Single source of truth shared by
|
|
6
|
+
* `MCPToolSource.readResource` (this file) and the tool-bound
|
|
7
|
+
* `McpToolAppMeta.readResource` (tool-calls.ts) so the two copies cannot drift.
|
|
8
|
+
*/
|
|
9
|
+
export interface McpResourceReadResult {
|
|
10
|
+
contents: Array<{
|
|
11
|
+
uri: string
|
|
12
|
+
mimeType?: string
|
|
13
|
+
text?: string
|
|
14
|
+
blob?: string
|
|
15
|
+
}>
|
|
16
|
+
}
|
|
17
|
+
|
|
3
18
|
/**
|
|
4
19
|
* Minimal structural shape that `chat({ mcp })` needs from an MCP client.
|
|
5
20
|
*
|
|
@@ -13,6 +28,14 @@ export interface MCPToolSource {
|
|
|
13
28
|
// forwards what is declared here.
|
|
14
29
|
tools: (options?: { lazy?: boolean }) => Promise<Array<ServerTool>>
|
|
15
30
|
close: () => Promise<void>
|
|
31
|
+
/**
|
|
32
|
+
* Reads an MCP resource by URI. Used by the chat manager to eagerly fetch
|
|
33
|
+
* `ui://` resource widgets (MCP Apps) after a tool result resolves.
|
|
34
|
+
*
|
|
35
|
+
* Optional — sources that do not serve `ui://` resources need not implement
|
|
36
|
+
* this method. `ai-mcp`'s `MCPClient` satisfies this structurally.
|
|
37
|
+
*/
|
|
38
|
+
readResource?: (uri: string) => Promise<McpResourceReadResult>
|
|
16
39
|
}
|
|
17
40
|
|
|
18
41
|
/**
|
|
@@ -322,6 +322,11 @@ function buildAssistantMessages(uiMessage: UIMessage): Array<ModelMessage> {
|
|
|
322
322
|
}
|
|
323
323
|
break
|
|
324
324
|
|
|
325
|
+
case 'ui-resource':
|
|
326
|
+
// MCP Apps widget — rendered client-side only. It must never enter
|
|
327
|
+
// model input, so it is intentionally dropped from the model message.
|
|
328
|
+
break
|
|
329
|
+
|
|
325
330
|
default:
|
|
326
331
|
break
|
|
327
332
|
}
|
|
@@ -11,6 +11,7 @@ import type {
|
|
|
11
11
|
ErrorInfo,
|
|
12
12
|
FinishInfo,
|
|
13
13
|
IterationInfo,
|
|
14
|
+
SandboxFileEvent,
|
|
14
15
|
StructuredOutputMiddlewareConfig,
|
|
15
16
|
ToolCallHookContext,
|
|
16
17
|
ToolPhaseCompleteInfo,
|
|
@@ -344,6 +345,39 @@ export class MiddlewareRunner<TContext = unknown> {
|
|
|
344
345
|
return chunks
|
|
345
346
|
}
|
|
346
347
|
|
|
348
|
+
/**
|
|
349
|
+
* Dispatch a sandbox file event to every middleware's `sandbox` hooks, in
|
|
350
|
+
* array order: the catch-all `onFile` then the type-specific hook. Errors are
|
|
351
|
+
* logged and swallowed so one bad hook can't break the run.
|
|
352
|
+
*/
|
|
353
|
+
async runSandboxFile(
|
|
354
|
+
ctx: ChatMiddlewareContext<TContext>,
|
|
355
|
+
event: SandboxFileEvent,
|
|
356
|
+
): Promise<void> {
|
|
357
|
+
const typed = (
|
|
358
|
+
{
|
|
359
|
+
create: 'onFileCreate',
|
|
360
|
+
change: 'onFileChange',
|
|
361
|
+
delete: 'onFileDelete',
|
|
362
|
+
} as const
|
|
363
|
+
)[event.type]
|
|
364
|
+
for (const mw of this.middlewares) {
|
|
365
|
+
const hooks = mw.sandbox
|
|
366
|
+
if (!hooks) continue
|
|
367
|
+
for (const fn of [hooks.onFile, hooks[typed]]) {
|
|
368
|
+
if (!fn) continue
|
|
369
|
+
try {
|
|
370
|
+
await fn(ctx, event)
|
|
371
|
+
} catch (error) {
|
|
372
|
+
this.logger.sandbox(
|
|
373
|
+
`hook=${typed} middleware=${mw.name ?? 'unnamed'} threw`,
|
|
374
|
+
{ middleware: mw.name ?? 'unnamed', error },
|
|
375
|
+
)
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
347
381
|
/**
|
|
348
382
|
* Run onBeforeToolCall through middleware in order.
|
|
349
383
|
* Returns the first non-void decision, or undefined to continue normally.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal runtime seam the chat engine PROVIDES so the sandbox middleware can
|
|
3
|
+
* surface file events without a public ctx method. `emit` runs every
|
|
4
|
+
* middleware's `sandbox` hooks AND emits a CUSTOM `sandbox.file` chunk into the
|
|
5
|
+
* stream; `logger` lets the sandbox layer log under the `sandbox` debug
|
|
6
|
+
* category. Consumed (optionally) by `withSandbox` in `@tanstack/ai-sandbox`.
|
|
7
|
+
*/
|
|
8
|
+
import { createCapability } from './capabilities'
|
|
9
|
+
import type { InternalLogger } from '../../../logger/internal-logger'
|
|
10
|
+
import type { SandboxFileEvent } from './types'
|
|
11
|
+
|
|
12
|
+
export interface SandboxRuntime {
|
|
13
|
+
emit: (event: SandboxFileEvent) => void
|
|
14
|
+
logger: InternalLogger
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const SandboxRuntimeCapability =
|
|
18
|
+
createCapability<SandboxRuntime>()('sandbox-runtime')
|
|
19
|
+
|
|
20
|
+
export const [getSandboxRuntime, provideSandboxRuntime] =
|
|
21
|
+
SandboxRuntimeCapability
|
|
@@ -13,6 +13,37 @@ import type {
|
|
|
13
13
|
CapabilityRegistry,
|
|
14
14
|
} from './capabilities'
|
|
15
15
|
|
|
16
|
+
/** A file change observed inside a sandbox during a chat run. */
|
|
17
|
+
export interface SandboxFileEvent {
|
|
18
|
+
type: 'create' | 'change' | 'delete'
|
|
19
|
+
/** Absolute path inside the sandbox (under the workspace root). */
|
|
20
|
+
path: string
|
|
21
|
+
timestamp: number
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Sandbox file-event hooks a chat middleware can declare. Fire server-side for
|
|
26
|
+
* every file create/change/delete observed in the sandbox during the run.
|
|
27
|
+
*/
|
|
28
|
+
export interface ChatSandboxHooks<TContext = unknown> {
|
|
29
|
+
onFile?: (
|
|
30
|
+
ctx: ChatMiddlewareContext<TContext>,
|
|
31
|
+
e: SandboxFileEvent,
|
|
32
|
+
) => void | Promise<void>
|
|
33
|
+
onFileCreate?: (
|
|
34
|
+
ctx: ChatMiddlewareContext<TContext>,
|
|
35
|
+
e: SandboxFileEvent,
|
|
36
|
+
) => void | Promise<void>
|
|
37
|
+
onFileChange?: (
|
|
38
|
+
ctx: ChatMiddlewareContext<TContext>,
|
|
39
|
+
e: SandboxFileEvent,
|
|
40
|
+
) => void | Promise<void>
|
|
41
|
+
onFileDelete?: (
|
|
42
|
+
ctx: ChatMiddlewareContext<TContext>,
|
|
43
|
+
e: SandboxFileEvent,
|
|
44
|
+
) => void | Promise<void>
|
|
45
|
+
}
|
|
46
|
+
|
|
16
47
|
// ===========================
|
|
17
48
|
// Middleware Context
|
|
18
49
|
// ===========================
|
|
@@ -539,6 +570,12 @@ export interface ChatMiddleware<TContext = unknown> {
|
|
|
539
570
|
ctx: ChatMiddlewareContext<TContext>,
|
|
540
571
|
info: ErrorInfo,
|
|
541
572
|
) => void | Promise<void>
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Sandbox file-event hooks. Fire when a sandbox provided by `withSandbox` is
|
|
576
|
+
* active during the run and a file is created/changed/deleted. Server-side.
|
|
577
|
+
*/
|
|
578
|
+
sandbox?: ChatSandboxHooks<TContext>
|
|
542
579
|
}
|
|
543
580
|
|
|
544
581
|
/** A `ChatMiddleware` with a permissive context — for use as a constraint. */
|
|
@@ -56,6 +56,8 @@ import type {
|
|
|
56
56
|
ToolCallPart,
|
|
57
57
|
ToolResultPart,
|
|
58
58
|
UIMessage,
|
|
59
|
+
UIResourceEvent,
|
|
60
|
+
UIResourcePart,
|
|
59
61
|
} from '../../../types'
|
|
60
62
|
|
|
61
63
|
/**
|
|
@@ -1624,6 +1626,46 @@ export class StreamProcessor {
|
|
|
1624
1626
|
return
|
|
1625
1627
|
}
|
|
1626
1628
|
|
|
1629
|
+
// Handle MCP Apps ui-resource events — materialize a UIResourcePart on the
|
|
1630
|
+
// active assistant message. Never falls through to onCustomEvent because
|
|
1631
|
+
// ui-resource is a system event, not a user-defined custom event.
|
|
1632
|
+
if (chunk.name === 'ui-resource' && chunk.value) {
|
|
1633
|
+
const v: UIResourceEvent['value'] = chunk.value
|
|
1634
|
+
// Resolve the target assistant message. When a toolCallId is present, the
|
|
1635
|
+
// tool call's OWNER message is authoritative, so prefer it first; fall
|
|
1636
|
+
// back to the active assistant id only if the tool call isn't mapped.
|
|
1637
|
+
// This avoids misattaching the widget to a different active message in a
|
|
1638
|
+
// multi-message session.
|
|
1639
|
+
const resolvedMessageId =
|
|
1640
|
+
this.toolCallToMessage.get(v.toolCallId) ?? messageId
|
|
1641
|
+
if (resolvedMessageId) {
|
|
1642
|
+
const part: UIResourcePart = {
|
|
1643
|
+
type: 'ui-resource',
|
|
1644
|
+
resource: v.resource,
|
|
1645
|
+
toolCallId: v.toolCallId,
|
|
1646
|
+
toolName: v.toolName,
|
|
1647
|
+
...(v.serverId !== undefined && { serverId: v.serverId }),
|
|
1648
|
+
...(v.meta !== undefined && { meta: v.meta }),
|
|
1649
|
+
}
|
|
1650
|
+
this.messages = this.messages.map((msg) =>
|
|
1651
|
+
msg.id === resolvedMessageId
|
|
1652
|
+
? { ...msg, parts: [...msg.parts, part] }
|
|
1653
|
+
: msg,
|
|
1654
|
+
)
|
|
1655
|
+
this.emitMessagesChange()
|
|
1656
|
+
} else {
|
|
1657
|
+
// No owner message and no active assistant id — the server read and
|
|
1658
|
+
// streamed a widget that has nowhere to attach (e.g. a toolCallId never
|
|
1659
|
+
// registered, or the event arrived after the run cleared its active
|
|
1660
|
+
// ids). Drop fail-soft, but warn: a vanished widget is otherwise
|
|
1661
|
+
// undebuggable from the client.
|
|
1662
|
+
console.warn(
|
|
1663
|
+
`[mcp-apps] dropped ui-resource: no target message for toolCallId "${v.toolCallId}" (toolName "${v.toolName}")`,
|
|
1664
|
+
)
|
|
1665
|
+
}
|
|
1666
|
+
return
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1627
1669
|
// Forward non-system custom events to onCustomEvent callback
|
|
1628
1670
|
if (this.events.onCustomEvent) {
|
|
1629
1671
|
const toolCallId =
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
AfterToolCallInfo,
|
|
19
19
|
BeforeToolCallDecision,
|
|
20
20
|
} from '../middleware/types'
|
|
21
|
+
import type { McpResourceReadResult } from '../mcp/types'
|
|
21
22
|
import type {
|
|
22
23
|
ContextFromTool,
|
|
23
24
|
DefinedContext,
|
|
@@ -33,6 +34,92 @@ function safeJsonParse(value: string): unknown {
|
|
|
33
34
|
}
|
|
34
35
|
}
|
|
35
36
|
|
|
37
|
+
/**
|
|
38
|
+
* MCP Apps metadata attached to a server tool at discovery (see
|
|
39
|
+
* `@tanstack/ai-mcp` discovery + `MCPManager.discover()`).
|
|
40
|
+
*
|
|
41
|
+
* - `uiResourceUri` / `serverId` are stamped by ai-mcp at tool discovery.
|
|
42
|
+
* - `readResource` is bound by `MCPManager.discover()` (the one site that has
|
|
43
|
+
* both the tool and its originating source) so the resource can be eagerly
|
|
44
|
+
* read at the emit site. Under `chat()`-managed MCP lifecycle
|
|
45
|
+
* (`connection:'close'`), the MCP source is not disposed until the run
|
|
46
|
+
* drains, so `readResource` is still live at this emit point. Note: a caller
|
|
47
|
+
* who closes the MCP source early (outside `chat()`'s managed lifecycle)
|
|
48
|
+
* degrades fail-soft — `readResource` may reject, the widget is absent, but
|
|
49
|
+
* the tool result still flows to the model.
|
|
50
|
+
* `@tanstack/ai` never imports `@tanstack/ai-mcp`; this travels structurally
|
|
51
|
+
* on the tool.
|
|
52
|
+
*/
|
|
53
|
+
interface McpToolAppMeta {
|
|
54
|
+
uiResourceUri?: string
|
|
55
|
+
serverId?: string
|
|
56
|
+
/** Server-native (unprefixed) MCP tool name — used as the renderer's toolName. */
|
|
57
|
+
serverToolName?: string
|
|
58
|
+
readResource?: (uri: string) => Promise<McpResourceReadResult>
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function readMcpAppMeta(tool: AnyTool): McpToolAppMeta | undefined {
|
|
62
|
+
const meta = (tool.metadata as { mcp?: McpToolAppMeta } | undefined)?.mcp
|
|
63
|
+
return meta
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Eagerly read a tool's linked `ui://` resource (MCP Apps) and emit a
|
|
68
|
+
* `ui-resource` CUSTOM event so the client can render the widget. The model
|
|
69
|
+
* still receives the normal text tool-result; the widget rides alongside and
|
|
70
|
+
* never enters model input.
|
|
71
|
+
*
|
|
72
|
+
* Fail-soft: any read error logs a warning and emits nothing — it never throws,
|
|
73
|
+
* so the normal tool-result still flows and a broken widget cannot break the run.
|
|
74
|
+
*/
|
|
75
|
+
async function emitUiResourceIfLinked<TContext>(
|
|
76
|
+
tool: AnyTool,
|
|
77
|
+
context: ToolExecutionContext<TContext>,
|
|
78
|
+
): Promise<void> {
|
|
79
|
+
const mcp = readMcpAppMeta(tool)
|
|
80
|
+
const uiUri = mcp?.uiResourceUri
|
|
81
|
+
if (!uiUri || !mcp.readResource) return
|
|
82
|
+
|
|
83
|
+
// The try covers ONLY the fallible read — keep `emitCustomEvent` out of it so
|
|
84
|
+
// an exception from the emit path can't be mislabeled as a read failure.
|
|
85
|
+
let matched: McpResourceReadResult['contents'][number] | undefined
|
|
86
|
+
try {
|
|
87
|
+
const res = await mcp.readResource(uiUri)
|
|
88
|
+
// Emit ONLY the content whose uri matches the requested `uiUri`. A source
|
|
89
|
+
// can return unrelated contents; falling back to `contents[0]` would risk
|
|
90
|
+
// rendering a widget that doesn't correspond to the linked resource. This
|
|
91
|
+
// is a display widget — a mismatched resource is worse than none, so if no
|
|
92
|
+
// content matches we fail-soft (warn + return) rather than emit.
|
|
93
|
+
matched = res.contents.find((c) => c.uri === uiUri)
|
|
94
|
+
} catch (err) {
|
|
95
|
+
// fail-soft — the text tool-result already flows; a broken widget must
|
|
96
|
+
// not break the run.
|
|
97
|
+
console.warn(`[mcp-apps] failed to read ui resource ${uiUri}:`, err)
|
|
98
|
+
return
|
|
99
|
+
}
|
|
100
|
+
if (!matched) {
|
|
101
|
+
console.warn(
|
|
102
|
+
`[mcp-apps] ui resource ${uiUri} returned no content matching that uri; not emitting`,
|
|
103
|
+
)
|
|
104
|
+
return
|
|
105
|
+
}
|
|
106
|
+
// NOTE: `toolCallId` is intentionally NOT set here — it is stamped onto
|
|
107
|
+
// every emitted event by the `executeToolCalls` context wrapper, so the
|
|
108
|
+
// UIResourceEvent.value.toolCallId / UIResourcePart.toolCallId contract is
|
|
109
|
+
// still satisfied downstream.
|
|
110
|
+
context.emitCustomEvent('ui-resource', {
|
|
111
|
+
resource: {
|
|
112
|
+
uri: matched.uri,
|
|
113
|
+
mimeType: matched.mimeType ?? 'text/html',
|
|
114
|
+
text: matched.text,
|
|
115
|
+
blob: matched.blob,
|
|
116
|
+
},
|
|
117
|
+
serverId: mcp.serverId,
|
|
118
|
+
toolName: mcp.serverToolName ?? tool.name,
|
|
119
|
+
meta: undefined,
|
|
120
|
+
})
|
|
121
|
+
}
|
|
122
|
+
|
|
36
123
|
/**
|
|
37
124
|
* Optional middleware hooks for tool execution.
|
|
38
125
|
* When provided, these callbacks are invoked before/after each tool execution.
|
|
@@ -456,7 +543,7 @@ async function applyBeforeToolCallDecision(
|
|
|
456
543
|
* Execute a server-side tool with event polling, output validation, and middleware hooks.
|
|
457
544
|
* Yields CustomEvent chunks during execution and pushes the result to the results array.
|
|
458
545
|
*/
|
|
459
|
-
async function* executeServerTool<TContext = unknown>(
|
|
546
|
+
export async function* executeServerTool<TContext = unknown>(
|
|
460
547
|
toolCall: ToolCall,
|
|
461
548
|
tool: AnyTool,
|
|
462
549
|
toolName: string,
|
|
@@ -475,7 +562,14 @@ async function* executeServerTool<TContext = unknown>(
|
|
|
475
562
|
let result = yield* executeWithEventPolling(executionPromise, pendingEvents)
|
|
476
563
|
const duration = Date.now() - startTime
|
|
477
564
|
|
|
478
|
-
//
|
|
565
|
+
// MCP Apps: if this tool links a ui:// resource, eagerly read it and queue
|
|
566
|
+
// a `ui-resource` CUSTOM event. The MCP source stays live until the run
|
|
567
|
+
// drains (MCPManager's `connection:'close'` policy disposes on completion),
|
|
568
|
+
// so `readResource` is callable here. Fail-soft: a read error warns and
|
|
569
|
+
// emits nothing — the text result still flows.
|
|
570
|
+
await emitUiResourceIfLinked(tool, context)
|
|
571
|
+
|
|
572
|
+
// Flush remaining events (including any queued ui-resource event)
|
|
479
573
|
let pendingEvent: CustomEvent | undefined
|
|
480
574
|
while ((pendingEvent = pendingEvents.shift()) !== undefined) {
|
|
481
575
|
yield pendingEvent
|
package/src/adapter-internals.ts
CHANGED
|
@@ -10,3 +10,9 @@ export {
|
|
|
10
10
|
toRunErrorPayload,
|
|
11
11
|
toRunErrorRawEvent,
|
|
12
12
|
} from './activities/error-payload'
|
|
13
|
+
export {
|
|
14
|
+
getSandboxRuntime,
|
|
15
|
+
provideSandboxRuntime,
|
|
16
|
+
SandboxRuntimeCapability,
|
|
17
|
+
} from './activities/chat/middleware/sandbox-runtime'
|
|
18
|
+
export type { SandboxRuntime } from './activities/chat/middleware/sandbox-runtime'
|
package/src/client.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -120,6 +120,8 @@ export type {
|
|
|
120
120
|
FinishInfo,
|
|
121
121
|
AbortInfo,
|
|
122
122
|
ErrorInfo,
|
|
123
|
+
SandboxFileEvent,
|
|
124
|
+
ChatSandboxHooks,
|
|
123
125
|
} from './activities/chat/middleware/index'
|
|
124
126
|
|
|
125
127
|
// Base, activity-agnostic middleware. The observe-only superset that media
|
|
@@ -149,6 +151,8 @@ export type {
|
|
|
149
151
|
CapabilityContext,
|
|
150
152
|
CapabilityGetter,
|
|
151
153
|
CapabilityProvider,
|
|
154
|
+
DefinedChatMiddleware,
|
|
155
|
+
AnyChatMiddleware,
|
|
152
156
|
} from './activities/chat/middleware/index'
|
|
153
157
|
|
|
154
158
|
// All types
|
|
@@ -31,6 +31,7 @@ const CATEGORY_EMOJI: Record<keyof ResolvedCategories, string> = {
|
|
|
31
31
|
agentLoop: '🔁',
|
|
32
32
|
config: '⚙️',
|
|
33
33
|
errors: '❌',
|
|
34
|
+
sandbox: '📦',
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
export class InternalLogger {
|
|
@@ -82,6 +83,11 @@ export class InternalLogger {
|
|
|
82
83
|
this.emit('debug', 'tools', message, meta)
|
|
83
84
|
}
|
|
84
85
|
|
|
86
|
+
/** Log sandbox internals (watcher, file events, hook dispatch). Chat-only. */
|
|
87
|
+
sandbox(message: string, meta?: Record<string, unknown>): void {
|
|
88
|
+
this.emit('debug', 'sandbox', message, meta)
|
|
89
|
+
}
|
|
90
|
+
|
|
85
91
|
/** Log an agent-loop iteration marker or phase transition. Chat-only. */
|
|
86
92
|
agentLoop(message: string, meta?: Record<string, unknown>): void {
|
|
87
93
|
this.emit('debug', 'agentLoop', message, meta)
|