@tanstack/ai 0.39.1 → 0.41.0

package/dist/esm/index.d.ts

@@ -18,7 +18,7 @@ export type { ProviderTool } from './tools/provider-tool.js';
 export { brandProviderTool } from './tools/provider-tool.js';
 export { maxIterations, untilFinishReason, combineStrategies, } from './activities/chat/agent-loop-strategies.js';
 export { createToolRegistry, createFrozenRegistry, type ToolRegistry, } from './tool-registry.js';
-export type { ChatMiddleware, ChatMiddlewareContext, ChatMiddlewarePhase, ChatMiddlewareConfig, StructuredOutputMiddlewareConfig, ToolCallHookContext, BeforeToolCallDecision, AfterToolCallInfo, IterationInfo, ToolPhaseCompleteInfo, UsageInfo, FinishInfo, AbortInfo, ErrorInfo, SandboxFileEvent, ChatSandboxHooks, } from './activities/chat/middleware/index.js';
+export type { ChatMiddleware, ChatMiddlewareContext, ChatMiddlewarePhase, ChatMiddlewareConfig, StructuredOutputMiddlewareConfig, ToolCallHookContext, BeforeToolCallDecision, AfterToolCallInfo, IterationInfo, ToolPhaseCompleteInfo, UsageInfo, FinishInfo, AbortInfo, ErrorInfo, SandboxFileEvent, SandboxFileHookEvent, ChatSandboxHooks, } from './activities/chat/middleware/index.js';
 export type { GenerationMiddleware, GenerationMiddlewareContext, GenerationActivity, GenerationUsageInfo, GenerationFinishInfo, GenerationAbortInfo, GenerationErrorInfo, AnyGenerationMiddleware, } from './activities/middleware/index.js';
 export { createCapability, defineChatMiddleware, createChatMiddleware, } from './activities/chat/middleware/index.js';
 export type { Capability, CapabilityHandle, CapabilityContext, CapabilityGetter, CapabilityProvider, DefinedChatMiddleware, AnyChatMiddleware, } from './activities/chat/middleware/index.js';
@@ -30,8 +30,8 @@ export type { ResolvedMediaPrompt } from './utilities/media-prompt.js';
 export type { SystemPrompt, NormalizedSystemPrompt } from './system-prompts.js';
 export { normalizeSystemPrompts } from './system-prompts.js';
 export { detectImageMimeType } from './utils.js';
-export { realtimeToken } from './realtime/index.js';
-export type { RealtimeToken, RealtimeTokenAdapter, RealtimeTokenOptions, RealtimeSessionConfig, VADConfig, RealtimeMessage, RealtimeMessagePart, RealtimeTextPart, RealtimeAudioPart, RealtimeToolCallPart, RealtimeToolResultPart, RealtimeImagePart, RealtimeStatus, RealtimeMode, AudioVisualization, RealtimeEvent, RealtimeEventPayloads, RealtimeEventHandler, RealtimeErrorCode, RealtimeError, RealtimeAdapter, RealtimeConnection, } from './realtime/index.js';
+export { realtimeToken, createRealtimeEventEmitter } from './realtime/index.js';
+export type { RealtimeToken, RealtimeTokenAdapter, RealtimeTokenOptions, RealtimeSessionConfig, RealtimeToolConfig, VADConfig, RealtimeMessage, RealtimeMessagePart, RealtimeTextPart, RealtimeAudioPart, RealtimeToolCallPart, RealtimeToolResultPart, RealtimeImagePart, RealtimeStatus, RealtimeMode, AudioVisualization, RealtimeEvent, RealtimeEventPayloads, RealtimeEventHandler, RealtimeErrorCode, RealtimeError, RealtimeAdapter, RealtimeConnection, } from './realtime/index.js';
 export { convertMessagesToModelMessages, generateMessageId, uiMessageToModelMessages, modelMessageToUIMessage, modelMessagesToUIMessages, normalizeToUIMessage, } from './activities/chat/messages.js';
 export { StreamProcessor, createReplayStream, ImmediateStrategy, PunctuationStrategy, BatchStrategy, WordBoundaryStrategy, CompositeStrategy, PartialJSONParser, defaultJSONParser, parsePartialJSON, } from './activities/chat/stream/index.js';
 export type { ChunkStrategy, ChunkRecording, InternalToolCallState, ProcessorResult, ProcessorState, StreamProcessorEvents, StreamProcessorOptions, ToolCallState, ToolResultState, JSONParser, } from './activities/chat/stream/index.js';

package/dist/esm/index.js

@@ -33,6 +33,7 @@ import { PartialJSONParser, defaultJSONParser, parsePartialJSON } from "./activi
 import { StreamProcessor, createReplayStream } from "./activities/chat/stream/processor.js";
 import { createCapability } from "./activities/chat/middleware/capabilities.js";
 import { createChatMiddleware } from "./activities/chat/middleware/builder.js";
+import { createRealtimeEventEmitter } from "./realtime/event-emitter.js";
 import { defineChatMiddleware } from "./activities/chat/middleware/define.js";
 export {
   BatchStrategy,
@@ -63,6 +64,7 @@ export {
   createFrozenRegistry,
   createImageOptions,
   createModel,
+  createRealtimeEventEmitter,
   createReplayStream,
   createSpeechOptions,
   createSummarizeOptions,

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}
+{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/dist/esm/realtime/event-emitter.d.ts

@@ -0,0 +1,5 @@
+import { RealtimeEvent, RealtimeEventHandler, RealtimeEventPayloads } from './types.js';
+export declare function createRealtimeEventEmitter(): {
+    emit<TEvent extends RealtimeEvent>(event: TEvent, payload: RealtimeEventPayloads[TEvent]): void;
+    on<TEvent extends RealtimeEvent>(event: TEvent, handler: RealtimeEventHandler<TEvent>): () => void;
+};

package/dist/esm/realtime/event-emitter.js

@@ -0,0 +1,27 @@
+function createRealtimeEventEmitter() {
+  const eventHandlers = /* @__PURE__ */ new Map();
+  return {
+    emit(event, payload) {
+      const handlers = eventHandlers.get(event);
+      if (!handlers) return;
+      for (const handler of handlers) {
+        handler(payload);
+      }
+    },
+    on(event, handler) {
+      let handlers = eventHandlers.get(event);
+      if (!handlers) {
+        handlers = /* @__PURE__ */ new Set();
+        eventHandlers.set(event, handlers);
+      }
+      handlers.add(handler);
+      return () => {
+        handlers.delete(handler);
+      };
+    }
+  };
+}
+export {
+  createRealtimeEventEmitter
+};
+//# sourceMappingURL=event-emitter.js.map

package/dist/esm/realtime/event-emitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"event-emitter.js","sources":["../../../src/realtime/event-emitter.ts"],"sourcesContent":["import type {\n  RealtimeEvent,\n  RealtimeEventHandler,\n  RealtimeEventPayloads,\n} from './types'\n\n/**\n * Handlers are stored with a `never` payload so any specific\n * `RealtimeEventHandler<TEvent>` is assignable in (contravariance), keeping the\n * heterogeneous handler map type-safe without `any`. `emit` narrows back to the\n * event's real payload type via its signature; the lone `as never` at the call\n * site is the inverse of that stored `never`.\n */\ntype StoredHandler = (payload: never) => void\n\nexport function createRealtimeEventEmitter() {\n  const eventHandlers = new Map<RealtimeEvent, Set<StoredHandler>>()\n\n  return {\n    emit<TEvent extends RealtimeEvent>(\n      event: TEvent,\n      payload: RealtimeEventPayloads[TEvent],\n    ) {\n      const handlers = eventHandlers.get(event)\n      if (!handlers) return\n      for (const handler of handlers) {\n        handler(payload as never)\n      }\n    },\n    on<TEvent extends RealtimeEvent>(\n      event: TEvent,\n      handler: RealtimeEventHandler<TEvent>,\n    ): () => void {\n      let handlers = eventHandlers.get(event)\n      if (!handlers) {\n        handlers = new Set<StoredHandler>()\n        eventHandlers.set(event, handlers)\n      }\n      handlers.add(handler)\n\n      return () => {\n        handlers.delete(handler)\n      }\n    },\n  }\n}\n"],"names":[],"mappings":"AAeO,SAAS,6BAA6B;AAC3C,QAAM,oCAAoB,IAAA;AAE1B,SAAO;AAAA,IACL,KACE,OACA,SACA;AACA,YAAM,WAAW,cAAc,IAAI,KAAK;AACxC,UAAI,CAAC,SAAU;AACf,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAgB;AAAA,MAC1B;AAAA,IACF;AAAA,IACA,GACE,OACA,SACY;AACZ,UAAI,WAAW,cAAc,IAAI,KAAK;AACtC,UAAI,CAAC,UAAU;AACb,uCAAe,IAAA;AACf,sBAAc,IAAI,OAAO,QAAQ;AAAA,MACnC;AACA,eAAS,IAAI,OAAO;AAEpB,aAAO,MAAM;AACX,iBAAS,OAAO,OAAO;AAAA,MACzB;AAAA,IACF;AAAA,EAAA;AAEJ;"}

package/dist/esm/realtime/index.d.ts

@@ -1,4 +1,5 @@
 import { RealtimeToken, RealtimeTokenOptions } from './types.js';
+export { createRealtimeEventEmitter } from './event-emitter.js';
 export type * from './types.js';
 /**
  * Generate a realtime token using the provided adapter.

package/dist/esm/realtime/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":["../../../src/realtime/index.ts"],"sourcesContent":["import type { RealtimeToken, RealtimeTokenOptions } from './types'\n\n// Re-export all types\nexport type * from './types'\n\n/**\n * Generate a realtime token using the provided adapter.\n *\n * This function is used on the server to generate ephemeral tokens\n * that clients can use to establish realtime connections.\n *\n * @param options - Token generation options including the adapter\n * @returns Promise resolving to a RealtimeToken\n *\n * @example\n * ```typescript\n * import { realtimeToken } from '@tanstack/ai'\n * import { openaiRealtimeToken } from '@tanstack/ai-openai'\n *\n * // Server function (TanStack Start example)\n * export const getRealtimeToken = createServerFn()\n *   .handler(async () => {\n *     return realtimeToken({\n *       adapter: openaiRealtimeToken({\n *         model: 'gpt-realtime',\n *       }),\n *     })\n *   })\n * ```\n */\nexport async function realtimeToken(\n  options: RealtimeTokenOptions,\n): Promise<RealtimeToken> {\n  const { adapter } = options\n  return adapter.generateToken()\n}\n"],"names":[],"mappings":"AA8BA,eAAsB,cACpB,SACwB;AACxB,QAAM,EAAE,YAAY;AACpB,SAAO,QAAQ,cAAA;AACjB;"}
+{"version":3,"file":"index.js","sources":["../../../src/realtime/index.ts"],"sourcesContent":["import type { RealtimeToken, RealtimeTokenOptions } from './types'\n\nexport { createRealtimeEventEmitter } from './event-emitter'\n\n// Re-export all types\nexport type * from './types'\n\n/**\n * Generate a realtime token using the provided adapter.\n *\n * This function is used on the server to generate ephemeral tokens\n * that clients can use to establish realtime connections.\n *\n * @param options - Token generation options including the adapter\n * @returns Promise resolving to a RealtimeToken\n *\n * @example\n * ```typescript\n * import { realtimeToken } from '@tanstack/ai'\n * import { openaiRealtimeToken } from '@tanstack/ai-openai'\n *\n * // Server function (TanStack Start example)\n * export const getRealtimeToken = createServerFn()\n *   .handler(async () => {\n *     return realtimeToken({\n *       adapter: openaiRealtimeToken({\n *         model: 'gpt-realtime',\n *       }),\n *     })\n *   })\n * ```\n */\nexport async function realtimeToken(\n  options: RealtimeTokenOptions,\n): Promise<RealtimeToken> {\n  const { adapter } = options\n  return adapter.generateToken()\n}\n"],"names":[],"mappings":"AAgCA,eAAsB,cACpB,SACwB;AACxB,QAAM,EAAE,YAAY;AACpB,SAAO,QAAQ,cAAA;AACjB;"}

package/dist/esm/realtime/types.d.ts

@@ -1,4 +1,5 @@
 import { AnyClientTool } from '../activities/chat/tools/tool-definition.js';
+import { UsageInfo } from '../activities/chat/middleware/types.js';
 /**
  * Voice activity detection configuration
  */
@@ -18,6 +19,7 @@ export interface RealtimeToolConfig {
     name: string;
     description: string;
     inputSchema?: Record<string, any>;
+    outputSchema?: Record<string, any>;
 }
 /**
  * Configuration for a realtime session
@@ -182,7 +184,7 @@ export interface AudioVisualization {
 /**
  * Events emitted by the realtime connection
  */
-export type RealtimeEvent = 'status_change' | 'mode_change' | 'transcript' | 'audio_chunk' | 'tool_call' | 'message_complete' | 'interrupted' | 'error';
+export type RealtimeEvent = 'status_change' | 'mode_change' | 'transcript' | 'audio_chunk' | 'tool_call' | 'message_complete' | 'interrupted' | 'error' | 'go_away' | 'usage';
 /**
  * Event payloads for realtime events
  */
@@ -216,6 +218,10 @@ export interface RealtimeEventPayloads {
     error: {
         error: Error;
     };
+    go_away: {
+        timeLeft?: string;
+    };
+    usage: UsageInfo;
 }
 /**
  * Handler type for realtime events
@@ -273,6 +279,8 @@ export interface RealtimeConnection {
     sendToolResult: (callId: string, result: string) => void;
     /** Update session configuration */
     updateSession: (config: Partial<RealtimeSessionConfig>) => void;
+    /** Update the ephemeral token (e.g. on refresh); provider may reconnect */
+    updateToken?: (token: RealtimeToken) => void;
     /** Interrupt the current response */
     interrupt: () => void;
     /** Subscribe to connection events */

package/dist/esm/types.d.ts

@@ -261,6 +261,15 @@ export interface ToolCallPart<TMetadata = unknown> {
     id: string;
     name: string;
     arguments: string;
+    /**
+     * Parsed tool input. Set from the parsed arguments once they are complete
+     * (`state: 'input-complete'` and later). `undefined` while the raw
+     * `arguments` string is still streaming, and may stay `undefined` for a call
+     * that terminates in an error state — the raw `arguments` string is always
+     * available as a fallback. Typed per-tool on the client `ToolCallPart` (see
+     * `@tanstack/ai-client`); `unknown` on this base type.
+     */
+    input?: unknown;
     state: ToolCallState;
     /** Approval metadata if tool requires user approval */
     approval?: {
@@ -1154,6 +1163,119 @@ export interface UIResourceEvent extends CustomEvent {
         meta?: Record<string, unknown>;
     };
 }
+export interface SandboxFileCustomEvent extends CustomEvent {
+    name: 'sandbox.file';
+    value: {
+        type: 'create' | 'change' | 'delete';
+        path: string;
+        timestamp: number;
+    };
+}
+export interface SandboxFileDiffEvent extends CustomEvent {
+    name: 'sandbox.file.diff';
+    value: {
+        path: string;
+        diff: string;
+    };
+}
+export interface FileChangedEvent extends CustomEvent {
+    name: 'file.changed';
+    value: {
+        path: string;
+        diff: string;
+    };
+}
+export interface SessionIdEvent extends CustomEvent {
+    name: `${string}.session-id`;
+    value: {
+        sessionId: string;
+    };
+}
+export interface CodeModeExecutionStartedEvent extends CustomEvent {
+    name: 'code_mode:execution_started';
+    value: {
+        timestamp: number;
+        codeLength: number;
+    };
+}
+export interface CodeModeConsoleEvent extends CustomEvent {
+    name: 'code_mode:console';
+    value: {
+        level: 'log' | 'warn' | 'error' | 'info';
+        message: string;
+        timestamp: number;
+    };
+}
+export interface CodeModeExternalCallEvent extends CustomEvent {
+    name: 'code_mode:external_call';
+    value: {
+        function: string;
+        args: unknown;
+        timestamp: number;
+    };
+}
+export interface CodeModeExternalResultEvent extends CustomEvent {
+    name: 'code_mode:external_result';
+    value: {
+        function: string;
+        result: unknown;
+        duration: number;
+    };
+}
+export interface CodeModeExternalErrorEvent extends CustomEvent {
+    name: 'code_mode:external_error';
+    value: {
+        function: string;
+        error: string;
+        duration: number;
+    };
+}
+export interface CodeModeSkillCallEvent extends CustomEvent {
+    name: 'code_mode:skill_call';
+    value: {
+        skill: string;
+        input: unknown;
+        timestamp: number;
+    };
+}
+export interface CodeModeSkillResultEvent extends CustomEvent {
+    name: 'code_mode:skill_result';
+    value: {
+        skill: string;
+        result: unknown;
+        duration: number;
+        timestamp: number;
+    };
+}
+export interface CodeModeSkillErrorEvent extends CustomEvent {
+    name: 'code_mode:skill_error';
+    value: {
+        skill: string;
+        error: string;
+        duration: number;
+        timestamp: number;
+    };
+}
+export interface SkillRegisteredEvent extends CustomEvent {
+    name: 'skill:registered';
+    value: {
+        id: string;
+        name: string;
+        description: string;
+        timestamp: number;
+    };
+}
+/**
+ * Every CUSTOM event TanStack AI itself emits, as a discriminated union on
+ * `name`. User-emitted custom events (via `emitCustomEvent` with a custom name)
+ * are intentionally absent — they still flow at runtime.
+ */
+export type KnownCustomEvent = SandboxFileCustomEvent | SandboxFileDiffEvent | FileChangedEvent | SessionIdEvent | CodeModeExecutionStartedEvent | CodeModeConsoleEvent | CodeModeExternalCallEvent | CodeModeExternalResultEvent | CodeModeExternalErrorEvent | CodeModeSkillCallEvent | CodeModeSkillResultEvent | CodeModeSkillErrorEvent | SkillRegisteredEvent | StructuredOutputStartEvent | StructuredOutputCompleteEvent | ApprovalRequestedEvent | ToolInputAvailableEvent | UIResourceEvent;
+/** The default chat streaming result: standard chunks plus every typed
+ *  framework CUSTOM event, with the `value: any` catch-all excluded so
+ *  literal-`name` narrowing types `value`. User-emitted custom names are typed
+ *  out (still flow at runtime — branch outside the name narrows or cast). */
+export type ChatStream = AsyncIterable<Exclude<StreamChunk, CustomEvent> | KnownCustomEvent>;
 /**
  * Public type for streams returned by `chat({ outputSchema, stream: true })`.
  *
@@ -1584,6 +1706,7 @@ export interface TTSResult {
  * Options for audio transcription.
  * These are the common options supported across providers.
  */
+export type TranscriptionResponseFormat = 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';
 export interface TranscriptionOptions<TProviderOptions extends object = object> {
     /** The model to use for transcription */
     model: string;
@@ -1594,7 +1717,7 @@ export interface TranscriptionOptions<TProviderOptions extends object = object>
     /** An optional prompt to guide the transcription */
     prompt?: string;
     /** The format of the transcription output */
-    responseFormat?: 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';
+    responseFormat?: TranscriptionResponseFormat;
     /** Model-specific options for transcription */
     modelOptions?: TProviderOptions;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai",
-  "version": "0.39.1",
+  "version": "0.41.0",
   "description": "Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.",
   "author": "Tanner Linsley",
   "license": "MIT",

package/skills/ai-core/adapter-configuration/SKILL.md

@@ -297,6 +297,16 @@ Per-provider sampling keys (all live inside `modelOptions`):
 some sampling options use provider-native names. Ollama nests all sampling under
 `modelOptions.options`.
 
+> **Anthropic `max_tokens` default:** Anthropic's API _requires_ `max_tokens`,
+> so the adapter always sends one. When you omit `modelOptions.max_tokens`, it
+> defaults to the selected model's full output ceiling (its `max_output_tokens`
+> from model metadata — e.g. 64K for Sonnet, 128K for Opus), not a low constant.
+> `max_tokens` is a ceiling, not a reservation (billing is per token generated),
+> so leaving it unset is the right default for codegen / agentic / long-form
+> output and avoids silent `stop_reason: "max_tokens"` truncation. Set it only to
+> cap output below the model ceiling. Other providers treat token limits as
+> optional and don't apply this flooring.
+
 ### 6. Capability Flag: `supportsCombinedToolsAndSchema`
 
 Adapters can declare an optional capability method:

package/skills/ai-core/ag-ui-protocol/SKILL.md

@@ -13,6 +13,7 @@ sources:
   - 'TanStack/ai:docs/protocol/chunk-definitions.md'
   - 'TanStack/ai:docs/protocol/sse-protocol.md'
   - 'TanStack/ai:docs/protocol/http-stream-protocol.md'
+  - 'TanStack/ai:docs/protocol/custom-events.md'
 ---
 
 # AG-UI Protocol
@@ -218,6 +219,62 @@ RUN_STARTED -> TEXT_MESSAGE_START -> TEXT_MESSAGE_CONTENT* -> TEXT_MESSAGE_END
 union of all event interfaces). `StreamChunkType` is an alias for `AGUIEventType`
 (the string union of all event type literals).
 
+### 4. Typed CUSTOM Events — `ChatStream` and `KnownCustomEvent`
+
+The `CUSTOM` row above describes the raw `StreamChunk` union, where the single
+generic `CustomEvent` member types `value` as `any` -- once merged into a
+union, that `any` poisons every other member too, so narrowing on `name`
+still leaves `value: any`. `chat()` doesn't return raw `StreamChunk`; by
+default (no `outputSchema`, `stream` not explicitly `false`) it returns
+`ChatStream`, which swaps that generic member for `KnownCustomEvent` -- a
+discriminated union of every `CUSTOM` event TanStack AI itself emits, each
+with a literal `name` and a concrete `value`. Narrow with a plain `if` --
+no helper, no cast:
+
+```typescript
+import { chat } from '@tanstack/ai'
+import { openaiText } from '@tanstack/ai-openai'
+
+const stream = chat({
+  adapter: openaiText('gpt-5.2'),
+  messages,
+})
+
+for await (const chunk of stream) {
+  if (chunk.type === 'CUSTOM' && chunk.name === 'sandbox.file.diff') {
+    console.log(chunk.value.path, chunk.value.diff) // typed, no helper, no cast
+  } else if (
+    chunk.type === 'CUSTOM' &&
+    chunk.name === 'structured-output.complete'
+  ) {
+    console.log(chunk.value.object) // typed, no helper, no cast
+  }
+}
+```
+
+**Caveat -- `.endsWith()` (or any non-literal check) does not narrow.**
+`SessionIdEvent['name']` is the template-literal type
+`` `${string}.session-id` ``. TypeScript's control-flow narrowing only
+understands exact comparisons (`===`) and `in`/type-predicate checks against
+a discriminant -- a runtime `chunk.name.endsWith('.session-id')` check
+doesn't inform the type system, so `chunk.value` stays the union of every
+`KnownCustomEvent`'s `value`, not `{ sessionId: string }`. Compare against
+the exact literal you expect, or write a user-defined type predicate
+(`(c): c is SessionIdEvent => c.name.endsWith('.session-id')`) and call that
+in the `if` instead.
+
+**User-emitted `emitCustomEvent` names are typed out of `ChatStream`.** Tools
+that call `context.emitCustomEvent('my-app:progress', ...)` still stream a
+`CUSTOM` chunk at runtime, but `'my-app:progress'` isn't one of
+`KnownCustomEvent`'s literal names, so it's intentionally absent from
+`ChatStream`'s type -- including a generic fallback member would reintroduce
+the `value: any` poison for every other event on the stream. To read your own
+event with a type, annotate the stream as the wider `StreamChunk` instead of
+`ChatStream` for that branch; its generic `CUSTOM` member already types
+`value` as `any`, so no cast is needed there either.
+
+Source: docs/protocol/custom-events.md
+
 ## Common Mistakes
 
 ### MEDIUM: Proxy buffering breaks SSE streaming
@@ -273,3 +330,5 @@ without transformation. See `docs/migration/ag-ui-compliance.md` for details.
 ## Cross-References
 
 - See also: `ai-core/custom-backend-integration/SKILL.md` -- Custom backends must implement SSE or HTTP stream format to work with TanStack AI client connection adapters.
+- See also: `ai-core/middleware/SKILL.md` -- `sandbox.file.diff`'s `{ path, diff }` value (one of `KnownCustomEvent`'s members) is populated from the same lazy `before()`/`after()`/`diff()` accessors documented there for `onFile*` middleware hooks.
+- Full CUSTOM event taxonomy: `docs/protocol/custom-events.md`.

package/skills/ai-core/media-generation/SKILL.md

@@ -261,6 +261,17 @@ await generateVideo({
 })
 ```
 
+**URL inputs that require an upload throw by default.** Most adapters pass a
+`type: 'url'` source straight through to the provider. Three paths can't —
+OpenAI `images.edit()`, OpenAI Sora `input_reference`, and Gemini **Veo** —
+because the provider only accepts uploaded bytes (Veo also takes a `gs://`
+reference). For those, an HTTP(S) URL would have to be downloaded and buffered
+in memory, which can OOM constrained runtimes, so they **throw** on an HTTP(S)
+URL image input by default. Pass a `data:` URI (or `gs://` for Veo), or opt in
+with `allowUrlFetch: true` on the adapter config
+(`createOpenaiImage(model, apiKey, { allowUrlFetch: true })`, and likewise on
+`createOpenaiVideo` / `createGeminiVideo`). `data:` URIs never need the flag.
+
 **Role hints** (`metadata.role`):
 
 | Role            | Maps to                                                                                               |
@@ -357,7 +368,7 @@ const { generate, result, isLoading } = useGenerateSpeech({
 ### 4. Audio Transcription
 
 Adapter: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
-gpt-4o-mini-transcribe).
+gpt-4o-mini-transcribe, gpt-4o-transcribe-diarize).
 
 > **Capturing audio in the browser:** Use `useAudioRecorder` from `@tanstack/ai-react` to record directly in the browser, then pass the recording as the `audio` input to `generate()`, or use `recording.part` as a prompt part in chat/generation calls. No transcoding or extra dependencies required — the recorder returns the native browser format (`audio/webm` or `audio/mp4`). For transcription, wrap it as a `data:` URL so the provider gets the real content type; passing raw `recording.base64` makes the adapter assume `audio/mpeg` and mislabel the webm/mp4 bytes.
 >
@@ -382,16 +393,21 @@ const result = await generateTranscription({
   language: 'en',
   responseFormat: 'verbose_json',
   modelOptions: {
-    include: ['segment', 'word'],
+    timestamp_granularities: ['word', 'segment'],
   },
 })
 
 // result.text       -- full transcribed text
 // result.language   -- detected/specified language
 // result.duration   -- audio duration in seconds
-// result.segments   -- timestamped segments with optional word-level timestamps
+// result.segments   -- timestamped segments (word-level timestamps are in result.words)
 ```
 
+For speaker diarization, use `openaiTranscription('gpt-4o-transcribe-diarize')`.
+When no response format is given it defaults the request to `response_format: 'diarized_json'`
+and `chunking_strategy: 'auto'` (a top-level `responseFormat` of `'json'`/`'text'` opts out of
+speaker segments); do not pass `prompt`, `include`, or `timestamp_granularities` with this model.
+
 Client hook:
 
 ```tsx
@@ -443,8 +459,8 @@ return toServerSentEventsResponse(stream)
 ```
 
 Google Veo (`@tanstack/ai-gemini`) uses the same jobs/polling flow. Its
-`duration` option is typed per model (e.g. `4 | 6 | 8` for Veo 3.x,
-`5 | 6 | 8` for Veo 2); use `adapter.snapDuration(seconds)` to coerce raw
+`duration` option is typed per model (`4 | 6 | 8` for the Veo 3.1 models);
+use `adapter.snapDuration(seconds)` to coerce raw
 seconds and `adapter.availableDurations()` to enumerate the valid set.
 Image prompt parts route by `metadata.role`: first un-roled /
 `'start_frame'` image → input image, `'end_frame'` → `lastFrame`,
@@ -467,6 +483,35 @@ const { jobId } = await generateVideo({
 // (x-goog-api-key header or ?key= query parameter).
 ```
 
+Gemini Omni Flash (`geminiVideo('gemini-omni-flash-preview')`) is served by
+the Interactions API instead of Veo's operations flow — same adapter, routed
+by model. Clips are 720p; `duration` is any number of seconds in the 3–10
+range (fractional ok, default 10 — availableDurations() reports the range),
+`size` is the aspect ratio (`'16:9' | '9:16'`), and the finished video arrives
+**inline** as a `data:video/mp4;base64,…` URL (no key needed to use it).
+Image/video prompt parts are sent as interaction content blocks, grouped
+as images, then videos, then text (no
+`metadata.role` routing); `data` sources go inline, `url` sources pass
+through as-is (never downloaded — use Gemini Files API URIs for remote
+media). For conversational editing, pass a prior generation's `jobId` as
+`modelOptions.previous_interaction_id` with a prompt describing the change:
+
+```typescript
+import { geminiVideo } from '@tanstack/ai-gemini'
+
+const omni = geminiVideo('gemini-omni-flash-preview')
+const first = await generateVideo({
+  adapter: omni,
+  prompt: 'A violinist outdoors',
+})
+// …poll first.jobId to completion, then edit it:
+const edited = await generateVideo({
+  adapter: omni,
+  prompt: 'Make the violin invisible',
+  modelOptions: { previous_interaction_id: first.jobId },
+})
+```
+
 Other video adapters: `openaiVideo('sora-2')` (pixel sizes like `'1280x720'`,
 durations 4/8/12s, single `input_reference` image prompt part), `grokVideo(...)`
 (`grok-imagine-video` does text-to-video + image-to-video; `grok-imagine-video-1.5` is

package/skills/ai-core/middleware/SKILL.md

@@ -11,6 +11,7 @@ library: tanstack-ai
 library_version: '0.10.0'
 sources:
   - 'TanStack/ai:docs/advanced/middleware.md'
+  - 'TanStack/ai:docs/sandbox/observability.md'
 ---
 
 # Middleware
@@ -371,6 +372,110 @@ Options: `maxSize` (default 100), `ttl` (default Infinity), `toolNames` (default
 `keyFn` (custom cache key), `storage` (custom backend like Redis). See
 `docs/advanced/middleware.md` for custom storage examples.
 
+## Sandbox File-Event Hooks (`sandbox` group)
+
+Declare a `sandbox: ChatSandboxHooks` group on `defineChatMiddleware` to react
+to every file created/changed/deleted inside a sandbox provided by
+`withSandbox` (from `@tanstack/ai-sandbox`). These fire **per-run**,
+server-side, and each handler receives the run's `ChatMiddlewareContext` as
+the first argument:
+
+```typescript
+import { defineChatMiddleware } from '@tanstack/ai'
+import { db } from './db'
+
+const auditMiddleware = defineChatMiddleware({
+  name: 'audit',
+  sandbox: {
+    onFile: (ctx, e) => console.log(ctx.runId, e.type, e.path),
+    onFileCreate: (ctx, e) => db.log({ run: ctx.runId, event: e }),
+  },
+})
+```
+
+| Hook           | Fires for                  |
+| -------------- | -------------------------- |
+| `onFile`       | Every create/change/delete |
+| `onFileCreate` | File creates only          |
+| `onFileChange` | File changes only          |
+| `onFileDelete` | File deletes only          |
+
+These are independent of the stream: the engine also emits a `sandbox.file`
+`CUSTOM` chunk per change regardless of whether any `sandbox` hooks are
+registered, so a client can react to the same edits without middleware. See
+`ai-core/ag-ui-protocol/SKILL.md` for reading that chunk (and the opt-in
+`sandbox.file.diff` chunk) off `ChatStream`.
+
+### `before()` / `after()` / `diff()` — lazy, git-backed content accessors
+
+Each hook receives a `SandboxFileHookEvent`: the serializable
+`{ type, path, timestamp }` plus three lazy accessors for the file's content:
+
+```ts
+interface SandboxFileHookEvent {
+  type: 'create' | 'change' | 'delete'
+  path: string
+  timestamp: number
+  before(): Promise<string> // content at the session baseline ('' if new / non-git)
+  after(): Promise<string> // current content ('' if deleted)
+  diff(): Promise<string> // unified patch vs the baseline
+}
+```
+
+```typescript
+import { defineChatMiddleware } from '@tanstack/ai'
+import { db } from './db'
+
+const auditMiddleware = defineChatMiddleware({
+  name: 'audit',
+  sandbox: {
+    onFileChange: async (ctx, e) => {
+      const [before, after] = await Promise.all([e.before(), e.after()])
+      db.log({ run: ctx.runId, path: e.path, before, after })
+    },
+  },
+})
+```
+
+**Lazy — path-only hooks pay nothing.** `before()`, `after()`, and `diff()`
+are methods, not fields: each only reads the file or shells out to `git` when
+called. A hook that only reads `e.path`/`e.type` (like the `onFile` logger
+above) never touches the filesystem or spawns a process.
+
+**Git session baseline.** The sandbox snapshots `git rev-parse HEAD` once at
+setup as the session baseline (empty string if the workspace isn't a git repo
+or has no commits). `before()` and `diff()` always diff against that same
+fixed baseline for the rest of the run, so `onFileChange` reports the file's
+**cumulative** change since the run started, not just the delta since the
+last poll. `after()` always reads current on-disk content. None of the three
+accessors throw: a deleted file resolves `after()` to `''` (it still has
+`before()`); a new file resolves `before()` to `''` (it still has `after()`);
+a non-git workspace resolves **both** `before()` and `after()` to `''` and
+makes `diff()` fall back to a synthesized add-patch built from `after()` —
+except for a `delete` event in a non-git workspace, where there's nothing to
+synthesize and `diff()` resolves to `''`. In a git workspace a file git
+**isn't tracking yet** (a file the agent created, and every later edit to it)
+diffs empty because `git diff` ignores untracked files, so `diff()` falls
+back to the same synthesized add-patch whenever the file is absent at the
+baseline — a create-or-edit of an untracked file never streams an empty diff.
+An empty diff for a **tracked** file (identical to the baseline) stays empty,
+as it should. A **git-ignored** file is withheld: the file event still fires
+(you're notified it changed) but `diff()` returns `''`, so a secret like a
+`.env` never has its contents surfaced in the diff feed.
+
+**Failures are logged, not silent.** Every git/exec/fs failure behind these
+accessors (and behind the `find`-poll watcher) still falls back to `''`/an
+empty snapshot, but logs first: real anomalies (a failed `git diff`, an
+unreadable file, a lost `find` poll) under the `errors` category (on by
+default); expected-empty conditions (a new file's `before()`, a non-git
+baseline) under the `sandbox` debug category.
+
+**Hook errors are swallowed per hook.** A throwing `sandbox` hook is caught
+and logged under the `errors` category (on by default) — it cannot break the
+run or stop other hooks (or the `sandbox.file` chunk) from continuing.
+
+Source: docs/sandbox/observability.md
+
 ## Common Mistakes
 
 ### a. MEDIUM: Trying to modify StreamChunks in middleware
@@ -451,3 +556,4 @@ Source: docs/advanced/middleware.md
 
 - See also: **ai-core/chat-experience/SKILL.md** -- Middleware hooks into the chat lifecycle
 - See also: **ai-core/structured-outputs/SKILL.md** -- Middleware now wraps the final structured-output call; use `onStructuredOutputConfig` for JSON-Schema transforms
+- See also: **ai-core/ag-ui-protocol/SKILL.md** -- Reading the `sandbox.file` / `sandbox.file.diff` `CUSTOM` chunks the sandbox runtime emits alongside these `sandbox` hooks, via `ChatStream`'s typed `KnownCustomEvent` narrowing

package/skills/ai-core/tool-calling/SKILL.md

@@ -289,7 +289,10 @@ function ChatPage() {
               return (
                 <div key={part.id}>
                   <p>Approve "{part.name}"?</p>
-                  <pre>{part.arguments}</pre>
+                  {/* `part.input` is the parsed, typed object (populated once
+                      the arguments are complete, as they are at approval
+                      time); `part.arguments` remains the raw JSON string. */}
+                  <pre>{JSON.stringify(part.input, null, 2)}</pre>
                   <button
                     onClick={() =>
                       addToolApprovalResponse({
@@ -322,6 +325,15 @@ function ChatPage() {
 }
 ```
 
+> **Type-safe approval:** With typed `tools`, `part.approval` exists **only**
+> on parts for tools defined with `needsApproval: true`. Tools without approval
+> have no `approval` field (reading it is a compile error). For a
+> tool-agnostic handler over a typed union, narrow with `'approval' in part`
+> (`if (part.type === 'tool-call' && 'approval' in part && part.approval)`),
+> or type a shared component against the base `ToolCallPart`. An untyped
+> `useChat()` keeps `approval` on every tool-call part, which is why the
+> snippet above (no `tools` generic) reads it directly.
+
 ### Pattern 4: Lazy Tool Discovery
 
 Set `lazy: true` on rarely-needed tools. The LLM sees their names via a synthetic