@tanstack/ai 0.58.0 → 0.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/dist/esm/activities/chat/adapter.d.ts +9 -0
  2. package/dist/esm/activities/chat/adapter.js +1 -0
  3. package/dist/esm/activities/chat/adapter.js.map +1 -1
  4. package/dist/esm/activities/chat/agents/define-agent.d.ts +81 -0
  5. package/dist/esm/activities/chat/agents/define-agent.js +34 -0
  6. package/dist/esm/activities/chat/agents/define-agent.js.map +1 -0
  7. package/dist/esm/activities/chat/agents/route.d.ts +53 -0
  8. package/dist/esm/activities/chat/agents/route.js +59 -0
  9. package/dist/esm/activities/chat/agents/route.js.map +1 -0
  10. package/dist/esm/activities/chat/agents/spawn.d.ts +124 -0
  11. package/dist/esm/activities/chat/agents/spawn.js +490 -0
  12. package/dist/esm/activities/chat/agents/spawn.js.map +1 -0
  13. package/dist/esm/activities/chat/agents/turn.d.ts +36 -0
  14. package/dist/esm/activities/chat/agents/turn.js +78 -0
  15. package/dist/esm/activities/chat/agents/turn.js.map +1 -0
  16. package/dist/esm/activities/chat/index.d.ts +13 -3
  17. package/dist/esm/activities/chat/index.js +462 -26
  18. package/dist/esm/activities/chat/index.js.map +1 -1
  19. package/dist/esm/activities/chat/messages.d.ts +7 -1
  20. package/dist/esm/activities/chat/messages.js +99 -19
  21. package/dist/esm/activities/chat/messages.js.map +1 -1
  22. package/dist/esm/activities/chat/middleware/run-store.d.ts +43 -7
  23. package/dist/esm/activities/chat/middleware/run-store.js +8 -1
  24. package/dist/esm/activities/chat/middleware/run-store.js.map +1 -1
  25. package/dist/esm/activities/chat/middleware/types.d.ts +47 -1
  26. package/dist/esm/activities/chat/middleware/types.js.map +1 -1
  27. package/dist/esm/activities/chat/stream/message-updaters.js +9 -2
  28. package/dist/esm/activities/chat/stream/message-updaters.js.map +1 -1
  29. package/dist/esm/activities/chat/stream/processor.d.ts +46 -1
  30. package/dist/esm/activities/chat/stream/processor.js +294 -18
  31. package/dist/esm/activities/chat/stream/processor.js.map +1 -1
  32. package/dist/esm/activities/chat/tools/tool-calls.d.ts +17 -3
  33. package/dist/esm/activities/chat/tools/tool-calls.js +56 -5
  34. package/dist/esm/activities/chat/tools/tool-calls.js.map +1 -1
  35. package/dist/esm/activities/embed/adapter.d.ts +7 -0
  36. package/dist/esm/activities/embed/adapter.js +1 -0
  37. package/dist/esm/activities/embed/adapter.js.map +1 -1
  38. package/dist/esm/activities/embed/index.js +2 -0
  39. package/dist/esm/activities/embed/index.js.map +1 -1
  40. package/dist/esm/activities/files/adapter.d.ts +97 -0
  41. package/dist/esm/activities/files/adapter.js +45 -0
  42. package/dist/esm/activities/files/adapter.js.map +1 -0
  43. package/dist/esm/activities/files/index.d.ts +66 -0
  44. package/dist/esm/activities/files/index.js +78 -0
  45. package/dist/esm/activities/files/index.js.map +1 -0
  46. package/dist/esm/activities/generateAudio/index.js +1 -1
  47. package/dist/esm/activities/generateImage/adapter.d.ts +8 -0
  48. package/dist/esm/activities/generateImage/adapter.js +1 -0
  49. package/dist/esm/activities/generateImage/adapter.js.map +1 -1
  50. package/dist/esm/activities/generateImage/index.js +3 -1
  51. package/dist/esm/activities/generateImage/index.js.map +1 -1
  52. package/dist/esm/activities/generateLiveVideo/index.js +1 -1
  53. package/dist/esm/activities/generateSpeech/index.js +1 -1
  54. package/dist/esm/activities/generateTranscription/index.js +1 -1
  55. package/dist/esm/activities/generateVideo/adapter.d.ts +8 -0
  56. package/dist/esm/activities/generateVideo/adapter.js +1 -0
  57. package/dist/esm/activities/generateVideo/adapter.js.map +1 -1
  58. package/dist/esm/activities/generateVideo/index.js +3 -0
  59. package/dist/esm/activities/generateVideo/index.js.map +1 -1
  60. package/dist/esm/activities/generateVoice/index.js +1 -1
  61. package/dist/esm/activities/generateWorld/adapter.d.ts +4 -2
  62. package/dist/esm/activities/generateWorld/adapter.js.map +1 -1
  63. package/dist/esm/activities/generateWorld/index.d.ts +4 -3
  64. package/dist/esm/activities/generateWorld/index.js +6 -5
  65. package/dist/esm/activities/generateWorld/index.js.map +1 -1
  66. package/dist/esm/activities/index.d.ts +9 -3
  67. package/dist/esm/activities/index.js +17 -13
  68. package/dist/esm/activities/summarize/chat-stream-summarize.d.ts +2 -0
  69. package/dist/esm/activities/summarize/chat-stream-summarize.js +8 -8
  70. package/dist/esm/activities/summarize/chat-stream-summarize.js.map +1 -1
  71. package/dist/esm/activities/summarize/index.js +1 -1
  72. package/dist/esm/client.d.ts +7 -36
  73. package/dist/esm/client.js +5 -37
  74. package/dist/esm/client.js.map +1 -1
  75. package/dist/esm/index.d.ts +8 -2
  76. package/dist/esm/index.js +9 -4
  77. package/dist/esm/middlewares/content-guard.js.map +1 -1
  78. package/dist/esm/types.d.ts +179 -98
  79. package/dist/esm/utilities/adapter-yield-chunk.d.ts +5 -1
  80. package/dist/esm/utilities/ag-ui-usage.d.ts +9 -9
  81. package/dist/esm/utilities/ag-ui-usage.js +66 -3
  82. package/dist/esm/utilities/ag-ui-usage.js.map +1 -1
  83. package/dist/esm/utilities/ag-ui-wire.js +90 -13
  84. package/dist/esm/utilities/ag-ui-wire.js.map +1 -1
  85. package/dist/esm/utilities/content-source.d.ts +60 -0
  86. package/dist/esm/utilities/content-source.js +85 -0
  87. package/dist/esm/utilities/content-source.js.map +1 -0
  88. package/dist/esm/utilities/normalize-stream-chunk.js +7 -2
  89. package/dist/esm/utilities/normalize-stream-chunk.js.map +1 -1
  90. package/dist/esm/utilities/provider-executed.d.ts +7 -0
  91. package/dist/esm/utilities/provider-executed.js +10 -1
  92. package/dist/esm/utilities/provider-executed.js.map +1 -1
  93. package/dist/esm/utilities/spec-event-keys.js +13 -8
  94. package/dist/esm/utilities/spec-event-keys.js.map +1 -1
  95. package/dist/esm/utilities/subagent-wire.d.ts +36 -0
  96. package/dist/esm/utilities/subagent-wire.js +131 -0
  97. package/dist/esm/utilities/subagent-wire.js.map +1 -0
  98. package/dist/esm/utilities/tool-result.d.ts +12 -2
  99. package/dist/esm/utilities/tool-result.js +23 -3
  100. package/dist/esm/utilities/tool-result.js.map +1 -1
  101. package/package.json +3 -3
  102. package/skills/ai-core/adapter-configuration/SKILL.md +62 -0
  103. package/skills/ai-core/adapter-configuration/references/grok-adapter.md +1 -1
  104. package/skills/ai-core/chat-experience/SKILL.md +14 -0
  105. package/skills/ai-core/media-generation/SKILL.md +10 -2
  106. package/skills/ai-core/middleware/SKILL.md +7 -4
  107. package/src/activities/chat/adapter.ts +10 -0
  108. package/src/activities/chat/agents/define-agent.ts +121 -0
  109. package/src/activities/chat/agents/route.ts +115 -0
  110. package/src/activities/chat/agents/spawn.ts +806 -0
  111. package/src/activities/chat/agents/turn.ts +151 -0
  112. package/src/activities/chat/index.ts +734 -30
  113. package/src/activities/chat/messages.ts +137 -16
  114. package/src/activities/chat/middleware/run-store.ts +56 -7
  115. package/src/activities/chat/middleware/types.ts +47 -0
  116. package/src/activities/chat/stream/message-updaters.ts +24 -2
  117. package/src/activities/chat/stream/processor.ts +452 -30
  118. package/src/activities/chat/tools/tool-calls.ts +83 -11
  119. package/src/activities/embed/adapter.ts +7 -0
  120. package/src/activities/embed/index.ts +5 -0
  121. package/src/activities/files/adapter.ts +120 -0
  122. package/src/activities/files/index.ts +113 -0
  123. package/src/activities/generateImage/adapter.ts +8 -0
  124. package/src/activities/generateImage/index.ts +4 -0
  125. package/src/activities/generateVideo/adapter.ts +8 -0
  126. package/src/activities/generateVideo/index.ts +7 -0
  127. package/src/activities/generateWorld/adapter.ts +4 -2
  128. package/src/activities/generateWorld/index.ts +7 -6
  129. package/src/activities/index.ts +40 -1
  130. package/src/activities/summarize/chat-stream-summarize.ts +22 -12
  131. package/src/client.ts +29 -35
  132. package/src/index.ts +39 -0
  133. package/src/middlewares/content-guard.ts +7 -5
  134. package/src/types.ts +226 -103
  135. package/src/utilities/adapter-yield-chunk.ts +10 -2
  136. package/src/utilities/ag-ui-usage.test.ts +38 -0
  137. package/src/utilities/ag-ui-usage.ts +98 -11
  138. package/src/utilities/ag-ui-wire.ts +134 -16
  139. package/src/utilities/content-source.ts +138 -0
  140. package/src/utilities/normalize-stream-chunk.ts +10 -2
  141. package/src/utilities/provider-executed.ts +13 -0
  142. package/src/utilities/spec-event-keys.ts +34 -7
  143. package/src/utilities/subagent-wire.ts +184 -0
  144. package/src/utilities/tool-result.ts +38 -2
@@ -1,5 +1,6 @@
1
1
  import { normalizeToolResult } from '../../../utilities/tool-result'
2
2
  import { tanstackMetadata } from '../../../utilities/merge-metadata'
3
+ import { isProviderExecutedToolCall } from '../../../utilities/provider-executed'
3
4
  import type { AdapterYieldChunk } from '../../../utilities/adapter-yield-chunk'
4
5
  import { isStandardSchema, parseWithStandardSchema } from './schema-converter'
5
6
  import type { ToolApprovalResolution } from '../../../interrupts'
@@ -8,8 +9,10 @@ import type {
8
9
  ContentPart,
9
10
  CustomEvent,
10
11
  EmitCustomEventOptions,
12
+ Interrupt,
11
13
  ModelMessage,
12
14
  RunFinishedEvent,
15
+ StreamChunk,
13
16
  Tool,
14
17
  ToolCall,
15
18
  ToolCallArgsEvent,
@@ -38,6 +41,25 @@ function safeJsonParse(value: string): unknown {
38
41
  }
39
42
  }
40
43
 
44
+ /** Marks the synthetic tool that runs a subagent. */
45
+ export const SUBAGENT_TOOL = Symbol.for('tanstack.ai.subagentTool')
46
+
47
+ /** Set on the tool context of a subagent tool. Streams a child chunk live. */
48
+ export const EMIT_STREAM_CHUNK = Symbol.for('tanstack.ai.emitStreamChunk')
49
+
50
+ /** What a subagent tool's `execute` returns. */
51
+ export interface SubagentToolOutcome {
52
+ subagentRunId: string
53
+ text: string
54
+ error?: string
55
+ /** Set when the child stopped for outside input. The tool call stays open. */
56
+ interrupts?: Array<Interrupt>
57
+ }
58
+
59
+ function isSubagentTool(tool: AnyTool): boolean {
60
+ return (tool as { [SUBAGENT_TOOL]?: true })[SUBAGENT_TOOL] === true
61
+ }
62
+
41
63
  /**
42
64
  * MCP Apps metadata attached to a server tool at discovery (see
43
65
  * `@tanstack/ai-mcp` discovery + `MCPManager.discover()`).
@@ -505,6 +527,8 @@ interface ExecuteToolCallsResult {
505
527
  needsApproval: Array<ApprovalRequest>
506
528
  /** Tools that need client-side execution */
507
529
  needsClientExecution: Array<ClientToolRequest>
530
+ /** Interrupts raised by subagents that run as tools */
531
+ subagentInterrupts: Array<Interrupt>
508
532
  }
509
533
 
510
534
  /**
@@ -514,8 +538,8 @@ interface ExecuteToolCallsResult {
514
538
  */
515
539
  async function* executeWithEventPolling<T>(
516
540
  executionPromise: Promise<T>,
517
- pendingEvents: Array<CustomEvent>,
518
- ): AsyncGenerator<CustomEvent, T, void> {
541
+ pendingEvents: Array<CustomEvent | StreamChunk>,
542
+ ): AsyncGenerator<CustomEvent | StreamChunk, T, void> {
519
543
  // Use an object to track mutable state across the async boundary
520
544
  const state = { done: false, result: undefined as T }
521
545
  const executionWithFlag = executionPromise.then((r) => {
@@ -532,14 +556,14 @@ async function* executeWithEventPolling<T>(
532
556
  ])
533
557
 
534
558
  // Flush any pending events
535
- let event: CustomEvent | undefined
559
+ let event: CustomEvent | StreamChunk | undefined
536
560
  while ((event = pendingEvents.shift()) !== undefined) {
537
561
  yield event
538
562
  }
539
563
  }
540
564
 
541
565
  // Final flush in case events were emitted right at completion
542
- let event: CustomEvent | undefined
566
+ let event: CustomEvent | StreamChunk | undefined
543
567
  while ((event = pendingEvents.shift()) !== undefined) {
544
568
  yield event
545
569
  }
@@ -612,19 +636,59 @@ export async function* executeServerTool<TContext = unknown>(
612
636
  toolName: string,
613
637
  input: unknown,
614
638
  context: ToolExecutionContext<TContext>,
615
- pendingEvents: Array<CustomEvent>,
639
+ pendingEvents: Array<CustomEvent | StreamChunk>,
616
640
  results: Array<ToolResult>,
617
641
  middlewareHooks?: ToolExecutionMiddlewareHooks,
618
- ): AsyncGenerator<CustomEvent, void, void> {
642
+ subagentInterrupts?: Array<Interrupt>,
643
+ ): AsyncGenerator<CustomEvent | StreamChunk, void, void> {
619
644
  const startTime = Date.now()
620
645
  try {
621
646
  if (!tool.execute) {
622
647
  throw new Error(`Tool ${toolName} has no execute() implementation`)
623
648
  }
649
+ const subagent = isSubagentTool(tool)
650
+ if (subagent) {
651
+ Object.assign(context, {
652
+ [EMIT_STREAM_CHUNK]: (chunk: StreamChunk) => pendingEvents.push(chunk),
653
+ })
654
+ }
624
655
  const executionPromise = Promise.resolve(tool.execute(input, context))
625
656
  let result = yield* executeWithEventPolling(executionPromise, pendingEvents)
626
657
  const duration = Date.now() - startTime
627
658
 
659
+ if (subagent) {
660
+ const outcome = result as SubagentToolOutcome
661
+ if (outcome.interrupts?.length) {
662
+ // The child waits for outside input. Leave the call open so the
663
+ // resume run executes it again and continues the same child.
664
+ subagentInterrupts?.push(...outcome.interrupts)
665
+ return
666
+ }
667
+ const modelResult = outcome.error
668
+ ? { subagentRunId: outcome.subagentRunId, error: outcome.error }
669
+ : { subagentRunId: outcome.subagentRunId, result: outcome.text }
670
+ results.push({
671
+ toolCallId: toolCall.id,
672
+ toolName,
673
+ result: modelResult,
674
+ input,
675
+ output: modelResult,
676
+ duration,
677
+ ...(outcome.error ? { state: 'output-error' as const } : {}),
678
+ })
679
+ await middlewareHooks?.onAfterToolCall?.({
680
+ toolCall,
681
+ tool,
682
+ toolName,
683
+ toolCallId: toolCall.id,
684
+ duration,
685
+ ...(outcome.error
686
+ ? { ok: false as const, error: new Error(outcome.error) }
687
+ : { ok: true as const, result: modelResult }),
688
+ })
689
+ return
690
+ }
691
+
628
692
  // MCP Apps: if this tool links a ui:// resource, eagerly read it and queue
629
693
  // a `ui-resource` CUSTOM event. The MCP source stays live until the run
630
694
  // drains (MCPManager's `connection:'close'` policy disposes on completion),
@@ -633,7 +697,7 @@ export async function* executeServerTool<TContext = unknown>(
633
697
  await emitUiResourceIfLinked(tool, context)
634
698
 
635
699
  // Flush remaining events (including any queued ui-resource event)
636
- let pendingEvent: CustomEvent | undefined
700
+ let pendingEvent: CustomEvent | StreamChunk | undefined
637
701
  while ((pendingEvent = pendingEvents.shift()) !== undefined) {
638
702
  yield pendingEvent
639
703
  }
@@ -671,7 +735,7 @@ export async function* executeServerTool<TContext = unknown>(
671
735
  const duration = Date.now() - startTime
672
736
 
673
737
  // Flush remaining events
674
- let pendingEvent: CustomEvent | undefined
738
+ let pendingEvent: CustomEvent | StreamChunk | undefined
675
739
  while ((pendingEvent = pendingEvents.shift()) !== undefined) {
676
740
  yield pendingEvent
677
741
  }
@@ -767,10 +831,11 @@ export async function* executeToolCalls<TContext = unknown>(
767
831
  userContext?: TContext,
768
832
  abortSignal?: AbortSignal,
769
833
  resumeState?: ToolResumeExecutionState,
770
- ): AsyncGenerator<CustomEvent, ExecuteToolCallsResult, void> {
834
+ ): AsyncGenerator<CustomEvent | StreamChunk, ExecuteToolCallsResult, void> {
771
835
  const results: Array<ToolResult> = []
772
836
  const needsApproval: Array<ApprovalRequest> = []
773
837
  const needsClientExecution: Array<ClientToolRequest> = []
838
+ const subagentInterrupts: Array<Interrupt> = []
774
839
 
775
840
  // Create tool lookup map
776
841
  const toolMap = new Map<string, AnyTool>()
@@ -790,6 +855,11 @@ export async function* executeToolCalls<TContext = unknown>(
790
855
  })
791
856
 
792
857
  for (const toolCall of toolCalls) {
858
+ // Provider-executed tools (Anthropic web_search / web_fetch) already ran
859
+ // inside the provider response and carry their result on the call's
860
+ // metadata. They have no execute() and must not become client requests.
861
+ if (isProviderExecutedToolCall(toolCall)) continue
862
+
793
863
  const tool = toolMap.get(toolCall.function.name)
794
864
  const toolName = toolCall.function.name
795
865
 
@@ -871,7 +941,7 @@ export async function* executeToolCalls<TContext = unknown>(
871
941
  }
872
942
 
873
943
  // Create a ToolExecutionContext for this tool call with event emission
874
- const pendingEvents: Array<CustomEvent> = []
944
+ const pendingEvents: Array<CustomEvent | StreamChunk> = []
875
945
  const context = {
876
946
  toolCallId: toolCall.id,
877
947
  context: userContext,
@@ -1007,6 +1077,7 @@ export async function* executeToolCalls<TContext = unknown>(
1007
1077
  pendingEvents,
1008
1078
  results,
1009
1079
  middlewareHooks,
1080
+ subagentInterrupts,
1010
1081
  )
1011
1082
  } else {
1012
1083
  // User declined
@@ -1055,8 +1126,9 @@ export async function* executeToolCalls<TContext = unknown>(
1055
1126
  pendingEvents,
1056
1127
  results,
1057
1128
  middlewareHooks,
1129
+ subagentInterrupts,
1058
1130
  )
1059
1131
  }
1060
1132
 
1061
- return { results, needsApproval, needsClientExecution }
1133
+ return { results, needsApproval, needsClientExecution, subagentInterrupts }
1062
1134
  }
@@ -39,6 +39,12 @@ export interface EmbeddingAdapter<
39
39
  readonly kind: 'embedding'
40
40
  /** Adapter name identifier */
41
41
  readonly name: string
42
+ /**
43
+ * Declares that this adapter can consume `{ type: 'file' }` content
44
+ * sources (provider Files API references). `embed()` rejects file sources
45
+ * in preflight for adapters that don't declare this.
46
+ */
47
+ readonly supportsFileSources?: boolean
42
48
  /** The model this adapter is configured for */
43
49
  readonly model: TModel
44
50
 
@@ -85,6 +91,7 @@ export abstract class BaseEmbeddingAdapter<
85
91
  > {
86
92
  readonly kind = 'embedding' as const
87
93
  abstract readonly name: string
94
+ readonly supportsFileSources: boolean = false
88
95
  readonly model: TModel
89
96
 
90
97
  // Type-only property - never assigned at runtime
@@ -15,6 +15,7 @@ import {
15
15
  runGenerationUsage,
16
16
  } from '../middleware/run'
17
17
  import { countEmbeddingInputModalities } from '../../utilities/embedding-input'
18
+ import { assertPromptFileSourceSupport } from '../../utilities/content-source'
18
19
  import type { InternalLogger } from '../../logger/internal-logger'
19
20
  import type { DebugOption } from '../../logger/types'
20
21
  import type { GenerationMiddleware } from '../middleware/types'
@@ -190,6 +191,10 @@ export async function embed<
190
191
  TAdapter extends EmbeddingAdapter<string, any, any, any>,
191
192
  >(options: EmbedOptions<TAdapter>): Promise<EmbeddingResult> {
192
193
  const { adapter, middleware } = options
194
+ // Fail closed on `{ type: 'file' }` sources before middleware start. No
195
+ // embedding adapter consumes file handles today; this matches chat /
196
+ // generateImage / generateVideo.
197
+ assertPromptFileSourceSupport(adapter, options.input)
193
198
  const model = adapter.model
194
199
  const requestId = createId('embedding')
195
200
  const startTime = Date.now()
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Files Adapter
3
+ *
4
+ * Base class and interface for the `files` activity — a provider's native Files
5
+ * API (upload a media asset once, reference it later by the returned handle
6
+ * instead of re-sending base64 or a public URL each request).
7
+ *
8
+ * Providers with a native surface expose a factory (`openaiFiles()`,
9
+ * `anthropicFiles()`, `geminiFiles()`, `falFiles()`). `upload` is required;
10
+ * `get`/`delete` are optional because not every provider has a lifecycle API
11
+ * (fal's storage is upload-only).
12
+ */
13
+
14
+ import { base64ToArrayBuffer } from '@tanstack/ai-utils'
15
+
16
+ /**
17
+ * Input to {@link FilesAdapter.upload}. Either a `Blob` (memory-efficient,
18
+ * preferred for large assets) or base64 `data` plus its `mimeType`.
19
+ */
20
+ export type FileUploadInput =
21
+ | Blob
22
+ | {
23
+ /** Base64-encoded file bytes. */
24
+ data: string
25
+ /** MIME type of the bytes (e.g. `'image/png'`, `'application/pdf'`). */
26
+ mimeType: string
27
+ /** Optional filename hint sent to providers that accept one. */
28
+ filename?: string
29
+ }
30
+
31
+ /**
32
+ * A provider-issued file handle returned by {@link FilesAdapter.upload} /
33
+ * {@link FilesAdapter.get}. Reference it in a message via a `{ type: 'file' }`
34
+ * content source — use `fileSourceFromHandle` to build one.
35
+ *
36
+ * `TProvider` carries the issuing provider's name as a literal (`'openai'`,
37
+ * `'gemini'`, ...) when the handle came from a concrete files adapter, so
38
+ * cross-provider lifecycle calls (`deleteFile` with a foreign handle) fail at
39
+ * compile time. It defaults to `string` so wire-deserialized handles still fit.
40
+ */
41
+ export interface FileHandle<TProvider extends string = string> {
42
+ /**
43
+ * Provider handle used for lifecycle operations (`get`/`delete`): the
44
+ * OpenAI/Anthropic `file_id`, the Gemini file resource name (`files/...`), or
45
+ * the fal storage URL (fal itself has no lifecycle API — the URL doubles as
46
+ * the wire reference).
47
+ */
48
+ id: string
49
+ /** The provider that issued the handle (`'openai'`, `'gemini'`, ...). */
50
+ provider: TProvider
51
+ /**
52
+ * The handle's URL form when the provider exposes one (Gemini file URI, fal
53
+ * storage URL). For providers whose handle is an opaque id (OpenAI,
54
+ * Anthropic) this is `undefined`.
55
+ */
56
+ uri?: string
57
+ /** MIME type reported by the provider (or echoed from the upload input). */
58
+ mimeType?: string
59
+ /** File size in bytes when the provider reports it. */
60
+ sizeBytes?: number
61
+ /** Expiry as epoch milliseconds when the handle is scheduled to expire. */
62
+ expiresAt?: number
63
+ /** Original filename when the provider reports it. */
64
+ filename?: string
65
+ }
66
+
67
+ /**
68
+ * The `files` adapter contract. `upload` is required; `get`/`delete` are
69
+ * optional and present only when the provider has a lifecycle API.
70
+ *
71
+ * `TName` is the provider name literal (`'openai'`, `'gemini'`, ...); concrete
72
+ * adapters bind it so the handles they issue carry their provenance in the
73
+ * type system.
74
+ */
75
+ export interface FilesAdapter<TName extends string = string> {
76
+ readonly kind: 'files'
77
+ readonly name: TName
78
+ upload: (input: FileUploadInput) => Promise<FileHandle<TName>>
79
+ get?: (id: string) => Promise<FileHandle<TName>>
80
+ delete?: (id: string) => Promise<void>
81
+ }
82
+
83
+ export type AnyFilesAdapter = FilesAdapter<string>
84
+
85
+ /**
86
+ * Normalize a {@link FileUploadInput} to a `Blob` (plus best-effort MIME /
87
+ * filename) so provider adapters can hand it straight to their SDK. A `Blob`
88
+ * input passes through; base64 `{ data }` is decoded to bytes. Shared so
89
+ * provider files adapters don't each re-implement the decode.
90
+ */
91
+ export function normalizeFileUploadInput(input: FileUploadInput): {
92
+ blob: Blob
93
+ mimeType?: string
94
+ filename?: string
95
+ } {
96
+ if (input instanceof Blob) {
97
+ return { blob: input, mimeType: input.type || undefined }
98
+ }
99
+ const bytes = base64ToArrayBuffer(input.data)
100
+ return {
101
+ blob: new Blob([bytes], { type: input.mimeType }),
102
+ mimeType: input.mimeType,
103
+ filename: input.filename,
104
+ }
105
+ }
106
+
107
+ /**
108
+ * Abstract base for provider files adapters. Subclasses bind `TName` to their
109
+ * provider literal, set `name`, implement `upload`, and may add `get`/`delete`
110
+ * (declared on {@link FilesAdapter}, not here, since not every provider has a
111
+ * lifecycle API).
112
+ */
113
+ export abstract class BaseFilesAdapter<
114
+ TName extends string = string,
115
+ > implements FilesAdapter<TName> {
116
+ readonly kind = 'files' as const
117
+ abstract readonly name: TName
118
+
119
+ abstract upload(input: FileUploadInput): Promise<FileHandle<TName>>
120
+ }
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Files Activity
3
+ *
4
+ * Dispatch functions for provider Files APIs. Each takes `{ adapter, ... }` and
5
+ * calls the adapter method directly (mirrors the other activity dispatchers).
6
+ * `get`/`delete` are optional on the adapter; the dispatchers throw a clear
7
+ * error when the selected provider has no lifecycle API.
8
+ */
9
+
10
+ import type { ContentPartFileSource } from '../../types'
11
+ import type { FileHandle, FileUploadInput, FilesAdapter } from './adapter'
12
+
13
+ /** The adapter kind this activity handles */
14
+ export const kind = 'files' as const
15
+
16
+ /**
17
+ * Upload a file to a provider's Files API and return its handle. The handle
18
+ * carries the provider name as a literal type, so passing it to another
19
+ * provider's lifecycle call is a compile error.
20
+ *
21
+ * @example
22
+ * ```ts
23
+ * const files = openaiFiles()
24
+ * const handle = await uploadFile({ adapter: files, input: { data, mimeType: 'image/png' } })
25
+ * ```
26
+ */
27
+ export async function uploadFile<TName extends string>(options: {
28
+ adapter: FilesAdapter<TName> & { kind: typeof kind }
29
+ input: FileUploadInput
30
+ }): Promise<FileHandle<TName>> {
31
+ return options.adapter.upload(options.input)
32
+ }
33
+
34
+ /**
35
+ * Resolve a lifecycle id from either a raw id string or a {@link FileHandle}
36
+ * (whose `id` — not its `uri`/wire value — is the lifecycle currency).
37
+ */
38
+ function toLifecycleId(id: string | FileHandle): string {
39
+ return typeof id === 'string' ? id : id.id
40
+ }
41
+
42
+ /**
43
+ * Fetch metadata for a previously uploaded file. Accepts the handle itself
44
+ * (preferred — the provider-literal type rejects a foreign provider's handle
45
+ * at compile time) or its raw lifecycle id.
46
+ *
47
+ * @throws if the provider's files adapter has no `get` (e.g. fal storage).
48
+ */
49
+ export async function getFile<TName extends string>(options: {
50
+ adapter: FilesAdapter<TName> & { kind: typeof kind }
51
+ // `NoInfer` so `TName` comes from the adapter only. Otherwise a foreign
52
+ // handle widens it to a union and the call compiles.
53
+ id: string | FileHandle<NoInfer<TName>>
54
+ }): Promise<FileHandle<TName>> {
55
+ const { adapter } = options
56
+ if (!adapter.get) {
57
+ throw new Error(
58
+ `${adapter.name}: files adapter does not support get() — this provider ` +
59
+ `has no file-retrieval API.`,
60
+ )
61
+ }
62
+ return adapter.get(toLifecycleId(options.id))
63
+ }
64
+
65
+ /**
66
+ * Delete a previously uploaded file. Accepts the handle itself (preferred —
67
+ * the provider-literal type rejects a foreign provider's handle at compile
68
+ * time) or its raw lifecycle id.
69
+ *
70
+ * @throws if the provider's files adapter has no `delete` (e.g. fal storage).
71
+ */
72
+ export async function deleteFile<TName extends string>(options: {
73
+ adapter: FilesAdapter<TName> & { kind: typeof kind }
74
+ id: string | FileHandle<NoInfer<TName>>
75
+ }): Promise<void> {
76
+ const { adapter } = options
77
+ if (!adapter.delete) {
78
+ throw new Error(
79
+ `${adapter.name}: files adapter does not support delete() — this ` +
80
+ `provider has no file-deletion API.`,
81
+ )
82
+ }
83
+ return adapter.delete(toLifecycleId(options.id))
84
+ }
85
+
86
+ /**
87
+ * Build a `{ type: 'file' }` content source from an uploaded
88
+ * {@link FileHandle}, for use in a chat message (image/audio/document part
89
+ * `source`).
90
+ *
91
+ * The source's `value` is the handle's wire form: the handle URL when the
92
+ * provider exposes one (Gemini, fal, Grok), otherwise the opaque id (OpenAI,
93
+ * Anthropic). `provider` records the issuer, so an adapter for a different
94
+ * provider rejects the source rather than sending a handle it cannot resolve.
95
+ *
96
+ * @example
97
+ * ```ts
98
+ * const handle = await uploadFile({ adapter: openaiFiles(), input })
99
+ * messages.push({ role: 'user', content: [
100
+ * { type: 'image', source: fileSourceFromHandle(handle) },
101
+ * ] })
102
+ * ```
103
+ */
104
+ export function fileSourceFromHandle<TProvider extends string>(
105
+ handle: FileHandle<TProvider>,
106
+ ): ContentPartFileSource<TProvider> {
107
+ return {
108
+ type: 'file',
109
+ value: handle.uri ?? handle.id,
110
+ provider: handle.provider,
111
+ ...(handle.mimeType ? { mimeType: handle.mimeType } : {}),
112
+ }
113
+ }
@@ -51,6 +51,13 @@ export interface ImageAdapter<
51
51
  readonly kind: 'image'
52
52
  /** Adapter name identifier */
53
53
  readonly name: string
54
+ /**
55
+ * Declares that this adapter can consume `{ type: 'file' }` content
56
+ * sources (provider Files API references). The activity dispatcher rejects
57
+ * file sources in preflight for adapters that don't declare this, so
58
+ * adapters written before the file arm existed fail closed.
59
+ */
60
+ readonly supportsFileSources?: boolean
54
61
  /** The model this adapter is configured for */
55
62
  readonly model: TModel
56
63
 
@@ -103,6 +110,7 @@ export abstract class BaseImageAdapter<
103
110
  > {
104
111
  readonly kind = 'image' as const
105
112
  abstract readonly name: string
113
+ readonly supportsFileSources: boolean = false
106
114
  readonly model: TModel
107
115
 
108
116
  // Type-only property - never assigned at runtime
@@ -24,6 +24,7 @@ import {
24
24
  raceWithAbort,
25
25
  } from '../../utilities/activity-abort'
26
26
  import { resolveMediaPrompt } from '../../utilities/media-prompt'
27
+ import { assertPromptFileSourceSupport } from '../../utilities/content-source'
27
28
  import type { InternalLogger } from '../../logger/internal-logger'
28
29
  import type { DebugOption } from '../../logger/types'
29
30
  import type { GenerationMiddleware } from '../middleware/types'
@@ -248,6 +249,9 @@ export function generateImage<
248
249
  >(
249
250
  options: ImageActivityOptions<TAdapter, TStream>,
250
251
  ): ImageActivityResult<TStream> {
252
+ // Fail closed before middleware start and before `stream: true` emits
253
+ // RUN_STARTED, so an unsupported file source never opens a run.
254
+ assertPromptFileSourceSupport(options.adapter, options.prompt)
251
255
  if (options.stream) {
252
256
  return streamGenerationResult(
253
257
  // Only `runId` is taken from the resolved wire identity. `threadId` stays
@@ -74,6 +74,13 @@ export interface VideoAdapter<
74
74
  readonly kind: 'video'
75
75
  /** Adapter name identifier */
76
76
  readonly name: string
77
+ /**
78
+ * Declares that this adapter can consume `{ type: 'file' }` content
79
+ * sources (provider Files API references). The activity dispatcher rejects
80
+ * file sources in preflight for adapters that don't declare this, so
81
+ * adapters written before the file arm existed fail closed.
82
+ */
83
+ readonly supportsFileSources?: boolean
77
84
  /** The model this adapter is configured for */
78
85
  readonly model: TModel
79
86
 
@@ -161,6 +168,7 @@ export abstract class BaseVideoAdapter<
161
168
  > {
162
169
  readonly kind = 'video' as const
163
170
  abstract readonly name: string
171
+ readonly supportsFileSources: boolean = false
164
172
  readonly model: TModel
165
173
 
166
174
  // Type-only property - never assigned at runtime
@@ -12,6 +12,7 @@
12
12
  import { aiEventClient } from '@tanstack/ai-event-client'
13
13
  import { toRunErrorPayload } from '../error-payload'
14
14
  import { resolveDebugOption } from '../../logger/resolve'
15
+ import { assertPromptFileSourceSupport } from '../../utilities/content-source'
15
16
  import {
16
17
  applyGenerationResultTransforms,
17
18
  createGenerationContext,
@@ -452,6 +453,9 @@ async function runCreateVideoJob<
452
453
  timeout,
453
454
  abortSignal: callerAbortSignal,
454
455
  } = options
456
+ // Fail closed on `{ type: 'file' }` sources for adapters that haven't
457
+ // declared support (see assertPromptFileSourceSupport).
458
+ assertPromptFileSourceSupport(adapter, prompt)
455
459
  const model = adapter.model
456
460
  const requestId = createId('video')
457
461
  const startTime = Date.now()
@@ -580,6 +584,9 @@ async function* runStreamingVideoGeneration<
580
584
  timeout,
581
585
  abortSignal: callerAbortSignal,
582
586
  } = options
587
+ // Fail closed on `{ type: 'file' }` sources for adapters that haven't
588
+ // declared support (see assertPromptFileSourceSupport).
589
+ assertPromptFileSourceSupport(adapter, prompt)
583
590
  const model = adapter.model
584
591
  const runId = options.runId ?? createId('run')
585
592
  const requestId = createId('video')
@@ -44,10 +44,12 @@ export interface WorldAdapter<
44
44
  }
45
45
 
46
46
  /**
47
- * Open a world session from a prompt.
47
+ * Create a world from a prompt.
48
48
  *
49
- * Server adapters typically mint a short-lived token and return it with the
49
+ * Live session adapters mint a short-lived token and return it with the
50
50
  * prompt so a browser can connect, set the prompt, and start streaming.
51
+ * Job adapters start generation and return a world URL (or an operation
52
+ * id while the job is still running).
51
53
  */
52
54
  createWorld: (
53
55
  options: WorldGenerationOptions<TProviderOptions>,
@@ -1,9 +1,9 @@
1
1
  /**
2
2
  * World Activity (Experimental)
3
3
  *
4
- * Mints a session token for a live, prompt-steerable world. Unlike
5
- * generateVideo (a job that finishes with a URL), the browser then connects
6
- * with the token, sets the prompt, and streams until pause/reset/close.
4
+ * Live adapters mint a session token for a prompt-steerable world. Job
5
+ * adapters start generation and return a viewer URL, or an operation id
6
+ * while the job is still running.
7
7
  *
8
8
  * @experimental World generation is an experimental feature and may change.
9
9
  */
@@ -76,7 +76,7 @@ export interface WorldActivityOptions<
76
76
  /** Provider-specific options for world generation */
77
77
  modelOptions?: WorldProviderOptions<TAdapter>
78
78
  /**
79
- * Whether to wrap the token result as StreamChunks for SSE transport.
79
+ * Whether to wrap the result as StreamChunks for SSE transport.
80
80
  * This is not the live video. When false or omitted, returns
81
81
  * Promise<WorldGenerationResult>.
82
82
  *
@@ -100,7 +100,7 @@ export interface WorldActivityOptions<
100
100
  /** Stable run id for correlating this run when persisted. */
101
101
  runId?: string
102
102
  /**
103
- * Maximum duration of the token mint in milliseconds.
103
+ * Maximum wait for the mint or job poll, in milliseconds.
104
104
  * No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
105
105
  */
106
106
  timeout?: number
@@ -135,7 +135,8 @@ function createId(prefix: string): string {
135
135
  // ===========================
136
136
 
137
137
  /**
138
- * World generation activity - opens a live, prompt-steerable world session.
138
+ * World generation activity. Live adapters mint a session token. Job
139
+ * adapters return a viewer URL or an in-progress operation id.
139
140
  *
140
141
  * @example Mint a session token on the server
141
142
  * ```ts