@tanstack/ai 0.58.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/activities/chat/adapter.d.ts +9 -0
- package/dist/esm/activities/chat/adapter.js +1 -0
- package/dist/esm/activities/chat/adapter.js.map +1 -1
- package/dist/esm/activities/chat/agents/define-agent.d.ts +81 -0
- package/dist/esm/activities/chat/agents/define-agent.js +34 -0
- package/dist/esm/activities/chat/agents/define-agent.js.map +1 -0
- package/dist/esm/activities/chat/agents/route.d.ts +53 -0
- package/dist/esm/activities/chat/agents/route.js +59 -0
- package/dist/esm/activities/chat/agents/route.js.map +1 -0
- package/dist/esm/activities/chat/agents/spawn.d.ts +124 -0
- package/dist/esm/activities/chat/agents/spawn.js +490 -0
- package/dist/esm/activities/chat/agents/spawn.js.map +1 -0
- package/dist/esm/activities/chat/agents/turn.d.ts +36 -0
- package/dist/esm/activities/chat/agents/turn.js +78 -0
- package/dist/esm/activities/chat/agents/turn.js.map +1 -0
- package/dist/esm/activities/chat/index.d.ts +13 -3
- package/dist/esm/activities/chat/index.js +462 -26
- package/dist/esm/activities/chat/index.js.map +1 -1
- package/dist/esm/activities/chat/messages.d.ts +7 -1
- package/dist/esm/activities/chat/messages.js +99 -19
- package/dist/esm/activities/chat/messages.js.map +1 -1
- package/dist/esm/activities/chat/middleware/run-store.d.ts +43 -7
- package/dist/esm/activities/chat/middleware/run-store.js +8 -1
- package/dist/esm/activities/chat/middleware/run-store.js.map +1 -1
- package/dist/esm/activities/chat/middleware/types.d.ts +47 -1
- package/dist/esm/activities/chat/middleware/types.js.map +1 -1
- package/dist/esm/activities/chat/stream/message-updaters.js +9 -2
- package/dist/esm/activities/chat/stream/message-updaters.js.map +1 -1
- package/dist/esm/activities/chat/stream/processor.d.ts +46 -1
- package/dist/esm/activities/chat/stream/processor.js +294 -18
- package/dist/esm/activities/chat/stream/processor.js.map +1 -1
- package/dist/esm/activities/chat/tools/tool-calls.d.ts +17 -3
- package/dist/esm/activities/chat/tools/tool-calls.js +56 -5
- package/dist/esm/activities/chat/tools/tool-calls.js.map +1 -1
- package/dist/esm/activities/embed/adapter.d.ts +7 -0
- package/dist/esm/activities/embed/adapter.js +1 -0
- package/dist/esm/activities/embed/adapter.js.map +1 -1
- package/dist/esm/activities/embed/index.js +2 -0
- package/dist/esm/activities/embed/index.js.map +1 -1
- package/dist/esm/activities/files/adapter.d.ts +97 -0
- package/dist/esm/activities/files/adapter.js +45 -0
- package/dist/esm/activities/files/adapter.js.map +1 -0
- package/dist/esm/activities/files/index.d.ts +66 -0
- package/dist/esm/activities/files/index.js +78 -0
- package/dist/esm/activities/files/index.js.map +1 -0
- package/dist/esm/activities/generateAudio/index.js +1 -1
- package/dist/esm/activities/generateImage/adapter.d.ts +8 -0
- package/dist/esm/activities/generateImage/adapter.js +1 -0
- package/dist/esm/activities/generateImage/adapter.js.map +1 -1
- package/dist/esm/activities/generateImage/index.js +3 -1
- package/dist/esm/activities/generateImage/index.js.map +1 -1
- package/dist/esm/activities/generateLiveVideo/index.js +1 -1
- package/dist/esm/activities/generateSpeech/index.js +1 -1
- package/dist/esm/activities/generateTranscription/index.js +1 -1
- package/dist/esm/activities/generateVideo/adapter.d.ts +8 -0
- package/dist/esm/activities/generateVideo/adapter.js +1 -0
- package/dist/esm/activities/generateVideo/adapter.js.map +1 -1
- package/dist/esm/activities/generateVideo/index.js +3 -0
- package/dist/esm/activities/generateVideo/index.js.map +1 -1
- package/dist/esm/activities/generateVoice/index.js +1 -1
- package/dist/esm/activities/generateWorld/adapter.d.ts +4 -2
- package/dist/esm/activities/generateWorld/adapter.js.map +1 -1
- package/dist/esm/activities/generateWorld/index.d.ts +4 -3
- package/dist/esm/activities/generateWorld/index.js +6 -5
- package/dist/esm/activities/generateWorld/index.js.map +1 -1
- package/dist/esm/activities/index.d.ts +9 -3
- package/dist/esm/activities/index.js +17 -13
- package/dist/esm/activities/summarize/chat-stream-summarize.d.ts +2 -0
- package/dist/esm/activities/summarize/chat-stream-summarize.js +8 -8
- package/dist/esm/activities/summarize/chat-stream-summarize.js.map +1 -1
- package/dist/esm/activities/summarize/index.js +1 -1
- package/dist/esm/client.d.ts +7 -36
- package/dist/esm/client.js +5 -37
- package/dist/esm/client.js.map +1 -1
- package/dist/esm/index.d.ts +8 -2
- package/dist/esm/index.js +9 -4
- package/dist/esm/middlewares/content-guard.js.map +1 -1
- package/dist/esm/types.d.ts +179 -98
- package/dist/esm/utilities/adapter-yield-chunk.d.ts +5 -1
- package/dist/esm/utilities/ag-ui-usage.d.ts +9 -9
- package/dist/esm/utilities/ag-ui-usage.js +66 -3
- package/dist/esm/utilities/ag-ui-usage.js.map +1 -1
- package/dist/esm/utilities/ag-ui-wire.js +90 -13
- package/dist/esm/utilities/ag-ui-wire.js.map +1 -1
- package/dist/esm/utilities/content-source.d.ts +60 -0
- package/dist/esm/utilities/content-source.js +85 -0
- package/dist/esm/utilities/content-source.js.map +1 -0
- package/dist/esm/utilities/normalize-stream-chunk.js +7 -2
- package/dist/esm/utilities/normalize-stream-chunk.js.map +1 -1
- package/dist/esm/utilities/provider-executed.d.ts +7 -0
- package/dist/esm/utilities/provider-executed.js +10 -1
- package/dist/esm/utilities/provider-executed.js.map +1 -1
- package/dist/esm/utilities/spec-event-keys.js +13 -8
- package/dist/esm/utilities/spec-event-keys.js.map +1 -1
- package/dist/esm/utilities/subagent-wire.d.ts +36 -0
- package/dist/esm/utilities/subagent-wire.js +131 -0
- package/dist/esm/utilities/subagent-wire.js.map +1 -0
- package/dist/esm/utilities/tool-result.d.ts +12 -2
- package/dist/esm/utilities/tool-result.js +23 -3
- package/dist/esm/utilities/tool-result.js.map +1 -1
- package/package.json +3 -3
- package/skills/ai-core/adapter-configuration/SKILL.md +62 -0
- package/skills/ai-core/adapter-configuration/references/grok-adapter.md +1 -1
- package/skills/ai-core/chat-experience/SKILL.md +14 -0
- package/skills/ai-core/media-generation/SKILL.md +10 -2
- package/skills/ai-core/middleware/SKILL.md +7 -4
- package/src/activities/chat/adapter.ts +10 -0
- package/src/activities/chat/agents/define-agent.ts +121 -0
- package/src/activities/chat/agents/route.ts +115 -0
- package/src/activities/chat/agents/spawn.ts +806 -0
- package/src/activities/chat/agents/turn.ts +151 -0
- package/src/activities/chat/index.ts +734 -30
- package/src/activities/chat/messages.ts +137 -16
- package/src/activities/chat/middleware/run-store.ts +56 -7
- package/src/activities/chat/middleware/types.ts +47 -0
- package/src/activities/chat/stream/message-updaters.ts +24 -2
- package/src/activities/chat/stream/processor.ts +452 -30
- package/src/activities/chat/tools/tool-calls.ts +83 -11
- package/src/activities/embed/adapter.ts +7 -0
- package/src/activities/embed/index.ts +5 -0
- package/src/activities/files/adapter.ts +120 -0
- package/src/activities/files/index.ts +113 -0
- package/src/activities/generateImage/adapter.ts +8 -0
- package/src/activities/generateImage/index.ts +4 -0
- package/src/activities/generateVideo/adapter.ts +8 -0
- package/src/activities/generateVideo/index.ts +7 -0
- package/src/activities/generateWorld/adapter.ts +4 -2
- package/src/activities/generateWorld/index.ts +7 -6
- package/src/activities/index.ts +40 -1
- package/src/activities/summarize/chat-stream-summarize.ts +22 -12
- package/src/client.ts +29 -35
- package/src/index.ts +39 -0
- package/src/middlewares/content-guard.ts +7 -5
- package/src/types.ts +226 -103
- package/src/utilities/adapter-yield-chunk.ts +10 -2
- package/src/utilities/ag-ui-usage.test.ts +38 -0
- package/src/utilities/ag-ui-usage.ts +98 -11
- package/src/utilities/ag-ui-wire.ts +134 -16
- package/src/utilities/content-source.ts +138 -0
- package/src/utilities/normalize-stream-chunk.ts +10 -2
- package/src/utilities/provider-executed.ts +13 -0
- package/src/utilities/spec-event-keys.ts +34 -7
- package/src/utilities/subagent-wire.ts +184 -0
- package/src/utilities/tool-result.ts +38 -2
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { normalizeToolResult } from '../../../utilities/tool-result'
|
|
2
2
|
import { tanstackMetadata } from '../../../utilities/merge-metadata'
|
|
3
|
+
import { isProviderExecutedToolCall } from '../../../utilities/provider-executed'
|
|
3
4
|
import type { AdapterYieldChunk } from '../../../utilities/adapter-yield-chunk'
|
|
4
5
|
import { isStandardSchema, parseWithStandardSchema } from './schema-converter'
|
|
5
6
|
import type { ToolApprovalResolution } from '../../../interrupts'
|
|
@@ -8,8 +9,10 @@ import type {
|
|
|
8
9
|
ContentPart,
|
|
9
10
|
CustomEvent,
|
|
10
11
|
EmitCustomEventOptions,
|
|
12
|
+
Interrupt,
|
|
11
13
|
ModelMessage,
|
|
12
14
|
RunFinishedEvent,
|
|
15
|
+
StreamChunk,
|
|
13
16
|
Tool,
|
|
14
17
|
ToolCall,
|
|
15
18
|
ToolCallArgsEvent,
|
|
@@ -38,6 +41,25 @@ function safeJsonParse(value: string): unknown {
|
|
|
38
41
|
}
|
|
39
42
|
}
|
|
40
43
|
|
|
44
|
+
/** Marks the synthetic tool that runs a subagent. */
|
|
45
|
+
export const SUBAGENT_TOOL = Symbol.for('tanstack.ai.subagentTool')
|
|
46
|
+
|
|
47
|
+
/** Set on the tool context of a subagent tool. Streams a child chunk live. */
|
|
48
|
+
export const EMIT_STREAM_CHUNK = Symbol.for('tanstack.ai.emitStreamChunk')
|
|
49
|
+
|
|
50
|
+
/** What a subagent tool's `execute` returns. */
|
|
51
|
+
export interface SubagentToolOutcome {
|
|
52
|
+
subagentRunId: string
|
|
53
|
+
text: string
|
|
54
|
+
error?: string
|
|
55
|
+
/** Set when the child stopped for outside input. The tool call stays open. */
|
|
56
|
+
interrupts?: Array<Interrupt>
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function isSubagentTool(tool: AnyTool): boolean {
|
|
60
|
+
return (tool as { [SUBAGENT_TOOL]?: true })[SUBAGENT_TOOL] === true
|
|
61
|
+
}
|
|
62
|
+
|
|
41
63
|
/**
|
|
42
64
|
* MCP Apps metadata attached to a server tool at discovery (see
|
|
43
65
|
* `@tanstack/ai-mcp` discovery + `MCPManager.discover()`).
|
|
@@ -505,6 +527,8 @@ interface ExecuteToolCallsResult {
|
|
|
505
527
|
needsApproval: Array<ApprovalRequest>
|
|
506
528
|
/** Tools that need client-side execution */
|
|
507
529
|
needsClientExecution: Array<ClientToolRequest>
|
|
530
|
+
/** Interrupts raised by subagents that run as tools */
|
|
531
|
+
subagentInterrupts: Array<Interrupt>
|
|
508
532
|
}
|
|
509
533
|
|
|
510
534
|
/**
|
|
@@ -514,8 +538,8 @@ interface ExecuteToolCallsResult {
|
|
|
514
538
|
*/
|
|
515
539
|
async function* executeWithEventPolling<T>(
|
|
516
540
|
executionPromise: Promise<T>,
|
|
517
|
-
pendingEvents: Array<CustomEvent>,
|
|
518
|
-
): AsyncGenerator<CustomEvent, T, void> {
|
|
541
|
+
pendingEvents: Array<CustomEvent | StreamChunk>,
|
|
542
|
+
): AsyncGenerator<CustomEvent | StreamChunk, T, void> {
|
|
519
543
|
// Use an object to track mutable state across the async boundary
|
|
520
544
|
const state = { done: false, result: undefined as T }
|
|
521
545
|
const executionWithFlag = executionPromise.then((r) => {
|
|
@@ -532,14 +556,14 @@ async function* executeWithEventPolling<T>(
|
|
|
532
556
|
])
|
|
533
557
|
|
|
534
558
|
// Flush any pending events
|
|
535
|
-
let event: CustomEvent | undefined
|
|
559
|
+
let event: CustomEvent | StreamChunk | undefined
|
|
536
560
|
while ((event = pendingEvents.shift()) !== undefined) {
|
|
537
561
|
yield event
|
|
538
562
|
}
|
|
539
563
|
}
|
|
540
564
|
|
|
541
565
|
// Final flush in case events were emitted right at completion
|
|
542
|
-
let event: CustomEvent | undefined
|
|
566
|
+
let event: CustomEvent | StreamChunk | undefined
|
|
543
567
|
while ((event = pendingEvents.shift()) !== undefined) {
|
|
544
568
|
yield event
|
|
545
569
|
}
|
|
@@ -612,19 +636,59 @@ export async function* executeServerTool<TContext = unknown>(
|
|
|
612
636
|
toolName: string,
|
|
613
637
|
input: unknown,
|
|
614
638
|
context: ToolExecutionContext<TContext>,
|
|
615
|
-
pendingEvents: Array<CustomEvent>,
|
|
639
|
+
pendingEvents: Array<CustomEvent | StreamChunk>,
|
|
616
640
|
results: Array<ToolResult>,
|
|
617
641
|
middlewareHooks?: ToolExecutionMiddlewareHooks,
|
|
618
|
-
|
|
642
|
+
subagentInterrupts?: Array<Interrupt>,
|
|
643
|
+
): AsyncGenerator<CustomEvent | StreamChunk, void, void> {
|
|
619
644
|
const startTime = Date.now()
|
|
620
645
|
try {
|
|
621
646
|
if (!tool.execute) {
|
|
622
647
|
throw new Error(`Tool ${toolName} has no execute() implementation`)
|
|
623
648
|
}
|
|
649
|
+
const subagent = isSubagentTool(tool)
|
|
650
|
+
if (subagent) {
|
|
651
|
+
Object.assign(context, {
|
|
652
|
+
[EMIT_STREAM_CHUNK]: (chunk: StreamChunk) => pendingEvents.push(chunk),
|
|
653
|
+
})
|
|
654
|
+
}
|
|
624
655
|
const executionPromise = Promise.resolve(tool.execute(input, context))
|
|
625
656
|
let result = yield* executeWithEventPolling(executionPromise, pendingEvents)
|
|
626
657
|
const duration = Date.now() - startTime
|
|
627
658
|
|
|
659
|
+
if (subagent) {
|
|
660
|
+
const outcome = result as SubagentToolOutcome
|
|
661
|
+
if (outcome.interrupts?.length) {
|
|
662
|
+
// The child waits for outside input. Leave the call open so the
|
|
663
|
+
// resume run executes it again and continues the same child.
|
|
664
|
+
subagentInterrupts?.push(...outcome.interrupts)
|
|
665
|
+
return
|
|
666
|
+
}
|
|
667
|
+
const modelResult = outcome.error
|
|
668
|
+
? { subagentRunId: outcome.subagentRunId, error: outcome.error }
|
|
669
|
+
: { subagentRunId: outcome.subagentRunId, result: outcome.text }
|
|
670
|
+
results.push({
|
|
671
|
+
toolCallId: toolCall.id,
|
|
672
|
+
toolName,
|
|
673
|
+
result: modelResult,
|
|
674
|
+
input,
|
|
675
|
+
output: modelResult,
|
|
676
|
+
duration,
|
|
677
|
+
...(outcome.error ? { state: 'output-error' as const } : {}),
|
|
678
|
+
})
|
|
679
|
+
await middlewareHooks?.onAfterToolCall?.({
|
|
680
|
+
toolCall,
|
|
681
|
+
tool,
|
|
682
|
+
toolName,
|
|
683
|
+
toolCallId: toolCall.id,
|
|
684
|
+
duration,
|
|
685
|
+
...(outcome.error
|
|
686
|
+
? { ok: false as const, error: new Error(outcome.error) }
|
|
687
|
+
: { ok: true as const, result: modelResult }),
|
|
688
|
+
})
|
|
689
|
+
return
|
|
690
|
+
}
|
|
691
|
+
|
|
628
692
|
// MCP Apps: if this tool links a ui:// resource, eagerly read it and queue
|
|
629
693
|
// a `ui-resource` CUSTOM event. The MCP source stays live until the run
|
|
630
694
|
// drains (MCPManager's `connection:'close'` policy disposes on completion),
|
|
@@ -633,7 +697,7 @@ export async function* executeServerTool<TContext = unknown>(
|
|
|
633
697
|
await emitUiResourceIfLinked(tool, context)
|
|
634
698
|
|
|
635
699
|
// Flush remaining events (including any queued ui-resource event)
|
|
636
|
-
let pendingEvent: CustomEvent | undefined
|
|
700
|
+
let pendingEvent: CustomEvent | StreamChunk | undefined
|
|
637
701
|
while ((pendingEvent = pendingEvents.shift()) !== undefined) {
|
|
638
702
|
yield pendingEvent
|
|
639
703
|
}
|
|
@@ -671,7 +735,7 @@ export async function* executeServerTool<TContext = unknown>(
|
|
|
671
735
|
const duration = Date.now() - startTime
|
|
672
736
|
|
|
673
737
|
// Flush remaining events
|
|
674
|
-
let pendingEvent: CustomEvent | undefined
|
|
738
|
+
let pendingEvent: CustomEvent | StreamChunk | undefined
|
|
675
739
|
while ((pendingEvent = pendingEvents.shift()) !== undefined) {
|
|
676
740
|
yield pendingEvent
|
|
677
741
|
}
|
|
@@ -767,10 +831,11 @@ export async function* executeToolCalls<TContext = unknown>(
|
|
|
767
831
|
userContext?: TContext,
|
|
768
832
|
abortSignal?: AbortSignal,
|
|
769
833
|
resumeState?: ToolResumeExecutionState,
|
|
770
|
-
): AsyncGenerator<CustomEvent, ExecuteToolCallsResult, void> {
|
|
834
|
+
): AsyncGenerator<CustomEvent | StreamChunk, ExecuteToolCallsResult, void> {
|
|
771
835
|
const results: Array<ToolResult> = []
|
|
772
836
|
const needsApproval: Array<ApprovalRequest> = []
|
|
773
837
|
const needsClientExecution: Array<ClientToolRequest> = []
|
|
838
|
+
const subagentInterrupts: Array<Interrupt> = []
|
|
774
839
|
|
|
775
840
|
// Create tool lookup map
|
|
776
841
|
const toolMap = new Map<string, AnyTool>()
|
|
@@ -790,6 +855,11 @@ export async function* executeToolCalls<TContext = unknown>(
|
|
|
790
855
|
})
|
|
791
856
|
|
|
792
857
|
for (const toolCall of toolCalls) {
|
|
858
|
+
// Provider-executed tools (Anthropic web_search / web_fetch) already ran
|
|
859
|
+
// inside the provider response and carry their result on the call's
|
|
860
|
+
// metadata. They have no execute() and must not become client requests.
|
|
861
|
+
if (isProviderExecutedToolCall(toolCall)) continue
|
|
862
|
+
|
|
793
863
|
const tool = toolMap.get(toolCall.function.name)
|
|
794
864
|
const toolName = toolCall.function.name
|
|
795
865
|
|
|
@@ -871,7 +941,7 @@ export async function* executeToolCalls<TContext = unknown>(
|
|
|
871
941
|
}
|
|
872
942
|
|
|
873
943
|
// Create a ToolExecutionContext for this tool call with event emission
|
|
874
|
-
const pendingEvents: Array<CustomEvent> = []
|
|
944
|
+
const pendingEvents: Array<CustomEvent | StreamChunk> = []
|
|
875
945
|
const context = {
|
|
876
946
|
toolCallId: toolCall.id,
|
|
877
947
|
context: userContext,
|
|
@@ -1007,6 +1077,7 @@ export async function* executeToolCalls<TContext = unknown>(
|
|
|
1007
1077
|
pendingEvents,
|
|
1008
1078
|
results,
|
|
1009
1079
|
middlewareHooks,
|
|
1080
|
+
subagentInterrupts,
|
|
1010
1081
|
)
|
|
1011
1082
|
} else {
|
|
1012
1083
|
// User declined
|
|
@@ -1055,8 +1126,9 @@ export async function* executeToolCalls<TContext = unknown>(
|
|
|
1055
1126
|
pendingEvents,
|
|
1056
1127
|
results,
|
|
1057
1128
|
middlewareHooks,
|
|
1129
|
+
subagentInterrupts,
|
|
1058
1130
|
)
|
|
1059
1131
|
}
|
|
1060
1132
|
|
|
1061
|
-
return { results, needsApproval, needsClientExecution }
|
|
1133
|
+
return { results, needsApproval, needsClientExecution, subagentInterrupts }
|
|
1062
1134
|
}
|
|
@@ -39,6 +39,12 @@ export interface EmbeddingAdapter<
|
|
|
39
39
|
readonly kind: 'embedding'
|
|
40
40
|
/** Adapter name identifier */
|
|
41
41
|
readonly name: string
|
|
42
|
+
/**
|
|
43
|
+
* Declares that this adapter can consume `{ type: 'file' }` content
|
|
44
|
+
* sources (provider Files API references). `embed()` rejects file sources
|
|
45
|
+
* in preflight for adapters that don't declare this.
|
|
46
|
+
*/
|
|
47
|
+
readonly supportsFileSources?: boolean
|
|
42
48
|
/** The model this adapter is configured for */
|
|
43
49
|
readonly model: TModel
|
|
44
50
|
|
|
@@ -85,6 +91,7 @@ export abstract class BaseEmbeddingAdapter<
|
|
|
85
91
|
> {
|
|
86
92
|
readonly kind = 'embedding' as const
|
|
87
93
|
abstract readonly name: string
|
|
94
|
+
readonly supportsFileSources: boolean = false
|
|
88
95
|
readonly model: TModel
|
|
89
96
|
|
|
90
97
|
// Type-only property - never assigned at runtime
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
runGenerationUsage,
|
|
16
16
|
} from '../middleware/run'
|
|
17
17
|
import { countEmbeddingInputModalities } from '../../utilities/embedding-input'
|
|
18
|
+
import { assertPromptFileSourceSupport } from '../../utilities/content-source'
|
|
18
19
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
19
20
|
import type { DebugOption } from '../../logger/types'
|
|
20
21
|
import type { GenerationMiddleware } from '../middleware/types'
|
|
@@ -190,6 +191,10 @@ export async function embed<
|
|
|
190
191
|
TAdapter extends EmbeddingAdapter<string, any, any, any>,
|
|
191
192
|
>(options: EmbedOptions<TAdapter>): Promise<EmbeddingResult> {
|
|
192
193
|
const { adapter, middleware } = options
|
|
194
|
+
// Fail closed on `{ type: 'file' }` sources before middleware start. No
|
|
195
|
+
// embedding adapter consumes file handles today; this matches chat /
|
|
196
|
+
// generateImage / generateVideo.
|
|
197
|
+
assertPromptFileSourceSupport(adapter, options.input)
|
|
193
198
|
const model = adapter.model
|
|
194
199
|
const requestId = createId('embedding')
|
|
195
200
|
const startTime = Date.now()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files Adapter
|
|
3
|
+
*
|
|
4
|
+
* Base class and interface for the `files` activity — a provider's native Files
|
|
5
|
+
* API (upload a media asset once, reference it later by the returned handle
|
|
6
|
+
* instead of re-sending base64 or a public URL each request).
|
|
7
|
+
*
|
|
8
|
+
* Providers with a native surface expose a factory (`openaiFiles()`,
|
|
9
|
+
* `anthropicFiles()`, `geminiFiles()`, `falFiles()`). `upload` is required;
|
|
10
|
+
* `get`/`delete` are optional because not every provider has a lifecycle API
|
|
11
|
+
* (fal's storage is upload-only).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { base64ToArrayBuffer } from '@tanstack/ai-utils'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Input to {@link FilesAdapter.upload}. Either a `Blob` (memory-efficient,
|
|
18
|
+
* preferred for large assets) or base64 `data` plus its `mimeType`.
|
|
19
|
+
*/
|
|
20
|
+
export type FileUploadInput =
|
|
21
|
+
| Blob
|
|
22
|
+
| {
|
|
23
|
+
/** Base64-encoded file bytes. */
|
|
24
|
+
data: string
|
|
25
|
+
/** MIME type of the bytes (e.g. `'image/png'`, `'application/pdf'`). */
|
|
26
|
+
mimeType: string
|
|
27
|
+
/** Optional filename hint sent to providers that accept one. */
|
|
28
|
+
filename?: string
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A provider-issued file handle returned by {@link FilesAdapter.upload} /
|
|
33
|
+
* {@link FilesAdapter.get}. Reference it in a message via a `{ type: 'file' }`
|
|
34
|
+
* content source — use `fileSourceFromHandle` to build one.
|
|
35
|
+
*
|
|
36
|
+
* `TProvider` carries the issuing provider's name as a literal (`'openai'`,
|
|
37
|
+
* `'gemini'`, ...) when the handle came from a concrete files adapter, so
|
|
38
|
+
* cross-provider lifecycle calls (`deleteFile` with a foreign handle) fail at
|
|
39
|
+
* compile time. It defaults to `string` so wire-deserialized handles still fit.
|
|
40
|
+
*/
|
|
41
|
+
export interface FileHandle<TProvider extends string = string> {
|
|
42
|
+
/**
|
|
43
|
+
* Provider handle used for lifecycle operations (`get`/`delete`): the
|
|
44
|
+
* OpenAI/Anthropic `file_id`, the Gemini file resource name (`files/...`), or
|
|
45
|
+
* the fal storage URL (fal itself has no lifecycle API — the URL doubles as
|
|
46
|
+
* the wire reference).
|
|
47
|
+
*/
|
|
48
|
+
id: string
|
|
49
|
+
/** The provider that issued the handle (`'openai'`, `'gemini'`, ...). */
|
|
50
|
+
provider: TProvider
|
|
51
|
+
/**
|
|
52
|
+
* The handle's URL form when the provider exposes one (Gemini file URI, fal
|
|
53
|
+
* storage URL). For providers whose handle is an opaque id (OpenAI,
|
|
54
|
+
* Anthropic) this is `undefined`.
|
|
55
|
+
*/
|
|
56
|
+
uri?: string
|
|
57
|
+
/** MIME type reported by the provider (or echoed from the upload input). */
|
|
58
|
+
mimeType?: string
|
|
59
|
+
/** File size in bytes when the provider reports it. */
|
|
60
|
+
sizeBytes?: number
|
|
61
|
+
/** Expiry as epoch milliseconds when the handle is scheduled to expire. */
|
|
62
|
+
expiresAt?: number
|
|
63
|
+
/** Original filename when the provider reports it. */
|
|
64
|
+
filename?: string
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The `files` adapter contract. `upload` is required; `get`/`delete` are
|
|
69
|
+
* optional and present only when the provider has a lifecycle API.
|
|
70
|
+
*
|
|
71
|
+
* `TName` is the provider name literal (`'openai'`, `'gemini'`, ...); concrete
|
|
72
|
+
* adapters bind it so the handles they issue carry their provenance in the
|
|
73
|
+
* type system.
|
|
74
|
+
*/
|
|
75
|
+
export interface FilesAdapter<TName extends string = string> {
|
|
76
|
+
readonly kind: 'files'
|
|
77
|
+
readonly name: TName
|
|
78
|
+
upload: (input: FileUploadInput) => Promise<FileHandle<TName>>
|
|
79
|
+
get?: (id: string) => Promise<FileHandle<TName>>
|
|
80
|
+
delete?: (id: string) => Promise<void>
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export type AnyFilesAdapter = FilesAdapter<string>
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Normalize a {@link FileUploadInput} to a `Blob` (plus best-effort MIME /
|
|
87
|
+
* filename) so provider adapters can hand it straight to their SDK. A `Blob`
|
|
88
|
+
* input passes through; base64 `{ data }` is decoded to bytes. Shared so
|
|
89
|
+
* provider files adapters don't each re-implement the decode.
|
|
90
|
+
*/
|
|
91
|
+
export function normalizeFileUploadInput(input: FileUploadInput): {
|
|
92
|
+
blob: Blob
|
|
93
|
+
mimeType?: string
|
|
94
|
+
filename?: string
|
|
95
|
+
} {
|
|
96
|
+
if (input instanceof Blob) {
|
|
97
|
+
return { blob: input, mimeType: input.type || undefined }
|
|
98
|
+
}
|
|
99
|
+
const bytes = base64ToArrayBuffer(input.data)
|
|
100
|
+
return {
|
|
101
|
+
blob: new Blob([bytes], { type: input.mimeType }),
|
|
102
|
+
mimeType: input.mimeType,
|
|
103
|
+
filename: input.filename,
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Abstract base for provider files adapters. Subclasses bind `TName` to their
|
|
109
|
+
* provider literal, set `name`, implement `upload`, and may add `get`/`delete`
|
|
110
|
+
* (declared on {@link FilesAdapter}, not here, since not every provider has a
|
|
111
|
+
* lifecycle API).
|
|
112
|
+
*/
|
|
113
|
+
export abstract class BaseFilesAdapter<
|
|
114
|
+
TName extends string = string,
|
|
115
|
+
> implements FilesAdapter<TName> {
|
|
116
|
+
readonly kind = 'files' as const
|
|
117
|
+
abstract readonly name: TName
|
|
118
|
+
|
|
119
|
+
abstract upload(input: FileUploadInput): Promise<FileHandle<TName>>
|
|
120
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files Activity
|
|
3
|
+
*
|
|
4
|
+
* Dispatch functions for provider Files APIs. Each takes `{ adapter, ... }` and
|
|
5
|
+
* calls the adapter method directly (mirrors the other activity dispatchers).
|
|
6
|
+
* `get`/`delete` are optional on the adapter; the dispatchers throw a clear
|
|
7
|
+
* error when the selected provider has no lifecycle API.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { ContentPartFileSource } from '../../types'
|
|
11
|
+
import type { FileHandle, FileUploadInput, FilesAdapter } from './adapter'
|
|
12
|
+
|
|
13
|
+
/** The adapter kind this activity handles */
|
|
14
|
+
export const kind = 'files' as const
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Upload a file to a provider's Files API and return its handle. The handle
|
|
18
|
+
* carries the provider name as a literal type, so passing it to another
|
|
19
|
+
* provider's lifecycle call is a compile error.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```ts
|
|
23
|
+
* const files = openaiFiles()
|
|
24
|
+
* const handle = await uploadFile({ adapter: files, input: { data, mimeType: 'image/png' } })
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export async function uploadFile<TName extends string>(options: {
|
|
28
|
+
adapter: FilesAdapter<TName> & { kind: typeof kind }
|
|
29
|
+
input: FileUploadInput
|
|
30
|
+
}): Promise<FileHandle<TName>> {
|
|
31
|
+
return options.adapter.upload(options.input)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Resolve a lifecycle id from either a raw id string or a {@link FileHandle}
|
|
36
|
+
* (whose `id` — not its `uri`/wire value — is the lifecycle currency).
|
|
37
|
+
*/
|
|
38
|
+
function toLifecycleId(id: string | FileHandle): string {
|
|
39
|
+
return typeof id === 'string' ? id : id.id
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Fetch metadata for a previously uploaded file. Accepts the handle itself
|
|
44
|
+
* (preferred — the provider-literal type rejects a foreign provider's handle
|
|
45
|
+
* at compile time) or its raw lifecycle id.
|
|
46
|
+
*
|
|
47
|
+
* @throws if the provider's files adapter has no `get` (e.g. fal storage).
|
|
48
|
+
*/
|
|
49
|
+
export async function getFile<TName extends string>(options: {
|
|
50
|
+
adapter: FilesAdapter<TName> & { kind: typeof kind }
|
|
51
|
+
// `NoInfer` so `TName` comes from the adapter only. Otherwise a foreign
|
|
52
|
+
// handle widens it to a union and the call compiles.
|
|
53
|
+
id: string | FileHandle<NoInfer<TName>>
|
|
54
|
+
}): Promise<FileHandle<TName>> {
|
|
55
|
+
const { adapter } = options
|
|
56
|
+
if (!adapter.get) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`${adapter.name}: files adapter does not support get() — this provider ` +
|
|
59
|
+
`has no file-retrieval API.`,
|
|
60
|
+
)
|
|
61
|
+
}
|
|
62
|
+
return adapter.get(toLifecycleId(options.id))
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Delete a previously uploaded file. Accepts the handle itself (preferred —
|
|
67
|
+
* the provider-literal type rejects a foreign provider's handle at compile
|
|
68
|
+
* time) or its raw lifecycle id.
|
|
69
|
+
*
|
|
70
|
+
* @throws if the provider's files adapter has no `delete` (e.g. fal storage).
|
|
71
|
+
*/
|
|
72
|
+
export async function deleteFile<TName extends string>(options: {
|
|
73
|
+
adapter: FilesAdapter<TName> & { kind: typeof kind }
|
|
74
|
+
id: string | FileHandle<NoInfer<TName>>
|
|
75
|
+
}): Promise<void> {
|
|
76
|
+
const { adapter } = options
|
|
77
|
+
if (!adapter.delete) {
|
|
78
|
+
throw new Error(
|
|
79
|
+
`${adapter.name}: files adapter does not support delete() — this ` +
|
|
80
|
+
`provider has no file-deletion API.`,
|
|
81
|
+
)
|
|
82
|
+
}
|
|
83
|
+
return adapter.delete(toLifecycleId(options.id))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Build a `{ type: 'file' }` content source from an uploaded
|
|
88
|
+
* {@link FileHandle}, for use in a chat message (image/audio/document part
|
|
89
|
+
* `source`).
|
|
90
|
+
*
|
|
91
|
+
* The source's `value` is the handle's wire form: the handle URL when the
|
|
92
|
+
* provider exposes one (Gemini, fal, Grok), otherwise the opaque id (OpenAI,
|
|
93
|
+
* Anthropic). `provider` records the issuer, so an adapter for a different
|
|
94
|
+
* provider rejects the source rather than sending a handle it cannot resolve.
|
|
95
|
+
*
|
|
96
|
+
* @example
|
|
97
|
+
* ```ts
|
|
98
|
+
* const handle = await uploadFile({ adapter: openaiFiles(), input })
|
|
99
|
+
* messages.push({ role: 'user', content: [
|
|
100
|
+
* { type: 'image', source: fileSourceFromHandle(handle) },
|
|
101
|
+
* ] })
|
|
102
|
+
* ```
|
|
103
|
+
*/
|
|
104
|
+
export function fileSourceFromHandle<TProvider extends string>(
|
|
105
|
+
handle: FileHandle<TProvider>,
|
|
106
|
+
): ContentPartFileSource<TProvider> {
|
|
107
|
+
return {
|
|
108
|
+
type: 'file',
|
|
109
|
+
value: handle.uri ?? handle.id,
|
|
110
|
+
provider: handle.provider,
|
|
111
|
+
...(handle.mimeType ? { mimeType: handle.mimeType } : {}),
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -51,6 +51,13 @@ export interface ImageAdapter<
|
|
|
51
51
|
readonly kind: 'image'
|
|
52
52
|
/** Adapter name identifier */
|
|
53
53
|
readonly name: string
|
|
54
|
+
/**
|
|
55
|
+
* Declares that this adapter can consume `{ type: 'file' }` content
|
|
56
|
+
* sources (provider Files API references). The activity dispatcher rejects
|
|
57
|
+
* file sources in preflight for adapters that don't declare this, so
|
|
58
|
+
* adapters written before the file arm existed fail closed.
|
|
59
|
+
*/
|
|
60
|
+
readonly supportsFileSources?: boolean
|
|
54
61
|
/** The model this adapter is configured for */
|
|
55
62
|
readonly model: TModel
|
|
56
63
|
|
|
@@ -103,6 +110,7 @@ export abstract class BaseImageAdapter<
|
|
|
103
110
|
> {
|
|
104
111
|
readonly kind = 'image' as const
|
|
105
112
|
abstract readonly name: string
|
|
113
|
+
readonly supportsFileSources: boolean = false
|
|
106
114
|
readonly model: TModel
|
|
107
115
|
|
|
108
116
|
// Type-only property - never assigned at runtime
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
raceWithAbort,
|
|
25
25
|
} from '../../utilities/activity-abort'
|
|
26
26
|
import { resolveMediaPrompt } from '../../utilities/media-prompt'
|
|
27
|
+
import { assertPromptFileSourceSupport } from '../../utilities/content-source'
|
|
27
28
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
28
29
|
import type { DebugOption } from '../../logger/types'
|
|
29
30
|
import type { GenerationMiddleware } from '../middleware/types'
|
|
@@ -248,6 +249,9 @@ export function generateImage<
|
|
|
248
249
|
>(
|
|
249
250
|
options: ImageActivityOptions<TAdapter, TStream>,
|
|
250
251
|
): ImageActivityResult<TStream> {
|
|
252
|
+
// Fail closed before middleware start and before `stream: true` emits
|
|
253
|
+
// RUN_STARTED, so an unsupported file source never opens a run.
|
|
254
|
+
assertPromptFileSourceSupport(options.adapter, options.prompt)
|
|
251
255
|
if (options.stream) {
|
|
252
256
|
return streamGenerationResult(
|
|
253
257
|
// Only `runId` is taken from the resolved wire identity. `threadId` stays
|
|
@@ -74,6 +74,13 @@ export interface VideoAdapter<
|
|
|
74
74
|
readonly kind: 'video'
|
|
75
75
|
/** Adapter name identifier */
|
|
76
76
|
readonly name: string
|
|
77
|
+
/**
|
|
78
|
+
* Declares that this adapter can consume `{ type: 'file' }` content
|
|
79
|
+
* sources (provider Files API references). The activity dispatcher rejects
|
|
80
|
+
* file sources in preflight for adapters that don't declare this, so
|
|
81
|
+
* adapters written before the file arm existed fail closed.
|
|
82
|
+
*/
|
|
83
|
+
readonly supportsFileSources?: boolean
|
|
77
84
|
/** The model this adapter is configured for */
|
|
78
85
|
readonly model: TModel
|
|
79
86
|
|
|
@@ -161,6 +168,7 @@ export abstract class BaseVideoAdapter<
|
|
|
161
168
|
> {
|
|
162
169
|
readonly kind = 'video' as const
|
|
163
170
|
abstract readonly name: string
|
|
171
|
+
readonly supportsFileSources: boolean = false
|
|
164
172
|
readonly model: TModel
|
|
165
173
|
|
|
166
174
|
// Type-only property - never assigned at runtime
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { aiEventClient } from '@tanstack/ai-event-client'
|
|
13
13
|
import { toRunErrorPayload } from '../error-payload'
|
|
14
14
|
import { resolveDebugOption } from '../../logger/resolve'
|
|
15
|
+
import { assertPromptFileSourceSupport } from '../../utilities/content-source'
|
|
15
16
|
import {
|
|
16
17
|
applyGenerationResultTransforms,
|
|
17
18
|
createGenerationContext,
|
|
@@ -452,6 +453,9 @@ async function runCreateVideoJob<
|
|
|
452
453
|
timeout,
|
|
453
454
|
abortSignal: callerAbortSignal,
|
|
454
455
|
} = options
|
|
456
|
+
// Fail closed on `{ type: 'file' }` sources for adapters that haven't
|
|
457
|
+
// declared support (see assertPromptFileSourceSupport).
|
|
458
|
+
assertPromptFileSourceSupport(adapter, prompt)
|
|
455
459
|
const model = adapter.model
|
|
456
460
|
const requestId = createId('video')
|
|
457
461
|
const startTime = Date.now()
|
|
@@ -580,6 +584,9 @@ async function* runStreamingVideoGeneration<
|
|
|
580
584
|
timeout,
|
|
581
585
|
abortSignal: callerAbortSignal,
|
|
582
586
|
} = options
|
|
587
|
+
// Fail closed on `{ type: 'file' }` sources for adapters that haven't
|
|
588
|
+
// declared support (see assertPromptFileSourceSupport).
|
|
589
|
+
assertPromptFileSourceSupport(adapter, prompt)
|
|
583
590
|
const model = adapter.model
|
|
584
591
|
const runId = options.runId ?? createId('run')
|
|
585
592
|
const requestId = createId('video')
|
|
@@ -44,10 +44,12 @@ export interface WorldAdapter<
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
/**
|
|
47
|
-
*
|
|
47
|
+
* Create a world from a prompt.
|
|
48
48
|
*
|
|
49
|
-
*
|
|
49
|
+
* Live session adapters mint a short-lived token and return it with the
|
|
50
50
|
* prompt so a browser can connect, set the prompt, and start streaming.
|
|
51
|
+
* Job adapters start generation and return a world URL (or an operation
|
|
52
|
+
* id while the job is still running).
|
|
51
53
|
*/
|
|
52
54
|
createWorld: (
|
|
53
55
|
options: WorldGenerationOptions<TProviderOptions>,
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* World Activity (Experimental)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Live adapters mint a session token for a prompt-steerable world. Job
|
|
5
|
+
* adapters start generation and return a viewer URL, or an operation id
|
|
6
|
+
* while the job is still running.
|
|
7
7
|
*
|
|
8
8
|
* @experimental World generation is an experimental feature and may change.
|
|
9
9
|
*/
|
|
@@ -76,7 +76,7 @@ export interface WorldActivityOptions<
|
|
|
76
76
|
/** Provider-specific options for world generation */
|
|
77
77
|
modelOptions?: WorldProviderOptions<TAdapter>
|
|
78
78
|
/**
|
|
79
|
-
* Whether to wrap the
|
|
79
|
+
* Whether to wrap the result as StreamChunks for SSE transport.
|
|
80
80
|
* This is not the live video. When false or omitted, returns
|
|
81
81
|
* Promise<WorldGenerationResult>.
|
|
82
82
|
*
|
|
@@ -100,7 +100,7 @@ export interface WorldActivityOptions<
|
|
|
100
100
|
/** Stable run id for correlating this run when persisted. */
|
|
101
101
|
runId?: string
|
|
102
102
|
/**
|
|
103
|
-
* Maximum
|
|
103
|
+
* Maximum wait for the mint or job poll, in milliseconds.
|
|
104
104
|
* No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
|
|
105
105
|
*/
|
|
106
106
|
timeout?: number
|
|
@@ -135,7 +135,8 @@ function createId(prefix: string): string {
|
|
|
135
135
|
// ===========================
|
|
136
136
|
|
|
137
137
|
/**
|
|
138
|
-
* World generation activity
|
|
138
|
+
* World generation activity. Live adapters mint a session token. Job
|
|
139
|
+
* adapters return a viewer URL or an in-progress operation id.
|
|
139
140
|
*
|
|
140
141
|
* @example Mint a session token on the server
|
|
141
142
|
* ```ts
|