@tanstack/ai 0.59.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/activities/chat/adapter.d.ts +9 -0
- package/dist/esm/activities/chat/adapter.js +1 -0
- package/dist/esm/activities/chat/adapter.js.map +1 -1
- package/dist/esm/activities/chat/index.js +134 -22
- package/dist/esm/activities/chat/index.js.map +1 -1
- package/dist/esm/activities/chat/messages.js +5 -1
- package/dist/esm/activities/chat/messages.js.map +1 -1
- package/dist/esm/activities/chat/stream/message-updaters.js +9 -2
- package/dist/esm/activities/chat/stream/message-updaters.js.map +1 -1
- package/dist/esm/activities/chat/stream/processor.d.ts +0 -1
- package/dist/esm/activities/chat/stream/processor.js +15 -12
- package/dist/esm/activities/chat/stream/processor.js.map +1 -1
- package/dist/esm/activities/chat/tools/tool-calls.js +2 -0
- package/dist/esm/activities/chat/tools/tool-calls.js.map +1 -1
- package/dist/esm/activities/embed/adapter.d.ts +7 -0
- package/dist/esm/activities/embed/adapter.js +1 -0
- package/dist/esm/activities/embed/adapter.js.map +1 -1
- package/dist/esm/activities/embed/index.js +2 -0
- package/dist/esm/activities/embed/index.js.map +1 -1
- package/dist/esm/activities/files/adapter.d.ts +97 -0
- package/dist/esm/activities/files/adapter.js +45 -0
- package/dist/esm/activities/files/adapter.js.map +1 -0
- package/dist/esm/activities/files/index.d.ts +66 -0
- package/dist/esm/activities/files/index.js +78 -0
- package/dist/esm/activities/files/index.js.map +1 -0
- package/dist/esm/activities/generateImage/adapter.d.ts +8 -0
- package/dist/esm/activities/generateImage/adapter.js +1 -0
- package/dist/esm/activities/generateImage/adapter.js.map +1 -1
- package/dist/esm/activities/generateImage/index.js +2 -0
- package/dist/esm/activities/generateImage/index.js.map +1 -1
- package/dist/esm/activities/generateVideo/adapter.d.ts +8 -0
- package/dist/esm/activities/generateVideo/adapter.js +1 -0
- package/dist/esm/activities/generateVideo/adapter.js.map +1 -1
- package/dist/esm/activities/generateVideo/index.js +3 -0
- package/dist/esm/activities/generateVideo/index.js.map +1 -1
- package/dist/esm/activities/generateWorld/adapter.d.ts +4 -2
- package/dist/esm/activities/generateWorld/adapter.js.map +1 -1
- package/dist/esm/activities/generateWorld/index.d.ts +4 -3
- package/dist/esm/activities/generateWorld/index.js +5 -4
- package/dist/esm/activities/generateWorld/index.js.map +1 -1
- package/dist/esm/activities/index.d.ts +6 -3
- package/dist/esm/activities/index.js +13 -11
- package/dist/esm/activities/summarize/chat-stream-summarize.d.ts +2 -0
- package/dist/esm/activities/summarize/chat-stream-summarize.js +8 -8
- package/dist/esm/activities/summarize/chat-stream-summarize.js.map +1 -1
- package/dist/esm/client.d.ts +2 -0
- package/dist/esm/client.js +2 -1
- package/dist/esm/client.js.map +1 -1
- package/dist/esm/index.d.ts +3 -2
- package/dist/esm/index.js +4 -2
- package/dist/esm/types.d.ts +72 -13
- package/dist/esm/utilities/ag-ui-wire.js +34 -13
- package/dist/esm/utilities/ag-ui-wire.js.map +1 -1
- package/dist/esm/utilities/content-source.d.ts +60 -0
- package/dist/esm/utilities/content-source.js +85 -0
- package/dist/esm/utilities/content-source.js.map +1 -0
- package/dist/esm/utilities/provider-executed.d.ts +7 -0
- package/dist/esm/utilities/provider-executed.js +10 -1
- package/dist/esm/utilities/provider-executed.js.map +1 -1
- package/dist/esm/utilities/tool-result.d.ts +12 -2
- package/dist/esm/utilities/tool-result.js +23 -3
- package/dist/esm/utilities/tool-result.js.map +1 -1
- package/package.json +2 -2
- package/skills/ai-core/adapter-configuration/SKILL.md +62 -0
- package/skills/ai-core/chat-experience/SKILL.md +14 -0
- package/skills/ai-core/media-generation/SKILL.md +8 -0
- package/src/activities/chat/adapter.ts +10 -0
- package/src/activities/chat/index.ts +226 -40
- package/src/activities/chat/messages.ts +12 -1
- package/src/activities/chat/stream/message-updaters.ts +24 -2
- package/src/activities/chat/stream/processor.ts +24 -22
- package/src/activities/chat/tools/tool-calls.ts +6 -0
- package/src/activities/embed/adapter.ts +7 -0
- package/src/activities/embed/index.ts +5 -0
- package/src/activities/files/adapter.ts +120 -0
- package/src/activities/files/index.ts +113 -0
- package/src/activities/generateImage/adapter.ts +8 -0
- package/src/activities/generateImage/index.ts +4 -0
- package/src/activities/generateVideo/adapter.ts +8 -0
- package/src/activities/generateVideo/index.ts +7 -0
- package/src/activities/generateWorld/adapter.ts +4 -2
- package/src/activities/generateWorld/index.ts +7 -6
- package/src/activities/index.ts +25 -1
- package/src/activities/summarize/chat-stream-summarize.ts +22 -12
- package/src/client.ts +7 -0
- package/src/index.ts +16 -0
- package/src/types.ts +76 -13
- package/src/utilities/ag-ui-wire.ts +60 -16
- package/src/utilities/content-source.ts +138 -0
- package/src/utilities/provider-executed.ts +13 -0
- package/src/utilities/tool-result.ts +38 -2
|
@@ -25,14 +25,20 @@ import {
|
|
|
25
25
|
uiMessageToModelMessages,
|
|
26
26
|
} from '../messages.js'
|
|
27
27
|
import { runErrorEventToError } from '../../../utilities/errors'
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
isAssistantSegmentOf,
|
|
30
|
+
isProviderExecutedToolCall,
|
|
31
|
+
} from '../../../utilities/provider-executed'
|
|
29
32
|
import {
|
|
30
33
|
mergeMetadata,
|
|
31
34
|
tanstackMetadata,
|
|
32
35
|
} from '../../../utilities/merge-metadata'
|
|
33
36
|
import { getChunkRunId } from '../../../utilities/chunk-ids'
|
|
34
37
|
import type { AdapterYieldChunk } from '../../../utilities/adapter-yield-chunk'
|
|
35
|
-
import {
|
|
38
|
+
import {
|
|
39
|
+
normalizeToolResult,
|
|
40
|
+
toolResultErrorText,
|
|
41
|
+
} from '../../../utilities/tool-result'
|
|
36
42
|
import { defaultJSONParser } from './json-parser'
|
|
37
43
|
import {
|
|
38
44
|
appendStructuredOutputDelta,
|
|
@@ -1518,20 +1524,30 @@ export class StreamProcessor {
|
|
|
1518
1524
|
pending.push(msg)
|
|
1519
1525
|
continue
|
|
1520
1526
|
}
|
|
1527
|
+
let next = msg
|
|
1521
1528
|
if (
|
|
1522
1529
|
msg.role === 'assistant' &&
|
|
1523
1530
|
pending.length > 0 &&
|
|
1524
1531
|
!isToolResultOnly(msg)
|
|
1525
1532
|
) {
|
|
1526
|
-
|
|
1533
|
+
next = {
|
|
1527
1534
|
...msg,
|
|
1528
1535
|
parts: [...pending.flatMap(thinkingParts), ...msg.parts],
|
|
1529
|
-
}
|
|
1536
|
+
}
|
|
1530
1537
|
pending = []
|
|
1538
|
+
} else {
|
|
1539
|
+
flushPending()
|
|
1540
|
+
}
|
|
1541
|
+
const prev = out.at(-1)
|
|
1542
|
+
if (
|
|
1543
|
+
prev?.role === 'assistant' &&
|
|
1544
|
+
next.role === 'assistant' &&
|
|
1545
|
+
isAssistantSegmentOf(next.id, prev.id)
|
|
1546
|
+
) {
|
|
1547
|
+
out[out.length - 1] = { ...prev, parts: [...prev.parts, ...next.parts] }
|
|
1531
1548
|
continue
|
|
1532
1549
|
}
|
|
1533
|
-
|
|
1534
|
-
out.push(msg)
|
|
1550
|
+
out.push(next)
|
|
1535
1551
|
}
|
|
1536
1552
|
flushPending()
|
|
1537
1553
|
return out
|
|
@@ -1705,9 +1721,7 @@ export class StreamProcessor {
|
|
|
1705
1721
|
}
|
|
1706
1722
|
}
|
|
1707
1723
|
const errorText =
|
|
1708
|
-
result.state === 'error'
|
|
1709
|
-
? this.extractToolResultError(output)
|
|
1710
|
-
: undefined
|
|
1724
|
+
result.state === 'error' ? toolResultErrorText(output) : undefined
|
|
1711
1725
|
next = {
|
|
1712
1726
|
...next,
|
|
1713
1727
|
output: errorText ? { error: errorText } : output,
|
|
@@ -2001,18 +2015,6 @@ export class StreamProcessor {
|
|
|
2001
2015
|
}
|
|
2002
2016
|
}
|
|
2003
2017
|
|
|
2004
|
-
private extractToolResultError(output: unknown): string {
|
|
2005
|
-
if (
|
|
2006
|
-
output &&
|
|
2007
|
-
typeof output === 'object' &&
|
|
2008
|
-
'error' in output &&
|
|
2009
|
-
typeof output.error === 'string'
|
|
2010
|
-
) {
|
|
2011
|
-
return output.error
|
|
2012
|
-
}
|
|
2013
|
-
return typeof output === 'string' ? output : 'Tool execution failed'
|
|
2014
|
-
}
|
|
2015
|
-
|
|
2016
2018
|
/**
|
|
2017
2019
|
* Handle TOOL_CALL_RESULT event (AG-UI spec).
|
|
2018
2020
|
*
|
|
@@ -2069,7 +2071,7 @@ export class StreamProcessor {
|
|
|
2069
2071
|
chunk.toolCallId,
|
|
2070
2072
|
aguiContentToContentParts(chunk.content),
|
|
2071
2073
|
resultState,
|
|
2072
|
-
resultState === 'error' ?
|
|
2074
|
+
resultState === 'error' ? toolResultErrorText(output) : undefined,
|
|
2073
2075
|
)
|
|
2074
2076
|
this.emitMessagesChange()
|
|
2075
2077
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { normalizeToolResult } from '../../../utilities/tool-result'
|
|
2
2
|
import { tanstackMetadata } from '../../../utilities/merge-metadata'
|
|
3
|
+
import { isProviderExecutedToolCall } from '../../../utilities/provider-executed'
|
|
3
4
|
import type { AdapterYieldChunk } from '../../../utilities/adapter-yield-chunk'
|
|
4
5
|
import { isStandardSchema, parseWithStandardSchema } from './schema-converter'
|
|
5
6
|
import type { ToolApprovalResolution } from '../../../interrupts'
|
|
@@ -854,6 +855,11 @@ export async function* executeToolCalls<TContext = unknown>(
|
|
|
854
855
|
})
|
|
855
856
|
|
|
856
857
|
for (const toolCall of toolCalls) {
|
|
858
|
+
// Provider-executed tools (Anthropic web_search / web_fetch) already ran
|
|
859
|
+
// inside the provider response and carry their result on the call's
|
|
860
|
+
// metadata. They have no execute() and must not become client requests.
|
|
861
|
+
if (isProviderExecutedToolCall(toolCall)) continue
|
|
862
|
+
|
|
857
863
|
const tool = toolMap.get(toolCall.function.name)
|
|
858
864
|
const toolName = toolCall.function.name
|
|
859
865
|
|
|
@@ -39,6 +39,12 @@ export interface EmbeddingAdapter<
|
|
|
39
39
|
readonly kind: 'embedding'
|
|
40
40
|
/** Adapter name identifier */
|
|
41
41
|
readonly name: string
|
|
42
|
+
/**
|
|
43
|
+
* Declares that this adapter can consume `{ type: 'file' }` content
|
|
44
|
+
* sources (provider Files API references). `embed()` rejects file sources
|
|
45
|
+
* in preflight for adapters that don't declare this.
|
|
46
|
+
*/
|
|
47
|
+
readonly supportsFileSources?: boolean
|
|
42
48
|
/** The model this adapter is configured for */
|
|
43
49
|
readonly model: TModel
|
|
44
50
|
|
|
@@ -85,6 +91,7 @@ export abstract class BaseEmbeddingAdapter<
|
|
|
85
91
|
> {
|
|
86
92
|
readonly kind = 'embedding' as const
|
|
87
93
|
abstract readonly name: string
|
|
94
|
+
readonly supportsFileSources: boolean = false
|
|
88
95
|
readonly model: TModel
|
|
89
96
|
|
|
90
97
|
// Type-only property - never assigned at runtime
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
runGenerationUsage,
|
|
16
16
|
} from '../middleware/run'
|
|
17
17
|
import { countEmbeddingInputModalities } from '../../utilities/embedding-input'
|
|
18
|
+
import { assertPromptFileSourceSupport } from '../../utilities/content-source'
|
|
18
19
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
19
20
|
import type { DebugOption } from '../../logger/types'
|
|
20
21
|
import type { GenerationMiddleware } from '../middleware/types'
|
|
@@ -190,6 +191,10 @@ export async function embed<
|
|
|
190
191
|
TAdapter extends EmbeddingAdapter<string, any, any, any>,
|
|
191
192
|
>(options: EmbedOptions<TAdapter>): Promise<EmbeddingResult> {
|
|
192
193
|
const { adapter, middleware } = options
|
|
194
|
+
// Fail closed on `{ type: 'file' }` sources before middleware start. No
|
|
195
|
+
// embedding adapter consumes file handles today; this matches chat /
|
|
196
|
+
// generateImage / generateVideo.
|
|
197
|
+
assertPromptFileSourceSupport(adapter, options.input)
|
|
193
198
|
const model = adapter.model
|
|
194
199
|
const requestId = createId('embedding')
|
|
195
200
|
const startTime = Date.now()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files Adapter
|
|
3
|
+
*
|
|
4
|
+
* Base class and interface for the `files` activity — a provider's native Files
|
|
5
|
+
* API (upload a media asset once, reference it later by the returned handle
|
|
6
|
+
* instead of re-sending base64 or a public URL each request).
|
|
7
|
+
*
|
|
8
|
+
* Providers with a native surface expose a factory (`openaiFiles()`,
|
|
9
|
+
* `anthropicFiles()`, `geminiFiles()`, `falFiles()`). `upload` is required;
|
|
10
|
+
* `get`/`delete` are optional because not every provider has a lifecycle API
|
|
11
|
+
* (fal's storage is upload-only).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { base64ToArrayBuffer } from '@tanstack/ai-utils'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Input to {@link FilesAdapter.upload}. Either a `Blob` (memory-efficient,
|
|
18
|
+
* preferred for large assets) or base64 `data` plus its `mimeType`.
|
|
19
|
+
*/
|
|
20
|
+
export type FileUploadInput =
|
|
21
|
+
| Blob
|
|
22
|
+
| {
|
|
23
|
+
/** Base64-encoded file bytes. */
|
|
24
|
+
data: string
|
|
25
|
+
/** MIME type of the bytes (e.g. `'image/png'`, `'application/pdf'`). */
|
|
26
|
+
mimeType: string
|
|
27
|
+
/** Optional filename hint sent to providers that accept one. */
|
|
28
|
+
filename?: string
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A provider-issued file handle returned by {@link FilesAdapter.upload} /
|
|
33
|
+
* {@link FilesAdapter.get}. Reference it in a message via a `{ type: 'file' }`
|
|
34
|
+
* content source — use `fileSourceFromHandle` to build one.
|
|
35
|
+
*
|
|
36
|
+
* `TProvider` carries the issuing provider's name as a literal (`'openai'`,
|
|
37
|
+
* `'gemini'`, ...) when the handle came from a concrete files adapter, so
|
|
38
|
+
* cross-provider lifecycle calls (`deleteFile` with a foreign handle) fail at
|
|
39
|
+
* compile time. It defaults to `string` so wire-deserialized handles still fit.
|
|
40
|
+
*/
|
|
41
|
+
export interface FileHandle<TProvider extends string = string> {
|
|
42
|
+
/**
|
|
43
|
+
* Provider handle used for lifecycle operations (`get`/`delete`): the
|
|
44
|
+
* OpenAI/Anthropic `file_id`, the Gemini file resource name (`files/...`), or
|
|
45
|
+
* the fal storage URL (fal itself has no lifecycle API — the URL doubles as
|
|
46
|
+
* the wire reference).
|
|
47
|
+
*/
|
|
48
|
+
id: string
|
|
49
|
+
/** The provider that issued the handle (`'openai'`, `'gemini'`, ...). */
|
|
50
|
+
provider: TProvider
|
|
51
|
+
/**
|
|
52
|
+
* The handle's URL form when the provider exposes one (Gemini file URI, fal
|
|
53
|
+
* storage URL). For providers whose handle is an opaque id (OpenAI,
|
|
54
|
+
* Anthropic) this is `undefined`.
|
|
55
|
+
*/
|
|
56
|
+
uri?: string
|
|
57
|
+
/** MIME type reported by the provider (or echoed from the upload input). */
|
|
58
|
+
mimeType?: string
|
|
59
|
+
/** File size in bytes when the provider reports it. */
|
|
60
|
+
sizeBytes?: number
|
|
61
|
+
/** Expiry as epoch milliseconds when the handle is scheduled to expire. */
|
|
62
|
+
expiresAt?: number
|
|
63
|
+
/** Original filename when the provider reports it. */
|
|
64
|
+
filename?: string
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The `files` adapter contract. `upload` is required; `get`/`delete` are
|
|
69
|
+
* optional and present only when the provider has a lifecycle API.
|
|
70
|
+
*
|
|
71
|
+
* `TName` is the provider name literal (`'openai'`, `'gemini'`, ...); concrete
|
|
72
|
+
* adapters bind it so the handles they issue carry their provenance in the
|
|
73
|
+
* type system.
|
|
74
|
+
*/
|
|
75
|
+
export interface FilesAdapter<TName extends string = string> {
|
|
76
|
+
readonly kind: 'files'
|
|
77
|
+
readonly name: TName
|
|
78
|
+
upload: (input: FileUploadInput) => Promise<FileHandle<TName>>
|
|
79
|
+
get?: (id: string) => Promise<FileHandle<TName>>
|
|
80
|
+
delete?: (id: string) => Promise<void>
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export type AnyFilesAdapter = FilesAdapter<string>
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Normalize a {@link FileUploadInput} to a `Blob` (plus best-effort MIME /
|
|
87
|
+
* filename) so provider adapters can hand it straight to their SDK. A `Blob`
|
|
88
|
+
* input passes through; base64 `{ data }` is decoded to bytes. Shared so
|
|
89
|
+
* provider files adapters don't each re-implement the decode.
|
|
90
|
+
*/
|
|
91
|
+
export function normalizeFileUploadInput(input: FileUploadInput): {
|
|
92
|
+
blob: Blob
|
|
93
|
+
mimeType?: string
|
|
94
|
+
filename?: string
|
|
95
|
+
} {
|
|
96
|
+
if (input instanceof Blob) {
|
|
97
|
+
return { blob: input, mimeType: input.type || undefined }
|
|
98
|
+
}
|
|
99
|
+
const bytes = base64ToArrayBuffer(input.data)
|
|
100
|
+
return {
|
|
101
|
+
blob: new Blob([bytes], { type: input.mimeType }),
|
|
102
|
+
mimeType: input.mimeType,
|
|
103
|
+
filename: input.filename,
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Abstract base for provider files adapters. Subclasses bind `TName` to their
|
|
109
|
+
* provider literal, set `name`, implement `upload`, and may add `get`/`delete`
|
|
110
|
+
* (declared on {@link FilesAdapter}, not here, since not every provider has a
|
|
111
|
+
* lifecycle API).
|
|
112
|
+
*/
|
|
113
|
+
export abstract class BaseFilesAdapter<
|
|
114
|
+
TName extends string = string,
|
|
115
|
+
> implements FilesAdapter<TName> {
|
|
116
|
+
readonly kind = 'files' as const
|
|
117
|
+
abstract readonly name: TName
|
|
118
|
+
|
|
119
|
+
abstract upload(input: FileUploadInput): Promise<FileHandle<TName>>
|
|
120
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files Activity
|
|
3
|
+
*
|
|
4
|
+
* Dispatch functions for provider Files APIs. Each takes `{ adapter, ... }` and
|
|
5
|
+
* calls the adapter method directly (mirrors the other activity dispatchers).
|
|
6
|
+
* `get`/`delete` are optional on the adapter; the dispatchers throw a clear
|
|
7
|
+
* error when the selected provider has no lifecycle API.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { ContentPartFileSource } from '../../types'
|
|
11
|
+
import type { FileHandle, FileUploadInput, FilesAdapter } from './adapter'
|
|
12
|
+
|
|
13
|
+
/** The adapter kind this activity handles */
|
|
14
|
+
export const kind = 'files' as const
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Upload a file to a provider's Files API and return its handle. The handle
|
|
18
|
+
* carries the provider name as a literal type, so passing it to another
|
|
19
|
+
* provider's lifecycle call is a compile error.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```ts
|
|
23
|
+
* const files = openaiFiles()
|
|
24
|
+
* const handle = await uploadFile({ adapter: files, input: { data, mimeType: 'image/png' } })
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export async function uploadFile<TName extends string>(options: {
|
|
28
|
+
adapter: FilesAdapter<TName> & { kind: typeof kind }
|
|
29
|
+
input: FileUploadInput
|
|
30
|
+
}): Promise<FileHandle<TName>> {
|
|
31
|
+
return options.adapter.upload(options.input)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Resolve a lifecycle id from either a raw id string or a {@link FileHandle}
|
|
36
|
+
* (whose `id` — not its `uri`/wire value — is the lifecycle currency).
|
|
37
|
+
*/
|
|
38
|
+
function toLifecycleId(id: string | FileHandle): string {
|
|
39
|
+
return typeof id === 'string' ? id : id.id
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Fetch metadata for a previously uploaded file. Accepts the handle itself
|
|
44
|
+
* (preferred — the provider-literal type rejects a foreign provider's handle
|
|
45
|
+
* at compile time) or its raw lifecycle id.
|
|
46
|
+
*
|
|
47
|
+
* @throws if the provider's files adapter has no `get` (e.g. fal storage).
|
|
48
|
+
*/
|
|
49
|
+
export async function getFile<TName extends string>(options: {
|
|
50
|
+
adapter: FilesAdapter<TName> & { kind: typeof kind }
|
|
51
|
+
// `NoInfer` so `TName` comes from the adapter only. Otherwise a foreign
|
|
52
|
+
// handle widens it to a union and the call compiles.
|
|
53
|
+
id: string | FileHandle<NoInfer<TName>>
|
|
54
|
+
}): Promise<FileHandle<TName>> {
|
|
55
|
+
const { adapter } = options
|
|
56
|
+
if (!adapter.get) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`${adapter.name}: files adapter does not support get() — this provider ` +
|
|
59
|
+
`has no file-retrieval API.`,
|
|
60
|
+
)
|
|
61
|
+
}
|
|
62
|
+
return adapter.get(toLifecycleId(options.id))
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Delete a previously uploaded file. Accepts the handle itself (preferred —
|
|
67
|
+
* the provider-literal type rejects a foreign provider's handle at compile
|
|
68
|
+
* time) or its raw lifecycle id.
|
|
69
|
+
*
|
|
70
|
+
* @throws if the provider's files adapter has no `delete` (e.g. fal storage).
|
|
71
|
+
*/
|
|
72
|
+
export async function deleteFile<TName extends string>(options: {
|
|
73
|
+
adapter: FilesAdapter<TName> & { kind: typeof kind }
|
|
74
|
+
id: string | FileHandle<NoInfer<TName>>
|
|
75
|
+
}): Promise<void> {
|
|
76
|
+
const { adapter } = options
|
|
77
|
+
if (!adapter.delete) {
|
|
78
|
+
throw new Error(
|
|
79
|
+
`${adapter.name}: files adapter does not support delete() — this ` +
|
|
80
|
+
`provider has no file-deletion API.`,
|
|
81
|
+
)
|
|
82
|
+
}
|
|
83
|
+
return adapter.delete(toLifecycleId(options.id))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Build a `{ type: 'file' }` content source from an uploaded
|
|
88
|
+
* {@link FileHandle}, for use in a chat message (image/audio/document part
|
|
89
|
+
* `source`).
|
|
90
|
+
*
|
|
91
|
+
* The source's `value` is the handle's wire form: the handle URL when the
|
|
92
|
+
* provider exposes one (Gemini, fal, Grok), otherwise the opaque id (OpenAI,
|
|
93
|
+
* Anthropic). `provider` records the issuer, so an adapter for a different
|
|
94
|
+
* provider rejects the source rather than sending a handle it cannot resolve.
|
|
95
|
+
*
|
|
96
|
+
* @example
|
|
97
|
+
* ```ts
|
|
98
|
+
* const handle = await uploadFile({ adapter: openaiFiles(), input })
|
|
99
|
+
* messages.push({ role: 'user', content: [
|
|
100
|
+
* { type: 'image', source: fileSourceFromHandle(handle) },
|
|
101
|
+
* ] })
|
|
102
|
+
* ```
|
|
103
|
+
*/
|
|
104
|
+
export function fileSourceFromHandle<TProvider extends string>(
|
|
105
|
+
handle: FileHandle<TProvider>,
|
|
106
|
+
): ContentPartFileSource<TProvider> {
|
|
107
|
+
return {
|
|
108
|
+
type: 'file',
|
|
109
|
+
value: handle.uri ?? handle.id,
|
|
110
|
+
provider: handle.provider,
|
|
111
|
+
...(handle.mimeType ? { mimeType: handle.mimeType } : {}),
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -51,6 +51,13 @@ export interface ImageAdapter<
|
|
|
51
51
|
readonly kind: 'image'
|
|
52
52
|
/** Adapter name identifier */
|
|
53
53
|
readonly name: string
|
|
54
|
+
/**
|
|
55
|
+
* Declares that this adapter can consume `{ type: 'file' }` content
|
|
56
|
+
* sources (provider Files API references). The activity dispatcher rejects
|
|
57
|
+
* file sources in preflight for adapters that don't declare this, so
|
|
58
|
+
* adapters written before the file arm existed fail closed.
|
|
59
|
+
*/
|
|
60
|
+
readonly supportsFileSources?: boolean
|
|
54
61
|
/** The model this adapter is configured for */
|
|
55
62
|
readonly model: TModel
|
|
56
63
|
|
|
@@ -103,6 +110,7 @@ export abstract class BaseImageAdapter<
|
|
|
103
110
|
> {
|
|
104
111
|
readonly kind = 'image' as const
|
|
105
112
|
abstract readonly name: string
|
|
113
|
+
readonly supportsFileSources: boolean = false
|
|
106
114
|
readonly model: TModel
|
|
107
115
|
|
|
108
116
|
// Type-only property - never assigned at runtime
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
raceWithAbort,
|
|
25
25
|
} from '../../utilities/activity-abort'
|
|
26
26
|
import { resolveMediaPrompt } from '../../utilities/media-prompt'
|
|
27
|
+
import { assertPromptFileSourceSupport } from '../../utilities/content-source'
|
|
27
28
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
28
29
|
import type { DebugOption } from '../../logger/types'
|
|
29
30
|
import type { GenerationMiddleware } from '../middleware/types'
|
|
@@ -248,6 +249,9 @@ export function generateImage<
|
|
|
248
249
|
>(
|
|
249
250
|
options: ImageActivityOptions<TAdapter, TStream>,
|
|
250
251
|
): ImageActivityResult<TStream> {
|
|
252
|
+
// Fail closed before middleware start and before `stream: true` emits
|
|
253
|
+
// RUN_STARTED, so an unsupported file source never opens a run.
|
|
254
|
+
assertPromptFileSourceSupport(options.adapter, options.prompt)
|
|
251
255
|
if (options.stream) {
|
|
252
256
|
return streamGenerationResult(
|
|
253
257
|
// Only `runId` is taken from the resolved wire identity. `threadId` stays
|
|
@@ -74,6 +74,13 @@ export interface VideoAdapter<
|
|
|
74
74
|
readonly kind: 'video'
|
|
75
75
|
/** Adapter name identifier */
|
|
76
76
|
readonly name: string
|
|
77
|
+
/**
|
|
78
|
+
* Declares that this adapter can consume `{ type: 'file' }` content
|
|
79
|
+
* sources (provider Files API references). The activity dispatcher rejects
|
|
80
|
+
* file sources in preflight for adapters that don't declare this, so
|
|
81
|
+
* adapters written before the file arm existed fail closed.
|
|
82
|
+
*/
|
|
83
|
+
readonly supportsFileSources?: boolean
|
|
77
84
|
/** The model this adapter is configured for */
|
|
78
85
|
readonly model: TModel
|
|
79
86
|
|
|
@@ -161,6 +168,7 @@ export abstract class BaseVideoAdapter<
|
|
|
161
168
|
> {
|
|
162
169
|
readonly kind = 'video' as const
|
|
163
170
|
abstract readonly name: string
|
|
171
|
+
readonly supportsFileSources: boolean = false
|
|
164
172
|
readonly model: TModel
|
|
165
173
|
|
|
166
174
|
// Type-only property - never assigned at runtime
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { aiEventClient } from '@tanstack/ai-event-client'
|
|
13
13
|
import { toRunErrorPayload } from '../error-payload'
|
|
14
14
|
import { resolveDebugOption } from '../../logger/resolve'
|
|
15
|
+
import { assertPromptFileSourceSupport } from '../../utilities/content-source'
|
|
15
16
|
import {
|
|
16
17
|
applyGenerationResultTransforms,
|
|
17
18
|
createGenerationContext,
|
|
@@ -452,6 +453,9 @@ async function runCreateVideoJob<
|
|
|
452
453
|
timeout,
|
|
453
454
|
abortSignal: callerAbortSignal,
|
|
454
455
|
} = options
|
|
456
|
+
// Fail closed on `{ type: 'file' }` sources for adapters that haven't
|
|
457
|
+
// declared support (see assertPromptFileSourceSupport).
|
|
458
|
+
assertPromptFileSourceSupport(adapter, prompt)
|
|
455
459
|
const model = adapter.model
|
|
456
460
|
const requestId = createId('video')
|
|
457
461
|
const startTime = Date.now()
|
|
@@ -580,6 +584,9 @@ async function* runStreamingVideoGeneration<
|
|
|
580
584
|
timeout,
|
|
581
585
|
abortSignal: callerAbortSignal,
|
|
582
586
|
} = options
|
|
587
|
+
// Fail closed on `{ type: 'file' }` sources for adapters that haven't
|
|
588
|
+
// declared support (see assertPromptFileSourceSupport).
|
|
589
|
+
assertPromptFileSourceSupport(adapter, prompt)
|
|
583
590
|
const model = adapter.model
|
|
584
591
|
const runId = options.runId ?? createId('run')
|
|
585
592
|
const requestId = createId('video')
|
|
@@ -44,10 +44,12 @@ export interface WorldAdapter<
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
/**
|
|
47
|
-
*
|
|
47
|
+
* Create a world from a prompt.
|
|
48
48
|
*
|
|
49
|
-
*
|
|
49
|
+
* Live session adapters mint a short-lived token and return it with the
|
|
50
50
|
* prompt so a browser can connect, set the prompt, and start streaming.
|
|
51
|
+
* Job adapters start generation and return a world URL (or an operation
|
|
52
|
+
* id while the job is still running).
|
|
51
53
|
*/
|
|
52
54
|
createWorld: (
|
|
53
55
|
options: WorldGenerationOptions<TProviderOptions>,
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* World Activity (Experimental)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Live adapters mint a session token for a prompt-steerable world. Job
|
|
5
|
+
* adapters start generation and return a viewer URL, or an operation id
|
|
6
|
+
* while the job is still running.
|
|
7
7
|
*
|
|
8
8
|
* @experimental World generation is an experimental feature and may change.
|
|
9
9
|
*/
|
|
@@ -76,7 +76,7 @@ export interface WorldActivityOptions<
|
|
|
76
76
|
/** Provider-specific options for world generation */
|
|
77
77
|
modelOptions?: WorldProviderOptions<TAdapter>
|
|
78
78
|
/**
|
|
79
|
-
* Whether to wrap the
|
|
79
|
+
* Whether to wrap the result as StreamChunks for SSE transport.
|
|
80
80
|
* This is not the live video. When false or omitted, returns
|
|
81
81
|
* Promise<WorldGenerationResult>.
|
|
82
82
|
*
|
|
@@ -100,7 +100,7 @@ export interface WorldActivityOptions<
|
|
|
100
100
|
/** Stable run id for correlating this run when persisted. */
|
|
101
101
|
runId?: string
|
|
102
102
|
/**
|
|
103
|
-
* Maximum
|
|
103
|
+
* Maximum wait for the mint or job poll, in milliseconds.
|
|
104
104
|
* No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
|
|
105
105
|
*/
|
|
106
106
|
timeout?: number
|
|
@@ -135,7 +135,8 @@ function createId(prefix: string): string {
|
|
|
135
135
|
// ===========================
|
|
136
136
|
|
|
137
137
|
/**
|
|
138
|
-
* World generation activity
|
|
138
|
+
* World generation activity. Live adapters mint a session token. Job
|
|
139
|
+
* adapters return a viewer URL or an in-progress operation id.
|
|
139
140
|
*
|
|
140
141
|
* @example Mint a session token on the server
|
|
141
142
|
* ```ts
|
package/src/activities/index.ts
CHANGED
|
@@ -27,6 +27,7 @@ import type { AnyRerankAdapter } from './rerank/adapter'
|
|
|
27
27
|
import type { AnyEvaluateAdapter } from './evaluate/adapter'
|
|
28
28
|
import type { AnyWorldAdapter } from './generateWorld/adapter'
|
|
29
29
|
import type { AnyLiveVideoAdapter } from './generateLiveVideo/adapter'
|
|
30
|
+
import type { AnyFilesAdapter } from './files/adapter'
|
|
30
31
|
|
|
31
32
|
// ===========================
|
|
32
33
|
// Chat Activity
|
|
@@ -330,11 +331,32 @@ export {
|
|
|
330
331
|
type AnyLiveVideoAdapter,
|
|
331
332
|
} from './generateLiveVideo/adapter'
|
|
332
333
|
|
|
334
|
+
// ===========================
|
|
335
|
+
// Files Activity
|
|
336
|
+
// ===========================
|
|
337
|
+
|
|
338
|
+
export {
|
|
339
|
+
kind as filesKind,
|
|
340
|
+
uploadFile,
|
|
341
|
+
getFile,
|
|
342
|
+
deleteFile,
|
|
343
|
+
fileSourceFromHandle,
|
|
344
|
+
} from './files/index'
|
|
345
|
+
|
|
346
|
+
export {
|
|
347
|
+
BaseFilesAdapter,
|
|
348
|
+
normalizeFileUploadInput,
|
|
349
|
+
type FilesAdapter,
|
|
350
|
+
type AnyFilesAdapter,
|
|
351
|
+
type FileHandle,
|
|
352
|
+
type FileUploadInput,
|
|
353
|
+
} from './files/adapter'
|
|
354
|
+
|
|
333
355
|
// ===========================
|
|
334
356
|
// Adapter Union Types
|
|
335
357
|
// ===========================
|
|
336
358
|
|
|
337
|
-
/** Union of all adapter types
|
|
359
|
+
/** Union of all adapter types across every activity kind */
|
|
338
360
|
export type AIAdapter =
|
|
339
361
|
| AnyTextAdapter
|
|
340
362
|
| AnySummarizeAdapter
|
|
@@ -349,6 +371,7 @@ export type AIAdapter =
|
|
|
349
371
|
| AnyEvaluateAdapter
|
|
350
372
|
| AnyWorldAdapter
|
|
351
373
|
| AnyLiveVideoAdapter
|
|
374
|
+
| AnyFilesAdapter
|
|
352
375
|
|
|
353
376
|
/** Union type of all adapter kinds */
|
|
354
377
|
export type AdapterKind =
|
|
@@ -365,3 +388,4 @@ export type AdapterKind =
|
|
|
365
388
|
| 'evaluate'
|
|
366
389
|
| 'world'
|
|
367
390
|
| 'liveVideo'
|
|
391
|
+
| 'files'
|
|
@@ -65,6 +65,8 @@ function throwRunError(
|
|
|
65
65
|
* `SummarizationOptions<TProviderOptions>` on the wrapper itself.
|
|
66
66
|
*/
|
|
67
67
|
export interface ChatStreamCapable {
|
|
68
|
+
/** Native token-limit option for adapters whose provider name is user-defined. */
|
|
69
|
+
readonly maxTokensKey?: string
|
|
68
70
|
chatStream: (options: TextOptions<any>) => AsyncIterable<AdapterYieldChunk>
|
|
69
71
|
}
|
|
70
72
|
|
|
@@ -151,8 +153,8 @@ function applyDefaultTemperature(
|
|
|
151
153
|
|
|
152
154
|
/**
|
|
153
155
|
* Resolve `maxLength` to the provider-native max-output-tokens key for the
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
+
* wrapped text adapter's explicit key, falling back to this wrapper's `name`,
|
|
157
|
+
* and merge it into a working copy of the caller's
|
|
156
158
|
* `modelOptions`. The caller always wins: if they already set any recognised
|
|
157
159
|
* token-limit key (flat or, for Ollama, nested `options.num_predict`), the
|
|
158
160
|
* default is left untouched. Unknown/unrecognised adapter names fall back to
|
|
@@ -172,10 +174,11 @@ function applyMaxLength(
|
|
|
172
174
|
adapterName: string,
|
|
173
175
|
maxLength: number,
|
|
174
176
|
modelOptions: Record<string, unknown>,
|
|
177
|
+
maxTokensKey?: string,
|
|
175
178
|
): Record<string, unknown> {
|
|
176
179
|
const merged: Record<string, unknown> = { ...modelOptions }
|
|
177
180
|
|
|
178
|
-
if (adapterName === 'ollama') {
|
|
181
|
+
if (adapterName === 'ollama' && maxTokensKey === undefined) {
|
|
179
182
|
// Honor a caller-set limit in either shape: a recognised flat key (e.g.
|
|
180
183
|
// left over from a migration) or the nested `options.num_predict`.
|
|
181
184
|
const callerSetFlatLimit = KNOWN_MAX_TOKENS_KEYS.some(
|
|
@@ -195,12 +198,12 @@ function applyMaxLength(
|
|
|
195
198
|
return merged
|
|
196
199
|
}
|
|
197
200
|
|
|
198
|
-
const key = MAX_TOKENS_KEY_BY_ADAPTER[adapterName]
|
|
201
|
+
const key = maxTokensKey ?? MAX_TOKENS_KEY_BY_ADAPTER[adapterName]
|
|
199
202
|
if (key === undefined) return merged
|
|
200
203
|
|
|
201
|
-
const callerSetLimit =
|
|
202
|
-
|
|
203
|
-
|
|
204
|
+
const callerSetLimit =
|
|
205
|
+
typeof merged[key] === 'number' ||
|
|
206
|
+
KNOWN_MAX_TOKENS_KEYS.some((k) => typeof merged[k] === 'number')
|
|
204
207
|
if (callerSetLimit) return merged
|
|
205
208
|
|
|
206
209
|
merged[key] = maxLength
|
|
@@ -372,17 +375,24 @@ export class ChatStreamSummarizeAdapter<
|
|
|
372
375
|
working = applyDefaultTemperature(this.name, 0.3, working)
|
|
373
376
|
// `maxLength` must reach the wire under the provider-native token key (it
|
|
374
377
|
// differs per provider, and no adapter reads a generic `maxTokens`).
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
// token limit.
|
|
378
|
+
// Prefer the wrapped adapter's explicit key, then this wrapper's name,
|
|
379
|
+
// never overriding a caller-supplied token limit.
|
|
378
380
|
if (options.maxLength !== undefined) {
|
|
379
|
-
if (
|
|
381
|
+
if (
|
|
382
|
+
this.textAdapter.maxTokensKey === undefined &&
|
|
383
|
+
!isKnownMaxTokensAdapter(this.name)
|
|
384
|
+
) {
|
|
380
385
|
options.logger.warn(
|
|
381
386
|
`summarize: maxLength=${options.maxLength} could not be mapped to a provider token key for adapter name "${this.name}" — it was dropped from modelOptions (the prompt still asks the model to stay under it). Construct ChatStreamSummarizeAdapter with a recognised provider name to forward the cap.`,
|
|
382
387
|
{ provider: this.name },
|
|
383
388
|
)
|
|
384
389
|
}
|
|
385
|
-
working = applyMaxLength(
|
|
390
|
+
working = applyMaxLength(
|
|
391
|
+
this.name,
|
|
392
|
+
options.maxLength,
|
|
393
|
+
working,
|
|
394
|
+
this.textAdapter.maxTokensKey,
|
|
395
|
+
)
|
|
386
396
|
}
|
|
387
397
|
const modelOptions = working as TProviderOptions
|
|
388
398
|
|