@tanstack/ai 0.39.1 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/activities/chat/index.d.ts +2 -2
- package/dist/esm/activities/chat/index.js +20 -3
- package/dist/esm/activities/chat/index.js.map +1 -1
- package/dist/esm/activities/chat/messages.js +7 -0
- package/dist/esm/activities/chat/messages.js.map +1 -1
- package/dist/esm/activities/chat/middleware/compose.d.ts +2 -2
- package/dist/esm/activities/chat/middleware/compose.js.map +1 -1
- package/dist/esm/activities/chat/middleware/index.d.ts +1 -1
- package/dist/esm/activities/chat/middleware/sandbox-runtime.d.ts +7 -2
- package/dist/esm/activities/chat/middleware/sandbox-runtime.js.map +1 -1
- package/dist/esm/activities/chat/middleware/types.d.ts +16 -4
- package/dist/esm/activities/chat/stream/message-updaters.d.ts +2 -0
- package/dist/esm/activities/chat/stream/message-updaters.js +3 -1
- package/dist/esm/activities/chat/stream/message-updaters.js.map +1 -1
- package/dist/esm/activities/chat/stream/processor.js +18 -1
- package/dist/esm/activities/chat/stream/processor.js.map +1 -1
- package/dist/esm/activities/chat/tools/tool-definition.d.ts +14 -11
- package/dist/esm/activities/chat/tools/tool-definition.js.map +1 -1
- package/dist/esm/activities/generateAudio/index.d.ts +1 -1
- package/dist/esm/activities/generateAudio/index.js.map +1 -1
- package/dist/esm/activities/generateImage/index.d.ts +1 -1
- package/dist/esm/activities/generateImage/index.js +1 -1
- package/dist/esm/activities/generateImage/index.js.map +1 -1
- package/dist/esm/activities/generateSpeech/index.d.ts +1 -1
- package/dist/esm/activities/generateSpeech/index.js.map +1 -1
- package/dist/esm/activities/generateTranscription/index.d.ts +3 -3
- package/dist/esm/activities/generateTranscription/index.js.map +1 -1
- package/dist/esm/activities/generateVideo/index.d.ts +12 -12
- package/dist/esm/activities/generateVideo/index.js.map +1 -1
- package/dist/esm/index.d.ts +3 -3
- package/dist/esm/index.js +2 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/realtime/event-emitter.d.ts +5 -0
- package/dist/esm/realtime/event-emitter.js +27 -0
- package/dist/esm/realtime/event-emitter.js.map +1 -0
- package/dist/esm/realtime/index.d.ts +1 -0
- package/dist/esm/realtime/index.js.map +1 -1
- package/dist/esm/realtime/types.d.ts +9 -1
- package/dist/esm/types.d.ts +124 -1
- package/package.json +1 -1
- package/skills/ai-core/adapter-configuration/SKILL.md +10 -0
- package/skills/ai-core/ag-ui-protocol/SKILL.md +59 -0
- package/skills/ai-core/media-generation/SKILL.md +50 -5
- package/skills/ai-core/middleware/SKILL.md +106 -0
- package/skills/ai-core/tool-calling/SKILL.md +13 -1
- package/src/activities/chat/index.ts +26 -6
- package/src/activities/chat/messages.ts +9 -0
- package/src/activities/chat/middleware/compose.ts +2 -2
- package/src/activities/chat/middleware/index.ts +1 -0
- package/src/activities/chat/middleware/sandbox-runtime.ts +4 -2
- package/src/activities/chat/middleware/types.ts +17 -4
- package/src/activities/chat/stream/message-updaters.ts +7 -1
- package/src/activities/chat/stream/processor.ts +30 -2
- package/src/activities/chat/tools/tool-definition.ts +33 -11
- package/src/activities/generateAudio/index.ts +2 -2
- package/src/activities/generateImage/index.ts +2 -2
- package/src/activities/generateSpeech/index.ts +2 -2
- package/src/activities/generateTranscription/index.ts +8 -4
- package/src/activities/generateVideo/index.ts +29 -15
- package/src/index.ts +3 -1
- package/src/realtime/event-emitter.ts +46 -0
- package/src/realtime/index.ts +2 -0
- package/src/realtime/types.ts +9 -0
- package/src/types.ts +116 -1
|
@@ -45,6 +45,7 @@ import type {
|
|
|
45
45
|
import type {
|
|
46
46
|
AgentLoopStrategy,
|
|
47
47
|
AnyTool,
|
|
48
|
+
ChatStream,
|
|
48
49
|
ConstrainedModelMessage,
|
|
49
50
|
CustomEvent,
|
|
50
51
|
InferSchemaType,
|
|
@@ -69,7 +70,7 @@ import type {
|
|
|
69
70
|
ChatMiddleware,
|
|
70
71
|
ChatMiddlewareConfig,
|
|
71
72
|
ChatMiddlewareContext,
|
|
72
|
-
|
|
73
|
+
SandboxFileHookEvent,
|
|
73
74
|
StructuredOutputMiddlewareConfig,
|
|
74
75
|
} from './middleware/types'
|
|
75
76
|
import type { CheckCoverage } from './middleware/builder'
|
|
@@ -404,7 +405,7 @@ export type TextActivityResult<
|
|
|
404
405
|
: Promise<InferSchemaType<TSchema>>
|
|
405
406
|
: [TStream] extends [false]
|
|
406
407
|
? Promise<string>
|
|
407
|
-
:
|
|
408
|
+
: ChatStream
|
|
408
409
|
|
|
409
410
|
// ===========================
|
|
410
411
|
// ChatEngine Implementation
|
|
@@ -712,11 +713,30 @@ class TextEngine<
|
|
|
712
713
|
// a `sandbox.file` custom chunk to be drained into the public stream.
|
|
713
714
|
provideSandboxRuntime(this.middlewareCtx, {
|
|
714
715
|
logger: this.logger,
|
|
715
|
-
emit: (event:
|
|
716
|
-
this.logger.sandbox(`file ${event.type} ${event.path}`, {
|
|
717
|
-
|
|
716
|
+
emit: (event: SandboxFileHookEvent) => {
|
|
717
|
+
this.logger.sandbox(`file ${event.type} ${event.path}`, {
|
|
718
|
+
event: {
|
|
719
|
+
type: event.type,
|
|
720
|
+
path: event.path,
|
|
721
|
+
timestamp: event.timestamp,
|
|
722
|
+
},
|
|
723
|
+
})
|
|
724
|
+
void this.middlewareRunner
|
|
725
|
+
.runSandboxFile(this.middlewareCtx, event)
|
|
726
|
+
.catch((err: unknown) => {
|
|
727
|
+
this.logger.errors('sandbox file hook failed', { error: err })
|
|
728
|
+
})
|
|
729
|
+
this.sandboxFileQueue.push(
|
|
730
|
+
this.createCustomEventChunk('sandbox.file', {
|
|
731
|
+
type: event.type,
|
|
732
|
+
path: event.path,
|
|
733
|
+
timestamp: event.timestamp,
|
|
734
|
+
}),
|
|
735
|
+
)
|
|
736
|
+
},
|
|
737
|
+
emitFileDiff: (value: { path: string; diff: string }) => {
|
|
718
738
|
this.sandboxFileQueue.push(
|
|
719
|
-
this.createCustomEventChunk('sandbox.file',
|
|
739
|
+
this.createCustomEventChunk('sandbox.file.diff', value),
|
|
720
740
|
)
|
|
721
741
|
},
|
|
722
742
|
})
|
|
@@ -444,12 +444,21 @@ export function modelMessageToUIMessage(
|
|
|
444
444
|
// Handle tool calls
|
|
445
445
|
if (modelMessage.toolCalls && modelMessage.toolCalls.length > 0) {
|
|
446
446
|
for (const toolCall of modelMessage.toolCalls) {
|
|
447
|
+
// Model-message arguments are complete, so surface the parsed input.
|
|
448
|
+
// A malformed arguments string just leaves `input` undefined.
|
|
449
|
+
let input: unknown
|
|
450
|
+
try {
|
|
451
|
+
input = JSON.parse(toolCall.function.arguments)
|
|
452
|
+
} catch {
|
|
453
|
+
input = undefined
|
|
454
|
+
}
|
|
447
455
|
parts.push({
|
|
448
456
|
type: 'tool-call',
|
|
449
457
|
id: toolCall.id,
|
|
450
458
|
name: toolCall.function.name,
|
|
451
459
|
arguments: toolCall.function.arguments,
|
|
452
460
|
state: 'input-complete', // Model messages have complete arguments
|
|
461
|
+
...(input !== undefined && { input }),
|
|
453
462
|
...(toolCall.metadata !== undefined && { metadata: toolCall.metadata }),
|
|
454
463
|
})
|
|
455
464
|
}
|
|
@@ -11,7 +11,7 @@ import type {
|
|
|
11
11
|
ErrorInfo,
|
|
12
12
|
FinishInfo,
|
|
13
13
|
IterationInfo,
|
|
14
|
-
|
|
14
|
+
SandboxFileHookEvent,
|
|
15
15
|
StructuredOutputMiddlewareConfig,
|
|
16
16
|
ToolCallHookContext,
|
|
17
17
|
ToolPhaseCompleteInfo,
|
|
@@ -352,7 +352,7 @@ export class MiddlewareRunner<TContext = unknown> {
|
|
|
352
352
|
*/
|
|
353
353
|
async runSandboxFile(
|
|
354
354
|
ctx: ChatMiddlewareContext<TContext>,
|
|
355
|
-
event:
|
|
355
|
+
event: SandboxFileHookEvent,
|
|
356
356
|
): Promise<void> {
|
|
357
357
|
const typed = (
|
|
358
358
|
{
|
|
@@ -7,10 +7,12 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { createCapability } from './capabilities'
|
|
9
9
|
import type { InternalLogger } from '../../../logger/internal-logger'
|
|
10
|
-
import type {
|
|
10
|
+
import type { SandboxFileHookEvent } from './types'
|
|
11
11
|
|
|
12
12
|
export interface SandboxRuntime {
|
|
13
|
-
emit: (event:
|
|
13
|
+
emit: (event: SandboxFileHookEvent) => void
|
|
14
|
+
/** Emit an opt-in per-file `sandbox.file.diff` CUSTOM chunk. */
|
|
15
|
+
emitFileDiff: (value: { path: string; diff: string }) => void
|
|
14
16
|
logger: InternalLogger
|
|
15
17
|
}
|
|
16
18
|
|
|
@@ -21,6 +21,19 @@ export interface SandboxFileEvent {
|
|
|
21
21
|
timestamp: number
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
/** The file event a sandbox hook receives: the serializable {@link SandboxFileEvent}
|
|
25
|
+
* plus lazy, git-backed content accessors. Accessors compute on call, so a hook
|
|
26
|
+
* that only reads `path`/`type` pays nothing. Never present on the serialized
|
|
27
|
+
* `sandbox.file` CUSTOM chunk. */
|
|
28
|
+
export interface SandboxFileHookEvent extends SandboxFileEvent {
|
|
29
|
+
/** Content at the session baseline (`''` for a new file or non-git workspace). */
|
|
30
|
+
before: () => Promise<string>
|
|
31
|
+
/** Current content (`''` when the event is a delete). */
|
|
32
|
+
after: () => Promise<string>
|
|
33
|
+
/** Unified patch vs the session baseline (synthesized add-patch when non-git). */
|
|
34
|
+
diff: () => Promise<string>
|
|
35
|
+
}
|
|
36
|
+
|
|
24
37
|
/**
|
|
25
38
|
* Sandbox file-event hooks a chat middleware can declare. Fire server-side for
|
|
26
39
|
* every file create/change/delete observed in the sandbox during the run.
|
|
@@ -28,19 +41,19 @@ export interface SandboxFileEvent {
|
|
|
28
41
|
export interface ChatSandboxHooks<TContext = unknown> {
|
|
29
42
|
onFile?: (
|
|
30
43
|
ctx: ChatMiddlewareContext<TContext>,
|
|
31
|
-
e:
|
|
44
|
+
e: SandboxFileHookEvent,
|
|
32
45
|
) => void | Promise<void>
|
|
33
46
|
onFileCreate?: (
|
|
34
47
|
ctx: ChatMiddlewareContext<TContext>,
|
|
35
|
-
e:
|
|
48
|
+
e: SandboxFileHookEvent,
|
|
36
49
|
) => void | Promise<void>
|
|
37
50
|
onFileChange?: (
|
|
38
51
|
ctx: ChatMiddlewareContext<TContext>,
|
|
39
|
-
e:
|
|
52
|
+
e: SandboxFileHookEvent,
|
|
40
53
|
) => void | Promise<void>
|
|
41
54
|
onFileDelete?: (
|
|
42
55
|
ctx: ChatMiddlewareContext<TContext>,
|
|
43
|
-
e:
|
|
56
|
+
e: SandboxFileHookEvent,
|
|
44
57
|
) => void | Promise<void>
|
|
45
58
|
}
|
|
46
59
|
|
|
@@ -58,6 +58,8 @@ export function updateToolCallPart(
|
|
|
58
58
|
name: string
|
|
59
59
|
arguments: string
|
|
60
60
|
state: ToolCallState
|
|
61
|
+
/** Parsed input — set when the arguments are complete. */
|
|
62
|
+
input?: unknown
|
|
61
63
|
metadata?: Record<string, unknown>
|
|
62
64
|
},
|
|
63
65
|
): Array<UIMessage> {
|
|
@@ -76,6 +78,9 @@ export function updateToolCallPart(
|
|
|
76
78
|
// Gemini's thoughtSignature on TOOL_CALL_START) we must not lose it on
|
|
77
79
|
// subsequent updates that don't re-supply it.
|
|
78
80
|
const metadata = toolCall.metadata ?? existing?.metadata
|
|
81
|
+
// Same for the parsed input: it's supplied once at completion, so
|
|
82
|
+
// subsequent arg-less updates (approval, etc.) must not drop it.
|
|
83
|
+
const input = toolCall.input ?? existing?.input
|
|
79
84
|
|
|
80
85
|
const toolCallPart: ToolCallPart = {
|
|
81
86
|
type: 'tool-call',
|
|
@@ -83,9 +88,10 @@ export function updateToolCallPart(
|
|
|
83
88
|
name: toolCall.name,
|
|
84
89
|
arguments: toolCall.arguments,
|
|
85
90
|
state: toolCall.state,
|
|
86
|
-
// Carry forward approval and
|
|
91
|
+
// Carry forward approval, output and parsed input from the existing part
|
|
87
92
|
...(existing?.approval && { approval: { ...existing.approval } }),
|
|
88
93
|
...(existing?.output !== undefined && { output: existing.output }),
|
|
94
|
+
...(input !== undefined && { input }),
|
|
89
95
|
...(metadata !== undefined && { metadata }),
|
|
90
96
|
}
|
|
91
97
|
|
|
@@ -1298,15 +1298,35 @@ export class StreamProcessor {
|
|
|
1298
1298
|
// received, back-fill the arguments string so the UIMessage ToolCallPart
|
|
1299
1299
|
// carries the correct value (defensive against adapters that skip ARGS).
|
|
1300
1300
|
if (chunk.input !== undefined && !existingToolCall.arguments) {
|
|
1301
|
-
|
|
1301
|
+
try {
|
|
1302
|
+
existingToolCall.arguments = JSON.stringify(chunk.input)
|
|
1303
|
+
} catch {
|
|
1304
|
+
// circular refs, BigInt, etc. — leave arguments empty rather than
|
|
1305
|
+
// aborting stream processing
|
|
1306
|
+
}
|
|
1302
1307
|
}
|
|
1303
1308
|
|
|
1304
1309
|
const index = msgState.toolCallOrder.indexOf(chunk.toolCallId)
|
|
1305
1310
|
this.completeToolCall(messageId, index, existingToolCall)
|
|
1306
1311
|
// If TOOL_CALL_END provides parsed input, use it as the canonical parsed
|
|
1307
1312
|
// arguments (overrides the accumulated string parse from completeToolCall)
|
|
1313
|
+
// and refresh the rendered part's `input` so it reflects the canonical
|
|
1314
|
+
// value rather than the possibly-divergent accumulated-args parse that
|
|
1315
|
+
// completeToolCall wrote (e.g. an adapter that coerces values differently
|
|
1316
|
+
// between the streamed args and the final structured input).
|
|
1308
1317
|
if (chunk.input !== undefined) {
|
|
1309
1318
|
existingToolCall.parsedArguments = chunk.input
|
|
1319
|
+
this.messages = updateToolCallPart(this.messages, messageId, {
|
|
1320
|
+
id: existingToolCall.id,
|
|
1321
|
+
name: existingToolCall.name,
|
|
1322
|
+
arguments: existingToolCall.arguments,
|
|
1323
|
+
state: 'input-complete',
|
|
1324
|
+
input: chunk.input,
|
|
1325
|
+
...(existingToolCall.metadata !== undefined && {
|
|
1326
|
+
metadata: existingToolCall.metadata,
|
|
1327
|
+
}),
|
|
1328
|
+
})
|
|
1329
|
+
this.emitMessagesChange()
|
|
1310
1330
|
}
|
|
1311
1331
|
}
|
|
1312
1332
|
|
|
@@ -1890,12 +1910,20 @@ export class StreamProcessor {
|
|
|
1890
1910
|
return
|
|
1891
1911
|
}
|
|
1892
1912
|
|
|
1893
|
-
// Update UIMessage
|
|
1913
|
+
// Update UIMessage. The arguments are complete now, so surface the parsed
|
|
1914
|
+
// input on the part. For adapters that skip TOOL_CALL_ARGS the arguments
|
|
1915
|
+
// string was back-filled from TOOL_CALL_END.input, so this parse matches
|
|
1916
|
+
// the canonical input. If a TOOL_CALL_END.input diverges from the
|
|
1917
|
+
// accumulated args, handleToolCallEndEvent re-updates the part with the
|
|
1918
|
+
// canonical value after this call.
|
|
1894
1919
|
this.messages = updateToolCallPart(this.messages, messageId, {
|
|
1895
1920
|
id: toolCall.id,
|
|
1896
1921
|
name: toolCall.name,
|
|
1897
1922
|
arguments: toolCall.arguments,
|
|
1898
1923
|
state: 'input-complete',
|
|
1924
|
+
...(toolCall.parsedArguments !== undefined && {
|
|
1925
|
+
input: toolCall.parsedArguments,
|
|
1926
|
+
}),
|
|
1899
1927
|
...(toolCall.metadata !== undefined && { metadata: toolCall.metadata }),
|
|
1900
1928
|
})
|
|
1901
1929
|
this.emitMessagesChange()
|
|
@@ -26,6 +26,10 @@ export interface ClientTool<
|
|
|
26
26
|
TOutput extends SchemaInput = SchemaInput,
|
|
27
27
|
TName extends string = string,
|
|
28
28
|
TContext = unknown,
|
|
29
|
+
// Captured as a literal (`true` / `false`) so downstream types — notably
|
|
30
|
+
// the tool-call part's `approval` field — can be gated on it. Defaults to
|
|
31
|
+
// `false` when the tool config omits `needsApproval`.
|
|
32
|
+
TNeedsApproval extends boolean = false,
|
|
29
33
|
> {
|
|
30
34
|
__toolSide: 'client'
|
|
31
35
|
name: TName
|
|
@@ -37,7 +41,7 @@ export interface ClientTool<
|
|
|
37
41
|
// because `undefined` doesn't extend the schema constraint.
|
|
38
42
|
inputSchema?: TInput
|
|
39
43
|
outputSchema?: TOutput
|
|
40
|
-
needsApproval?:
|
|
44
|
+
needsApproval?: TNeedsApproval
|
|
41
45
|
lazy?: boolean
|
|
42
46
|
metadata?: Record<string, unknown>
|
|
43
47
|
execute?: ToolExecuteFunction<TInput, TOutput, TContext>
|
|
@@ -51,18 +55,22 @@ export interface ToolDefinitionInstance<
|
|
|
51
55
|
TOutput extends SchemaInput = SchemaInput,
|
|
52
56
|
TName extends string = string,
|
|
53
57
|
TContext = unknown,
|
|
58
|
+
TNeedsApproval extends boolean = false,
|
|
54
59
|
> extends Tool<TInput, TOutput, TName, TContext> {
|
|
55
60
|
__toolSide: 'definition'
|
|
61
|
+
// Narrow the base `needsApproval?: boolean` to the captured literal so it
|
|
62
|
+
// survives into `ToolCallPartForTool`'s approval gate.
|
|
63
|
+
needsApproval?: TNeedsApproval
|
|
56
64
|
}
|
|
57
65
|
|
|
58
66
|
/**
|
|
59
67
|
* Union type for any kind of client-side tool (client tool or definition)
|
|
60
68
|
*/
|
|
61
69
|
export type AnyClientTool =
|
|
62
|
-
| (Omit<ClientTool<any, any, string, any>, 'execute'> & {
|
|
70
|
+
| (Omit<ClientTool<any, any, string, any, boolean>, 'execute'> & {
|
|
63
71
|
execute?: ((args: any, context?: any) => any) | undefined
|
|
64
72
|
})
|
|
65
|
-
| (Omit<ToolDefinitionInstance<any, any, string, any>, 'execute'> & {
|
|
73
|
+
| (Omit<ToolDefinitionInstance<any, any, string, any, boolean>, 'execute'> & {
|
|
66
74
|
execute?: ((args: any, context?: any) => any) | undefined
|
|
67
75
|
})
|
|
68
76
|
|
|
@@ -100,12 +108,13 @@ export interface ToolDefinitionConfig<
|
|
|
100
108
|
TInput extends SchemaInput = SchemaInput,
|
|
101
109
|
TOutput extends SchemaInput = SchemaInput,
|
|
102
110
|
TName extends string = string,
|
|
111
|
+
TNeedsApproval extends boolean = false,
|
|
103
112
|
> {
|
|
104
113
|
name: TName
|
|
105
114
|
description: string
|
|
106
115
|
inputSchema?: TInput
|
|
107
116
|
outputSchema?: TOutput
|
|
108
|
-
needsApproval?:
|
|
117
|
+
needsApproval?: TNeedsApproval
|
|
109
118
|
lazy?: boolean
|
|
110
119
|
metadata?: Record<string, unknown>
|
|
111
120
|
}
|
|
@@ -117,7 +126,14 @@ export interface ToolDefinition<
|
|
|
117
126
|
TInput extends SchemaInput = SchemaInput,
|
|
118
127
|
TOutput extends SchemaInput = SchemaInput,
|
|
119
128
|
TName extends string = string,
|
|
120
|
-
|
|
129
|
+
TNeedsApproval extends boolean = false,
|
|
130
|
+
> extends ToolDefinitionInstance<
|
|
131
|
+
TInput,
|
|
132
|
+
TOutput,
|
|
133
|
+
TName,
|
|
134
|
+
unknown,
|
|
135
|
+
TNeedsApproval
|
|
136
|
+
> {
|
|
121
137
|
/**
|
|
122
138
|
* Create a server-side tool with execute function
|
|
123
139
|
*/
|
|
@@ -126,11 +142,13 @@ export interface ToolDefinition<
|
|
|
126
142
|
) => ServerTool<TInput, TOutput, TName, TContext>
|
|
127
143
|
|
|
128
144
|
/**
|
|
129
|
-
* Create a client-side tool with optional execute function
|
|
145
|
+
* Create a client-side tool with optional execute function.
|
|
146
|
+
* Carries the definition's `needsApproval` literal through to the client
|
|
147
|
+
* tool so the tool-call part's `approval` field stays gated on it.
|
|
130
148
|
*/
|
|
131
149
|
client: <TContext = unknown>(
|
|
132
150
|
execute?: ToolExecuteFunction<TInput, TOutput, TContext>,
|
|
133
|
-
) => ClientTool<TInput, TOutput, TName, TContext>
|
|
151
|
+
) => ClientTool<TInput, TOutput, TName, TContext, TNeedsApproval>
|
|
134
152
|
}
|
|
135
153
|
|
|
136
154
|
/**
|
|
@@ -192,10 +210,14 @@ export function toolDefinition<
|
|
|
192
210
|
TInput extends SchemaInput = SchemaInput,
|
|
193
211
|
TOutput extends SchemaInput = SchemaInput,
|
|
194
212
|
TName extends string = string,
|
|
213
|
+
// `const` forces the literal (`true` / `false`) to be captured from the
|
|
214
|
+
// config's optional `needsApproval` — without it TS widens to `boolean`,
|
|
215
|
+
// which collapses the approval gate in `ToolCallPartForTool`.
|
|
216
|
+
const TNeedsApproval extends boolean = false,
|
|
195
217
|
>(
|
|
196
|
-
config: ToolDefinitionConfig<TInput, TOutput, TName>,
|
|
197
|
-
): ToolDefinition<TInput, TOutput, TName> {
|
|
198
|
-
const definition: ToolDefinition<TInput, TOutput, TName> = {
|
|
218
|
+
config: ToolDefinitionConfig<TInput, TOutput, TName, TNeedsApproval>,
|
|
219
|
+
): ToolDefinition<TInput, TOutput, TName, TNeedsApproval> {
|
|
220
|
+
const definition: ToolDefinition<TInput, TOutput, TName, TNeedsApproval> = {
|
|
199
221
|
__toolSide: 'definition',
|
|
200
222
|
...config,
|
|
201
223
|
server<TContext = unknown>(
|
|
@@ -210,7 +232,7 @@ export function toolDefinition<
|
|
|
210
232
|
|
|
211
233
|
client<TContext = unknown>(
|
|
212
234
|
execute?: ToolExecuteFunction<TInput, TOutput, TContext>,
|
|
213
|
-
): ClientTool<TInput, TOutput, TName, TContext> {
|
|
235
|
+
): ClientTool<TInput, TOutput, TName, TContext, TNeedsApproval> {
|
|
214
236
|
return {
|
|
215
237
|
__toolSide: 'client',
|
|
216
238
|
...config,
|
|
@@ -14,10 +14,10 @@ import {
|
|
|
14
14
|
runGenerationFinish,
|
|
15
15
|
runGenerationStart,
|
|
16
16
|
runGenerationUsage,
|
|
17
|
-
} from '../middleware'
|
|
17
|
+
} from '../middleware/run'
|
|
18
18
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
19
19
|
import type { DebugOption } from '../../logger/types'
|
|
20
|
-
import type { GenerationMiddleware } from '../middleware'
|
|
20
|
+
import type { GenerationMiddleware } from '../middleware/types'
|
|
21
21
|
import type { AudioAdapter } from './adapter'
|
|
22
22
|
import type { AudioGenerationResult, StreamChunk } from '../../types'
|
|
23
23
|
|
|
@@ -14,11 +14,11 @@ import {
|
|
|
14
14
|
runGenerationFinish,
|
|
15
15
|
runGenerationStart,
|
|
16
16
|
runGenerationUsage,
|
|
17
|
-
} from '../middleware'
|
|
17
|
+
} from '../middleware/run'
|
|
18
18
|
import { resolveMediaPrompt } from '../../utilities/media-prompt'
|
|
19
19
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
20
20
|
import type { DebugOption } from '../../logger/types'
|
|
21
|
-
import type { GenerationMiddleware } from '../middleware'
|
|
21
|
+
import type { GenerationMiddleware } from '../middleware/types'
|
|
22
22
|
import type { ImageAdapter } from './adapter'
|
|
23
23
|
import type {
|
|
24
24
|
ImageGenerationResult,
|
|
@@ -14,10 +14,10 @@ import {
|
|
|
14
14
|
runGenerationFinish,
|
|
15
15
|
runGenerationStart,
|
|
16
16
|
runGenerationUsage,
|
|
17
|
-
} from '../middleware'
|
|
17
|
+
} from '../middleware/run'
|
|
18
18
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
19
19
|
import type { DebugOption } from '../../logger/types'
|
|
20
|
-
import type { GenerationMiddleware } from '../middleware'
|
|
20
|
+
import type { GenerationMiddleware } from '../middleware/types'
|
|
21
21
|
import type { TTSAdapter } from './adapter'
|
|
22
22
|
import type { StreamChunk, TTSResult } from '../../types'
|
|
23
23
|
|
|
@@ -14,12 +14,16 @@ import {
|
|
|
14
14
|
runGenerationFinish,
|
|
15
15
|
runGenerationStart,
|
|
16
16
|
runGenerationUsage,
|
|
17
|
-
} from '../middleware'
|
|
17
|
+
} from '../middleware/run'
|
|
18
18
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
19
19
|
import type { DebugOption } from '../../logger/types'
|
|
20
|
-
import type { GenerationMiddleware } from '../middleware'
|
|
20
|
+
import type { GenerationMiddleware } from '../middleware/types'
|
|
21
21
|
import type { TranscriptionAdapter } from './adapter'
|
|
22
|
-
import type {
|
|
22
|
+
import type {
|
|
23
|
+
StreamChunk,
|
|
24
|
+
TranscriptionResponseFormat,
|
|
25
|
+
TranscriptionResult,
|
|
26
|
+
} from '../../types'
|
|
23
27
|
|
|
24
28
|
// ===========================
|
|
25
29
|
// Activity Kind
|
|
@@ -67,7 +71,7 @@ export interface TranscriptionActivityOptions<
|
|
|
67
71
|
/** An optional prompt to guide the transcription */
|
|
68
72
|
prompt?: string
|
|
69
73
|
/** The format of the transcription output */
|
|
70
|
-
responseFormat?:
|
|
74
|
+
responseFormat?: TranscriptionResponseFormat
|
|
71
75
|
/** Provider-specific options for transcription */
|
|
72
76
|
modelOptions?: TranscriptionProviderOptions<TAdapter>
|
|
73
77
|
/**
|
|
@@ -17,10 +17,10 @@ import {
|
|
|
17
17
|
runGenerationFinish,
|
|
18
18
|
runGenerationStart,
|
|
19
19
|
runGenerationUsage,
|
|
20
|
-
} from '../middleware'
|
|
20
|
+
} from '../middleware/run'
|
|
21
21
|
import type { InternalLogger } from '../../logger/internal-logger'
|
|
22
22
|
import type { DebugOption } from '../../logger/types'
|
|
23
|
-
import type { GenerationMiddleware } from '../middleware'
|
|
23
|
+
import type { GenerationMiddleware } from '../middleware/types'
|
|
24
24
|
import type { VideoAdapter } from './adapter'
|
|
25
25
|
import type {
|
|
26
26
|
MediaPrompt,
|
|
@@ -47,7 +47,7 @@ export const kind = 'video' as const
|
|
|
47
47
|
* Extract provider options from a VideoAdapter via ~types.
|
|
48
48
|
*/
|
|
49
49
|
export type VideoProviderOptions<TAdapter> =
|
|
50
|
-
TAdapter extends VideoAdapter<any, any, any, any>
|
|
50
|
+
TAdapter extends VideoAdapter<any, any, any, any, any, any>
|
|
51
51
|
? TAdapter['~types']['providerOptions']
|
|
52
52
|
: object
|
|
53
53
|
|
|
@@ -55,7 +55,14 @@ export type VideoProviderOptions<TAdapter> =
|
|
|
55
55
|
* Extract the size type for a VideoAdapter's model via ~types.
|
|
56
56
|
*/
|
|
57
57
|
export type VideoSizeForAdapter<TAdapter> =
|
|
58
|
-
TAdapter extends VideoAdapter<
|
|
58
|
+
TAdapter extends VideoAdapter<
|
|
59
|
+
infer TModel,
|
|
60
|
+
any,
|
|
61
|
+
any,
|
|
62
|
+
infer TSizeMap,
|
|
63
|
+
any,
|
|
64
|
+
any
|
|
65
|
+
>
|
|
59
66
|
? TModel extends keyof TSizeMap
|
|
60
67
|
? TSizeMap[TModel]
|
|
61
68
|
: string
|
|
@@ -68,7 +75,14 @@ export type VideoSizeForAdapter<TAdapter> =
|
|
|
68
75
|
* without a map fall back to the full MediaPrompt.
|
|
69
76
|
*/
|
|
70
77
|
export type VideoPromptForAdapter<TAdapter> =
|
|
71
|
-
TAdapter extends VideoAdapter<
|
|
78
|
+
TAdapter extends VideoAdapter<
|
|
79
|
+
infer TModel,
|
|
80
|
+
any,
|
|
81
|
+
any,
|
|
82
|
+
any,
|
|
83
|
+
infer ModsByName,
|
|
84
|
+
any
|
|
85
|
+
>
|
|
72
86
|
? string extends keyof ModsByName
|
|
73
87
|
? MediaPrompt
|
|
74
88
|
: TModel extends keyof ModsByName
|
|
@@ -108,7 +122,7 @@ function createId(prefix: string): string {
|
|
|
108
122
|
* The model is extracted from the adapter's model property.
|
|
109
123
|
*/
|
|
110
124
|
interface VideoActivityBaseOptions<
|
|
111
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
125
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
112
126
|
> {
|
|
113
127
|
/** The video adapter to use (must be created with a model) */
|
|
114
128
|
adapter: TAdapter & { kind: typeof kind }
|
|
@@ -124,7 +138,7 @@ interface VideoActivityBaseOptions<
|
|
|
124
138
|
* @experimental Video generation is an experimental feature and may change.
|
|
125
139
|
*/
|
|
126
140
|
export type VideoCreateOptions<
|
|
127
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
141
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
128
142
|
TStream extends boolean = false,
|
|
129
143
|
> = VideoActivityBaseOptions<TAdapter> & {
|
|
130
144
|
/** Request type - create a new job (default if not specified) */
|
|
@@ -191,7 +205,7 @@ export type VideoCreateOptions<
|
|
|
191
205
|
* @experimental Video generation is an experimental feature and may change.
|
|
192
206
|
*/
|
|
193
207
|
export interface VideoStatusOptions<
|
|
194
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
208
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
195
209
|
> extends VideoActivityBaseOptions<TAdapter> {
|
|
196
210
|
/** Request type - get job status */
|
|
197
211
|
request: 'status'
|
|
@@ -205,7 +219,7 @@ export interface VideoStatusOptions<
|
|
|
205
219
|
* @experimental Video generation is an experimental feature and may change.
|
|
206
220
|
*/
|
|
207
221
|
export interface VideoUrlOptions<
|
|
208
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
222
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
209
223
|
> extends VideoActivityBaseOptions<TAdapter> {
|
|
210
224
|
/** Request type - get video URL */
|
|
211
225
|
request: 'url'
|
|
@@ -220,7 +234,7 @@ export interface VideoUrlOptions<
|
|
|
220
234
|
* @experimental Video generation is an experimental feature and may change.
|
|
221
235
|
*/
|
|
222
236
|
export type VideoActivityOptions<
|
|
223
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
237
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
224
238
|
TRequest extends 'create' | 'status' | 'url' = 'create',
|
|
225
239
|
TStream extends boolean = false,
|
|
226
240
|
> = TRequest extends 'status'
|
|
@@ -296,7 +310,7 @@ export type VideoActivityResult<
|
|
|
296
310
|
* ```
|
|
297
311
|
*/
|
|
298
312
|
export function generateVideo<
|
|
299
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
313
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
300
314
|
TStream extends boolean = false,
|
|
301
315
|
>(
|
|
302
316
|
options: VideoCreateOptions<TAdapter, TStream>,
|
|
@@ -314,7 +328,7 @@ export function generateVideo<
|
|
|
314
328
|
* Internal implementation of non-streaming video job creation.
|
|
315
329
|
*/
|
|
316
330
|
async function runCreateVideoJob<
|
|
317
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
331
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
318
332
|
>(options: VideoCreateOptions<TAdapter, boolean>): Promise<VideoJobResult> {
|
|
319
333
|
const { adapter, prompt, size, duration, modelOptions, middleware } = options
|
|
320
334
|
const model = adapter.model
|
|
@@ -383,7 +397,7 @@ function sleep(ms: number): Promise<void> {
|
|
|
383
397
|
* Handles the full job lifecycle: create job → poll for status → stream updates → yield final result.
|
|
384
398
|
*/
|
|
385
399
|
async function* runStreamingVideoGeneration<
|
|
386
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
400
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
387
401
|
>(options: VideoCreateOptions<TAdapter, true>): AsyncIterable<StreamChunk> {
|
|
388
402
|
const { adapter, prompt, size, duration, modelOptions, middleware } = options
|
|
389
403
|
const model = adapter.model
|
|
@@ -582,7 +596,7 @@ async function* runStreamingVideoGeneration<
|
|
|
582
596
|
* ```
|
|
583
597
|
*/
|
|
584
598
|
export async function getVideoJobStatus<
|
|
585
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
599
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
586
600
|
>(options: {
|
|
587
601
|
adapter: TAdapter & { kind: typeof kind }
|
|
588
602
|
jobId: string
|
|
@@ -692,7 +706,7 @@ export async function getVideoJobStatus<
|
|
|
692
706
|
* Create typed options for the generateVideo() function without executing.
|
|
693
707
|
*/
|
|
694
708
|
export function createVideoOptions<
|
|
695
|
-
TAdapter extends VideoAdapter<string, any, any, any>,
|
|
709
|
+
TAdapter extends VideoAdapter<string, any, any, any, any, any>,
|
|
696
710
|
TStream extends boolean = false,
|
|
697
711
|
>(
|
|
698
712
|
options: VideoCreateOptions<TAdapter, TStream>,
|
package/src/index.ts
CHANGED
|
@@ -121,6 +121,7 @@ export type {
|
|
|
121
121
|
AbortInfo,
|
|
122
122
|
ErrorInfo,
|
|
123
123
|
SandboxFileEvent,
|
|
124
|
+
SandboxFileHookEvent,
|
|
124
125
|
ChatSandboxHooks,
|
|
125
126
|
} from './activities/chat/middleware/index'
|
|
126
127
|
|
|
@@ -178,12 +179,13 @@ export { normalizeSystemPrompts } from './system-prompts'
|
|
|
178
179
|
export { detectImageMimeType } from './utils'
|
|
179
180
|
|
|
180
181
|
// Realtime
|
|
181
|
-
export { realtimeToken } from './realtime/index'
|
|
182
|
+
export { realtimeToken, createRealtimeEventEmitter } from './realtime/index'
|
|
182
183
|
export type {
|
|
183
184
|
RealtimeToken,
|
|
184
185
|
RealtimeTokenAdapter,
|
|
185
186
|
RealtimeTokenOptions,
|
|
186
187
|
RealtimeSessionConfig,
|
|
188
|
+
RealtimeToolConfig,
|
|
187
189
|
VADConfig,
|
|
188
190
|
RealtimeMessage,
|
|
189
191
|
RealtimeMessagePart,
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
RealtimeEvent,
|
|
3
|
+
RealtimeEventHandler,
|
|
4
|
+
RealtimeEventPayloads,
|
|
5
|
+
} from './types'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Handlers are stored with a `never` payload so any specific
|
|
9
|
+
* `RealtimeEventHandler<TEvent>` is assignable in (contravariance), keeping the
|
|
10
|
+
* heterogeneous handler map type-safe without `any`. `emit` narrows back to the
|
|
11
|
+
* event's real payload type via its signature; the lone `as never` at the call
|
|
12
|
+
* site is the inverse of that stored `never`.
|
|
13
|
+
*/
|
|
14
|
+
type StoredHandler = (payload: never) => void
|
|
15
|
+
|
|
16
|
+
export function createRealtimeEventEmitter() {
|
|
17
|
+
const eventHandlers = new Map<RealtimeEvent, Set<StoredHandler>>()
|
|
18
|
+
|
|
19
|
+
return {
|
|
20
|
+
emit<TEvent extends RealtimeEvent>(
|
|
21
|
+
event: TEvent,
|
|
22
|
+
payload: RealtimeEventPayloads[TEvent],
|
|
23
|
+
) {
|
|
24
|
+
const handlers = eventHandlers.get(event)
|
|
25
|
+
if (!handlers) return
|
|
26
|
+
for (const handler of handlers) {
|
|
27
|
+
handler(payload as never)
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
on<TEvent extends RealtimeEvent>(
|
|
31
|
+
event: TEvent,
|
|
32
|
+
handler: RealtimeEventHandler<TEvent>,
|
|
33
|
+
): () => void {
|
|
34
|
+
let handlers = eventHandlers.get(event)
|
|
35
|
+
if (!handlers) {
|
|
36
|
+
handlers = new Set<StoredHandler>()
|
|
37
|
+
eventHandlers.set(event, handlers)
|
|
38
|
+
}
|
|
39
|
+
handlers.add(handler)
|
|
40
|
+
|
|
41
|
+
return () => {
|
|
42
|
+
handlers.delete(handler)
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
}
|
|
46
|
+
}
|