@librechat/agents 3.2.42 → 3.2.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/agents/AgentContext.cjs +4 -3
- package/dist/cjs/agents/AgentContext.cjs.map +1 -1
- package/dist/cjs/graphs/Graph.cjs +1 -1
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/hooks/createWorkspacePolicyHook.cjs +1 -1
- package/dist/cjs/llm/bedrock/index.cjs +1 -1
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/main.cjs +2 -0
- package/dist/cjs/messages/cache.cjs +26 -4
- package/dist/cjs/messages/cache.cjs.map +1 -1
- package/dist/cjs/session/JsonlSessionStore.cjs +1 -1
- package/dist/cjs/tools/cloudflare/CloudflareSandboxExecutionEngine.cjs +32 -16
- package/dist/cjs/tools/cloudflare/CloudflareSandboxExecutionEngine.cjs.map +1 -1
- package/dist/cjs/tools/local/CompileCheckTool.cjs +11 -3
- package/dist/cjs/tools/local/CompileCheckTool.cjs.map +1 -1
- package/dist/cjs/tools/local/FileCheckpointer.cjs +8 -3
- package/dist/cjs/tools/local/FileCheckpointer.cjs.map +1 -1
- package/dist/cjs/tools/local/LocalCodingTools.cjs +26 -7
- package/dist/cjs/tools/local/LocalCodingTools.cjs.map +1 -1
- package/dist/cjs/tools/local/LocalExecutionEngine.cjs +2 -2
- package/dist/cjs/tools/local/syntaxCheck.cjs +6 -2
- package/dist/cjs/tools/local/syntaxCheck.cjs.map +1 -1
- package/dist/cjs/tools/local/workspaceFS.cjs +20 -0
- package/dist/cjs/tools/local/workspaceFS.cjs.map +1 -1
- package/dist/esm/agents/AgentContext.mjs +5 -4
- package/dist/esm/agents/AgentContext.mjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +2 -2
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/hooks/createWorkspacePolicyHook.mjs +1 -1
- package/dist/esm/llm/bedrock/index.mjs +2 -2
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/main.mjs +3 -3
- package/dist/esm/messages/cache.mjs +26 -5
- package/dist/esm/messages/cache.mjs.map +1 -1
- package/dist/esm/session/JsonlSessionStore.mjs +1 -1
- package/dist/esm/tools/cloudflare/CloudflareSandboxExecutionEngine.mjs +32 -17
- package/dist/esm/tools/cloudflare/CloudflareSandboxExecutionEngine.mjs.map +1 -1
- package/dist/esm/tools/local/CompileCheckTool.mjs +11 -3
- package/dist/esm/tools/local/CompileCheckTool.mjs.map +1 -1
- package/dist/esm/tools/local/FileCheckpointer.mjs +9 -4
- package/dist/esm/tools/local/FileCheckpointer.mjs.map +1 -1
- package/dist/esm/tools/local/LocalCodingTools.mjs +26 -7
- package/dist/esm/tools/local/LocalCodingTools.mjs.map +1 -1
- package/dist/esm/tools/local/LocalExecutionEngine.mjs +2 -2
- package/dist/esm/tools/local/syntaxCheck.mjs +6 -2
- package/dist/esm/tools/local/syntaxCheck.mjs.map +1 -1
- package/dist/esm/tools/local/workspaceFS.mjs +19 -1
- package/dist/esm/tools/local/workspaceFS.mjs.map +1 -1
- package/dist/types/agents/AgentContext.d.ts +3 -2
- package/dist/types/messages/cache.d.ts +12 -2
- package/dist/types/tools/cloudflare/CloudflareSandboxExecutionEngine.d.ts +9 -0
- package/dist/types/tools/local/workspaceFS.d.ts +13 -0
- package/package.json +1 -1
- package/src/agents/AgentContext.ts +8 -3
- package/src/graphs/Graph.ts +7 -1
- package/src/llm/bedrock/index.ts +2 -2
- package/src/messages/cache.test.ts +32 -0
- package/src/messages/cache.ts +31 -5
- package/src/specs/vllm-reasoning-toolcalls.test.ts +340 -0
- package/src/tools/__tests__/CloudflareSandboxExecution.test.ts +192 -0
- package/src/tools/__tests__/LocalExecutionTools.test.ts +54 -0
- package/src/tools/cloudflare/CloudflareSandboxExecutionEngine.ts +106 -30
- package/src/tools/local/CompileCheckTool.ts +19 -3
- package/src/tools/local/FileCheckpointer.ts +20 -4
- package/src/tools/local/LocalCodingTools.ts +61 -8
- package/src/tools/local/__tests__/FileCheckpointer.test.ts +42 -0
- package/src/tools/local/syntaxCheck.ts +14 -2
- package/src/tools/local/workspaceFS.ts +27 -0
package/src/graphs/Graph.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
syncBudgetDerivedFields,
|
|
29
29
|
addTailCacheControl,
|
|
30
30
|
resolvePromptCacheTtl,
|
|
31
|
+
resolveBedrockPromptCacheTtl,
|
|
31
32
|
supportsBedrockToolCache,
|
|
32
33
|
getMessageId,
|
|
33
34
|
makeIsDeferred,
|
|
@@ -1880,9 +1881,14 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
|
|
|
1880
1881
|
const bedrockOptions = agentContext.clientOptions as
|
|
1881
1882
|
| t.BedrockAnthropicClientOptions
|
|
1882
1883
|
| undefined;
|
|
1884
|
+
// Non-Claude models (Nova) reject the extended 1h TTL, so resolve it
|
|
1885
|
+
// against the model — message/system caching stays on, clamped to 5m.
|
|
1883
1886
|
finalMessages = addBedrockTailCacheControl<BaseMessage>(
|
|
1884
1887
|
finalMessages,
|
|
1885
|
-
|
|
1888
|
+
resolveBedrockPromptCacheTtl(
|
|
1889
|
+
bedrockOptions?.promptCacheTtl,
|
|
1890
|
+
(bedrockOptions as { model?: string } | undefined)?.model
|
|
1891
|
+
)
|
|
1886
1892
|
);
|
|
1887
1893
|
}
|
|
1888
1894
|
|
package/src/llm/bedrock/index.ts
CHANGED
|
@@ -40,7 +40,7 @@ import {
|
|
|
40
40
|
handleConverseStreamMetadata,
|
|
41
41
|
} from './utils';
|
|
42
42
|
import {
|
|
43
|
-
|
|
43
|
+
resolveBedrockPromptCacheTtl,
|
|
44
44
|
supportsBedrockToolCache,
|
|
45
45
|
type PromptCacheTtl,
|
|
46
46
|
} from '@/messages/cache';
|
|
@@ -184,7 +184,7 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
|
|
|
184
184
|
? insertBedrockToolCachePoint(
|
|
185
185
|
baseParams.toolConfig,
|
|
186
186
|
true,
|
|
187
|
-
|
|
187
|
+
resolveBedrockPromptCacheTtl(this.promptCacheTtl, this.cacheModelId)
|
|
188
188
|
)
|
|
189
189
|
: baseParams.toolConfig;
|
|
190
190
|
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
buildAnthropicCacheControl,
|
|
20
20
|
buildBedrockCachePoint,
|
|
21
21
|
resolvePromptCacheTtl,
|
|
22
|
+
resolveBedrockPromptCacheTtl,
|
|
22
23
|
supportsBedrockToolCache,
|
|
23
24
|
DEFAULT_PROMPT_CACHE_TTL,
|
|
24
25
|
} from './cache';
|
|
@@ -926,6 +927,37 @@ describe('supportsBedrockToolCache', () => {
|
|
|
926
927
|
});
|
|
927
928
|
});
|
|
928
929
|
|
|
930
|
+
describe('resolveBedrockPromptCacheTtl', () => {
|
|
931
|
+
test('Claude: defaults to 1h and honors an explicit ttl', () => {
|
|
932
|
+
const claude = 'us.anthropic.claude-sonnet-4-5-20250929-v1:0';
|
|
933
|
+
expect(resolveBedrockPromptCacheTtl(undefined, claude)).toBe('1h');
|
|
934
|
+
expect(resolveBedrockPromptCacheTtl('1h', claude)).toBe('1h');
|
|
935
|
+
expect(resolveBedrockPromptCacheTtl('5m', claude)).toBe('5m');
|
|
936
|
+
});
|
|
937
|
+
|
|
938
|
+
test('Nova: clamps to 5m even when 1h is configured (extended TTL Anthropic-only)', () => {
|
|
939
|
+
const nova = 'us.amazon.nova-lite-v1:0';
|
|
940
|
+
expect(resolveBedrockPromptCacheTtl(undefined, nova)).toBe('5m');
|
|
941
|
+
expect(resolveBedrockPromptCacheTtl('1h', nova)).toBe('5m');
|
|
942
|
+
expect(resolveBedrockPromptCacheTtl('5m', nova)).toBe('5m');
|
|
943
|
+
});
|
|
944
|
+
|
|
945
|
+
test('other non-Claude families clamp to 5m', () => {
|
|
946
|
+
expect(
|
|
947
|
+
resolveBedrockPromptCacheTtl('1h', 'meta.llama3-1-70b-instruct-v1:0')
|
|
948
|
+
).toBe('5m');
|
|
949
|
+
expect(
|
|
950
|
+
resolveBedrockPromptCacheTtl('1h', 'mistral.mistral-large-2407-v1:0')
|
|
951
|
+
).toBe('5m');
|
|
952
|
+
});
|
|
953
|
+
|
|
954
|
+
test('omitted model defaults to Claude behavior (1h)', () => {
|
|
955
|
+
expect(resolveBedrockPromptCacheTtl(undefined, undefined)).toBe('1h');
|
|
956
|
+
expect(resolveBedrockPromptCacheTtl(undefined, null)).toBe('1h');
|
|
957
|
+
expect(resolveBedrockPromptCacheTtl('1h', undefined)).toBe('1h');
|
|
958
|
+
});
|
|
959
|
+
});
|
|
960
|
+
|
|
929
961
|
describe('stripAnthropicCacheControl', () => {
|
|
930
962
|
it('removes cache_control fields from content blocks', () => {
|
|
931
963
|
const messages: TestMsg[] = [
|
package/src/messages/cache.ts
CHANGED
|
@@ -79,6 +79,16 @@ export function buildBedrockCachePoint(
|
|
|
79
79
|
return ttl === '1h' ? { type: 'default', ttl: '1h' } : { type: 'default' };
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Whether a Bedrock model id is an Anthropic Claude model. Claude is the only
|
|
84
|
+
* Bedrock family that accepts the explicit cache features below; Amazon Nova and
|
|
85
|
+
* others reject them. Matches both the `anthropic.` and cross-region
|
|
86
|
+
* (`us.anthropic.`) / bare `claude` forms.
|
|
87
|
+
*/
|
|
88
|
+
function isBedrockAnthropicModel(model: string | undefined | null): boolean {
|
|
89
|
+
return typeof model === 'string' && /claude|anthropic/i.test(model);
|
|
90
|
+
}
|
|
91
|
+
|
|
82
92
|
/**
|
|
83
93
|
* A `cachePoint` under `toolConfig.tools` is only accepted on Anthropic Claude
|
|
84
94
|
* models. Amazon Nova rejects it outright — `Malformed input request:
|
|
@@ -89,8 +99,7 @@ export function buildBedrockCachePoint(
|
|
|
89
99
|
*
|
|
90
100
|
* Gate ONLY the tool checkpoint on this, so a `promptCache: true` config on Nova
|
|
91
101
|
* keeps its valid message/system caching instead of either 400-ing (tool point)
|
|
92
|
-
* or losing caching entirely.
|
|
93
|
-
* outright), distinct from the TTL value which Bedrock downgrades gracefully.
|
|
102
|
+
* or losing caching entirely.
|
|
94
103
|
*
|
|
95
104
|
* @see https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html
|
|
96
105
|
* @see https://github.com/danny-avila/LibreChat/issues/13838
|
|
@@ -98,10 +107,27 @@ export function buildBedrockCachePoint(
|
|
|
98
107
|
export function supportsBedrockToolCache(
|
|
99
108
|
model: string | undefined | null
|
|
100
109
|
): boolean {
|
|
101
|
-
|
|
102
|
-
|
|
110
|
+
return isBedrockAnthropicModel(model);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Resolve the Bedrock `cachePoint` TTL for a model. The extended `1h` TTL is
|
|
115
|
+
* Anthropic-only: non-Claude models (e.g. Nova) hard-reject it with
|
|
116
|
+
* `Extended TTL prompt caching is only supported for Anthropic models`, so they
|
|
117
|
+
* are clamped to `5m` even when `1h` is configured (verified live). An omitted
|
|
118
|
+
* model defaults to a Claude model, and Claude downgrades an unsupported `1h` to
|
|
119
|
+
* `5m` server-side, so the `1h` default is safe for the Claude/undefined paths.
|
|
120
|
+
*
|
|
121
|
+
* @see https://github.com/danny-avila/LibreChat/issues/13838
|
|
122
|
+
*/
|
|
123
|
+
export function resolveBedrockPromptCacheTtl(
|
|
124
|
+
ttl: PromptCacheTtl | undefined,
|
|
125
|
+
model: string | undefined | null
|
|
126
|
+
): PromptCacheTtl {
|
|
127
|
+
if (model != null && !isBedrockAnthropicModel(model)) {
|
|
128
|
+
return '5m';
|
|
103
129
|
}
|
|
104
|
-
return
|
|
130
|
+
return resolvePromptCacheTtl(ttl);
|
|
105
131
|
}
|
|
106
132
|
|
|
107
133
|
/**
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression coverage for a generic `custom` OpenAI-compatible endpoint
|
|
3
|
+
* (provider `openai`, non-OpenAI `baseURL`, default `reasoningKey`,
|
|
4
|
+
* non-standard model name) that streams reasoning in the modern vLLM
|
|
5
|
+
* `reasoning` field — `reasoning_content` is `null` throughout — and streams
|
|
6
|
+
* `tool_calls` with fragmented arguments (vLLM `--reasoning-parser qwen3` +
|
|
7
|
+
* `--tool-call-parser qwen3_coder`).
|
|
8
|
+
*
|
|
9
|
+
* A non-OpenAI `baseURL` is required so the model stays on the custom-endpoint
|
|
10
|
+
* (final-signal) path rather than the official-OpenAI sequential-seal path.
|
|
11
|
+
*
|
|
12
|
+
* The wire shapes mirror the captures in LibreChat discussion #13849:
|
|
13
|
+
* - reasoning must surface as `think` content (the Thoughts block), and
|
|
14
|
+
* - the fragmented streamed tool call must be assembled by the graph into a
|
|
15
|
+
* structured call and executed.
|
|
16
|
+
*/
|
|
17
|
+
import { DynamicStructuredTool } from '@langchain/core/tools';
|
|
18
|
+
import { AIMessage, HumanMessage } from '@langchain/core/messages';
|
|
19
|
+
import type { UsageMetadata } from '@langchain/core/messages';
|
|
20
|
+
import type { OpenAIClient } from '@langchain/openai';
|
|
21
|
+
import type * as t from '@/types';
|
|
22
|
+
import { ContentTypes, GraphEvents, Providers, StepTypes } from '@/common';
|
|
23
|
+
import { ToolEndHandler, ModelEndHandler } from '@/events';
|
|
24
|
+
import { createContentAggregator } from '@/stream';
|
|
25
|
+
import { ChatOpenAI } from '@/llm/openai';
|
|
26
|
+
import { Run } from '@/run';
|
|
27
|
+
|
|
28
|
+
type CompletionChunk = OpenAIClient.Chat.Completions.ChatCompletionChunk;
|
|
29
|
+
type CompletionChoice =
|
|
30
|
+
OpenAIClient.Chat.Completions.ChatCompletionChunk.Choice;
|
|
31
|
+
type CompletionDelta = CompletionChoice['delta'] & {
|
|
32
|
+
reasoning?: string | null;
|
|
33
|
+
reasoning_content?: string | null;
|
|
34
|
+
};
|
|
35
|
+
type StreamingCompletions = {
|
|
36
|
+
completionWithRetry: () => Promise<AsyncIterable<CompletionChunk>>;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const MODEL = 'custom_llm_thinking';
|
|
40
|
+
const BASE_URL = 'http://vllm.internal:8000/v1';
|
|
41
|
+
|
|
42
|
+
function completionChunk(
|
|
43
|
+
delta: CompletionDelta,
|
|
44
|
+
finishReason: CompletionChoice['finish_reason'] = null
|
|
45
|
+
): CompletionChunk {
|
|
46
|
+
return {
|
|
47
|
+
id: 'chatcmpl-vllm',
|
|
48
|
+
object: 'chat.completion.chunk',
|
|
49
|
+
created: 0,
|
|
50
|
+
model: MODEL,
|
|
51
|
+
choices: [{ index: 0, delta, finish_reason: finishReason }],
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function customEndpointModel(): ChatOpenAI {
|
|
56
|
+
return new ChatOpenAI({
|
|
57
|
+
model: MODEL,
|
|
58
|
+
apiKey: 'test-key',
|
|
59
|
+
streaming: true,
|
|
60
|
+
configuration: { baseURL: BASE_URL },
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function setCompletionStream(
|
|
65
|
+
model: ChatOpenAI,
|
|
66
|
+
stream: () => AsyncIterable<CompletionChunk>
|
|
67
|
+
): void {
|
|
68
|
+
(
|
|
69
|
+
model as unknown as { completions: StreamingCompletions }
|
|
70
|
+
).completions.completionWithRetry = async () => stream();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
describe('custom OpenAI-compatible endpoint (vLLM reasoning + qwen3_coder tool calls)', () => {
|
|
74
|
+
const config = {
|
|
75
|
+
configurable: { thread_id: 'vllm-reasoning-toolcalls' },
|
|
76
|
+
streamMode: 'values' as const,
|
|
77
|
+
version: 'v2' as const,
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
it('renders reasoning from delta.reasoning when reasoning_content is null', async () => {
|
|
81
|
+
const reasoningTokens = [
|
|
82
|
+
'Here',
|
|
83
|
+
'\'s a thinking',
|
|
84
|
+
' process: 17*23',
|
|
85
|
+
' = 391',
|
|
86
|
+
];
|
|
87
|
+
const contentTokens = ['\n\nUm ', '17 ×', ' 23 = **391', '**.'];
|
|
88
|
+
|
|
89
|
+
const reasoningDeltas: t.ReasoningDeltaEvent[] = [];
|
|
90
|
+
const messageDeltas: t.MessageDeltaEvent[] = [];
|
|
91
|
+
const { contentParts, aggregateContent } = createContentAggregator();
|
|
92
|
+
|
|
93
|
+
const run = await Run.create<t.IState>({
|
|
94
|
+
runId: 'vllm-reasoning',
|
|
95
|
+
graphConfig: {
|
|
96
|
+
type: 'standard',
|
|
97
|
+
llmConfig: { provider: Providers.OPENAI, streamUsage: false },
|
|
98
|
+
},
|
|
99
|
+
returnContent: true,
|
|
100
|
+
skipCleanup: true,
|
|
101
|
+
customHandlers: {
|
|
102
|
+
[GraphEvents.ON_RUN_STEP]: {
|
|
103
|
+
handle: (
|
|
104
|
+
event: GraphEvents.ON_RUN_STEP,
|
|
105
|
+
data: t.StreamEventData
|
|
106
|
+
): void => aggregateContent({ event, data: data as t.RunStep }),
|
|
107
|
+
},
|
|
108
|
+
[GraphEvents.ON_MESSAGE_DELTA]: {
|
|
109
|
+
handle: (
|
|
110
|
+
event: GraphEvents.ON_MESSAGE_DELTA,
|
|
111
|
+
data: t.StreamEventData
|
|
112
|
+
): void => {
|
|
113
|
+
messageDeltas.push(data as t.MessageDeltaEvent);
|
|
114
|
+
aggregateContent({ event, data: data as t.MessageDeltaEvent });
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
[GraphEvents.ON_REASONING_DELTA]: {
|
|
118
|
+
handle: (
|
|
119
|
+
event: GraphEvents.ON_REASONING_DELTA,
|
|
120
|
+
data: t.StreamEventData
|
|
121
|
+
): void => {
|
|
122
|
+
reasoningDeltas.push(data as t.ReasoningDeltaEvent);
|
|
123
|
+
aggregateContent({ event, data: data as t.ReasoningDeltaEvent });
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
if (!run.Graph) {
|
|
129
|
+
throw new Error('Expected graph to be initialized');
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const model = customEndpointModel();
|
|
133
|
+
setCompletionStream(
|
|
134
|
+
model,
|
|
135
|
+
async function* streamChunks(): AsyncGenerator<CompletionChunk> {
|
|
136
|
+
yield completionChunk({ role: 'assistant', content: '' });
|
|
137
|
+
for (const reasoning of reasoningTokens) {
|
|
138
|
+
yield completionChunk({
|
|
139
|
+
reasoning,
|
|
140
|
+
reasoning_content: null,
|
|
141
|
+
content: null,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
for (let i = 0; i < contentTokens.length; i++) {
|
|
145
|
+
yield completionChunk(
|
|
146
|
+
{
|
|
147
|
+
reasoning: null,
|
|
148
|
+
reasoning_content: null,
|
|
149
|
+
content: contentTokens[i],
|
|
150
|
+
},
|
|
151
|
+
i === contentTokens.length - 1 ? 'stop' : null
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
);
|
|
156
|
+
run.Graph.overrideModel = model as t.ChatModel;
|
|
157
|
+
|
|
158
|
+
await run.processStream(
|
|
159
|
+
{
|
|
160
|
+
messages: [
|
|
161
|
+
new HumanMessage('Was ist 17*23? Denk Schritt für Schritt.'),
|
|
162
|
+
],
|
|
163
|
+
},
|
|
164
|
+
config
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
const thoughts = reasoningDeltas
|
|
168
|
+
.flatMap((delta) => delta.delta.content ?? [])
|
|
169
|
+
.map((part) => (part as { think?: string }).think ?? '')
|
|
170
|
+
.join('');
|
|
171
|
+
const answer = messageDeltas
|
|
172
|
+
.flatMap((delta) => delta.delta.content ?? [])
|
|
173
|
+
.map((part) => (part as { text?: string }).text ?? '')
|
|
174
|
+
.join('');
|
|
175
|
+
|
|
176
|
+
expect(reasoningDeltas).toHaveLength(reasoningTokens.length);
|
|
177
|
+
expect(thoughts).toBe(reasoningTokens.join(''));
|
|
178
|
+
expect(answer).toBe(contentTokens.join(''));
|
|
179
|
+
expect(contentParts.map((part) => part?.type)).toEqual([
|
|
180
|
+
ContentTypes.THINK,
|
|
181
|
+
ContentTypes.TEXT,
|
|
182
|
+
]);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it('assembles fragmented streamed tool_calls and executes the tool through the graph', async () => {
|
|
186
|
+
let weatherArgs: unknown;
|
|
187
|
+
const getWeather = new DynamicStructuredTool({
|
|
188
|
+
name: 'get_weather',
|
|
189
|
+
description: 'Get the current weather for a location',
|
|
190
|
+
schema: {
|
|
191
|
+
type: 'object',
|
|
192
|
+
properties: { location: { type: 'string' } },
|
|
193
|
+
required: ['location'],
|
|
194
|
+
},
|
|
195
|
+
func: async (input: unknown): Promise<string> => {
|
|
196
|
+
weatherArgs = input;
|
|
197
|
+
return JSON.stringify({
|
|
198
|
+
location: 'Berlin',
|
|
199
|
+
summary: 'sunny',
|
|
200
|
+
temp_c: 18,
|
|
201
|
+
});
|
|
202
|
+
},
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
const { aggregateContent } = createContentAggregator();
|
|
206
|
+
const collectedUsage: UsageMetadata[] = [];
|
|
207
|
+
const runSteps: t.RunStep[] = [];
|
|
208
|
+
const streamedToolArgs: string[] = [];
|
|
209
|
+
|
|
210
|
+
const run = await Run.create<t.IState>({
|
|
211
|
+
runId: 'vllm-tool-calls',
|
|
212
|
+
graphConfig: {
|
|
213
|
+
type: 'standard',
|
|
214
|
+
llmConfig: { provider: Providers.OPENAI, streamUsage: false },
|
|
215
|
+
tools: [getWeather],
|
|
216
|
+
},
|
|
217
|
+
returnContent: true,
|
|
218
|
+
skipCleanup: true,
|
|
219
|
+
customHandlers: {
|
|
220
|
+
[GraphEvents.TOOL_END]: new ToolEndHandler(),
|
|
221
|
+
[GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
|
|
222
|
+
[GraphEvents.ON_RUN_STEP]: {
|
|
223
|
+
handle: (
|
|
224
|
+
event: GraphEvents.ON_RUN_STEP,
|
|
225
|
+
data: t.StreamEventData
|
|
226
|
+
): void => {
|
|
227
|
+
runSteps.push(data as t.RunStep);
|
|
228
|
+
aggregateContent({ event, data: data as t.RunStep });
|
|
229
|
+
},
|
|
230
|
+
},
|
|
231
|
+
[GraphEvents.ON_RUN_STEP_DELTA]: {
|
|
232
|
+
handle: (
|
|
233
|
+
_event: GraphEvents.ON_RUN_STEP_DELTA,
|
|
234
|
+
data: t.StreamEventData
|
|
235
|
+
): void => {
|
|
236
|
+
const { delta } = data as t.RunStepDeltaEvent;
|
|
237
|
+
if (
|
|
238
|
+
delta.type !== StepTypes.TOOL_CALLS ||
|
|
239
|
+
delta.tool_calls == null
|
|
240
|
+
) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
for (const toolCallDelta of delta.tool_calls) {
|
|
244
|
+
if (typeof toolCallDelta.args === 'string') {
|
|
245
|
+
streamedToolArgs.push(toolCallDelta.args);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
},
|
|
249
|
+
},
|
|
250
|
+
[GraphEvents.ON_RUN_STEP_COMPLETED]: {
|
|
251
|
+
handle: (
|
|
252
|
+
event: GraphEvents.ON_RUN_STEP_COMPLETED,
|
|
253
|
+
data: t.StreamEventData
|
|
254
|
+
): void =>
|
|
255
|
+
aggregateContent({
|
|
256
|
+
event,
|
|
257
|
+
data: data as unknown as { result: t.ToolEndEvent },
|
|
258
|
+
}),
|
|
259
|
+
},
|
|
260
|
+
[GraphEvents.ON_MESSAGE_DELTA]: {
|
|
261
|
+
handle: (
|
|
262
|
+
event: GraphEvents.ON_MESSAGE_DELTA,
|
|
263
|
+
data: t.StreamEventData
|
|
264
|
+
): void =>
|
|
265
|
+
aggregateContent({ event, data: data as t.MessageDeltaEvent }),
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
if (!run.Graph) {
|
|
270
|
+
throw new Error('Expected graph to be initialized');
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const model = customEndpointModel();
|
|
274
|
+
async function* toolCallStream(): AsyncGenerator<CompletionChunk> {
|
|
275
|
+
yield completionChunk({ role: 'assistant', content: '\n\n' });
|
|
276
|
+
yield completionChunk({
|
|
277
|
+
tool_calls: [
|
|
278
|
+
{
|
|
279
|
+
id: 'call_0065144d618f4f33be1491af',
|
|
280
|
+
type: 'function',
|
|
281
|
+
index: 0,
|
|
282
|
+
function: { name: 'get_weather', arguments: '' },
|
|
283
|
+
},
|
|
284
|
+
],
|
|
285
|
+
});
|
|
286
|
+
yield completionChunk({
|
|
287
|
+
tool_calls: [{ index: 0, function: { arguments: '{' } }],
|
|
288
|
+
});
|
|
289
|
+
yield completionChunk({
|
|
290
|
+
tool_calls: [
|
|
291
|
+
{ index: 0, function: { arguments: '"location": "Berlin"' } },
|
|
292
|
+
],
|
|
293
|
+
});
|
|
294
|
+
yield completionChunk(
|
|
295
|
+
{ tool_calls: [{ index: 0, function: { arguments: '}' } }] },
|
|
296
|
+
'tool_calls'
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
async function* finalAnswerStream(): AsyncGenerator<CompletionChunk> {
|
|
300
|
+
yield completionChunk({
|
|
301
|
+
role: 'assistant',
|
|
302
|
+
content: 'Das Wetter in Berlin ist sonnig.',
|
|
303
|
+
});
|
|
304
|
+
yield completionChunk({ content: '' }, 'stop');
|
|
305
|
+
}
|
|
306
|
+
let modelCall = 0;
|
|
307
|
+
setCompletionStream(model, () => {
|
|
308
|
+
modelCall += 1;
|
|
309
|
+
return modelCall === 1 ? toolCallStream() : finalAnswerStream();
|
|
310
|
+
});
|
|
311
|
+
run.Graph.overrideModel = model as t.ChatModel;
|
|
312
|
+
|
|
313
|
+
await run.processStream(
|
|
314
|
+
{ messages: [new HumanMessage('Wie ist das Wetter in Berlin?')] },
|
|
315
|
+
config
|
|
316
|
+
);
|
|
317
|
+
|
|
318
|
+
const toolCallStepNames = runSteps
|
|
319
|
+
.filter((step) => step.stepDetails.type === StepTypes.TOOL_CALLS)
|
|
320
|
+
.flatMap(
|
|
321
|
+
(step) => (step.stepDetails as t.ToolCallsDetails).tool_calls ?? []
|
|
322
|
+
)
|
|
323
|
+
.map((call) => ('function' in call ? call.function.name : call.name));
|
|
324
|
+
|
|
325
|
+
const messages = run.getRunMessages() ?? [];
|
|
326
|
+
const finalAnswer = messages
|
|
327
|
+
.filter((message): message is AIMessage => message.getType() === 'ai')
|
|
328
|
+
.map((message) =>
|
|
329
|
+
typeof message.content === 'string' ? message.content : ''
|
|
330
|
+
)
|
|
331
|
+
.join('');
|
|
332
|
+
|
|
333
|
+
expect(modelCall).toBe(2);
|
|
334
|
+
expect(toolCallStepNames).toContain('get_weather');
|
|
335
|
+
expect(streamedToolArgs.join('')).toBe('{"location": "Berlin"}');
|
|
336
|
+
expect(weatherArgs).toEqual({ location: 'Berlin' });
|
|
337
|
+
expect(messages.some((message) => message.getType() === 'tool')).toBe(true);
|
|
338
|
+
expect(finalAnswer).toContain('Das Wetter in Berlin ist sonnig.');
|
|
339
|
+
});
|
|
340
|
+
});
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
createCloudflareBashProgrammaticToolCallingTool,
|
|
11
11
|
createCloudflareProgrammaticToolCallingTool,
|
|
12
12
|
} from '../cloudflare/CloudflareProgrammaticToolCalling';
|
|
13
|
+
import { isWorkspaceClientTimeoutError } from '../local/workspaceFS';
|
|
13
14
|
import { createCloudflareBridgeRuntime } from '../cloudflare/CloudflareBridgeRuntime';
|
|
14
15
|
import { resolveLocalToolsForBinding } from '../local/resolveLocalExecutionTools';
|
|
15
16
|
import { spawnLocalProcess } from '../local/LocalExecutionEngine';
|
|
@@ -388,6 +389,197 @@ describe('Cloudflare sandbox execution backend', () => {
|
|
|
388
389
|
expect(received).not.toHaveProperty('signal');
|
|
389
390
|
});
|
|
390
391
|
|
|
392
|
+
it('rejects with a client-side timeout when sandbox readFile stalls', async () => {
|
|
393
|
+
// The native-DO file-IO RPCs (readFile/writeFile/listFiles/...) have the same
|
|
394
|
+
// stall hazard exec() does: no signal, no enforced timeout. A cold/unresponsive
|
|
395
|
+
// container otherwise hangs the host await until the run-level abort, burning
|
|
396
|
+
// the whole budget on one read (observed live: a `read_file` stalled ~552s).
|
|
397
|
+
jest.useFakeTimers();
|
|
398
|
+
try {
|
|
399
|
+
let readCalls = 0;
|
|
400
|
+
const fs = createCloudflareWorkspaceFS({
|
|
401
|
+
workspaceRoot: '/workspace',
|
|
402
|
+
timeoutMs: 1000,
|
|
403
|
+
sandbox: createRuntime({
|
|
404
|
+
readFile: () => {
|
|
405
|
+
readCalls += 1;
|
|
406
|
+
return new Promise<string>(() => undefined);
|
|
407
|
+
},
|
|
408
|
+
}),
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
const error = (fs.readFile as (p: string) => Promise<unknown>)(
|
|
412
|
+
'/workspace/a.txt'
|
|
413
|
+
).catch((e: unknown) => e);
|
|
414
|
+
// Client backstop = clientFsTimeoutMs(1000) = 6000ms; advance past it.
|
|
415
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
416
|
+
const settled = await error;
|
|
417
|
+
// Must be the DISTINGUISHABLE timeout error so ENOENT-only callers rethrow
|
|
418
|
+
// it instead of mistaking a stalled read for a missing file.
|
|
419
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
420
|
+
expect((settled as Error).message).toMatch(/client-side timeout/);
|
|
421
|
+
expect(readCalls).toBe(1);
|
|
422
|
+
} finally {
|
|
423
|
+
jest.useRealTimers();
|
|
424
|
+
}
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
it('keeps the backstop active while draining a streamed file read', async () => {
|
|
428
|
+
// sandbox.readFile resolves to { content: ReadableStream } whose stream never
|
|
429
|
+
// ends. The race must cover the drain (normalizeReadFileContent), not just the
|
|
430
|
+
// initial RPC, or read_file/open/stat still hang to the run-level abort.
|
|
431
|
+
jest.useFakeTimers();
|
|
432
|
+
try {
|
|
433
|
+
const fs = createCloudflareWorkspaceFS({
|
|
434
|
+
workspaceRoot: '/workspace',
|
|
435
|
+
timeoutMs: 1000,
|
|
436
|
+
sandbox: createRuntime({
|
|
437
|
+
readFile: async () => ({
|
|
438
|
+
content: new ReadableStream<Uint8Array>({
|
|
439
|
+
// start() never enqueues or closes -> the drain stalls forever.
|
|
440
|
+
start() {},
|
|
441
|
+
}),
|
|
442
|
+
}),
|
|
443
|
+
}),
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
const error = (fs.readFile as (p: string) => Promise<unknown>)(
|
|
447
|
+
'/workspace/a.txt'
|
|
448
|
+
).catch((e: unknown) => e);
|
|
449
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
450
|
+
const settled = await error;
|
|
451
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
452
|
+
} finally {
|
|
453
|
+
jest.useRealTimers();
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
it('rethrows a stat directory-probe timeout instead of falling through to readFile', async () => {
|
|
458
|
+
// findChildInfo returns nothing -> the directory probe (listFiles) runs; if it
|
|
459
|
+
// STALLS it must surface, not fall through to the readFile branch (which would
|
|
460
|
+
// burn a SECOND full backstop, ~2x the timeout, before the caller sees it).
|
|
461
|
+
jest.useFakeTimers();
|
|
462
|
+
try {
|
|
463
|
+
let readFileCalls = 0;
|
|
464
|
+
const fs = createCloudflareWorkspaceFS({
|
|
465
|
+
workspaceRoot: '/workspace',
|
|
466
|
+
timeoutMs: 1000,
|
|
467
|
+
sandbox: createRuntime({
|
|
468
|
+
listFiles: (dir) =>
|
|
469
|
+
dir === '/workspace/probe-me'
|
|
470
|
+
? new Promise(() => undefined) // the probe stalls
|
|
471
|
+
: Promise.resolve([]), // findChildInfo's parent listing returns fast
|
|
472
|
+
readFile: () => {
|
|
473
|
+
readFileCalls += 1;
|
|
474
|
+
return Promise.resolve('');
|
|
475
|
+
},
|
|
476
|
+
}),
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
const error = fs.stat('/workspace/probe-me').catch((e: unknown) => e);
|
|
480
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
481
|
+
const settled = await error;
|
|
482
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
483
|
+
// Must NOT have fallen through to the readFile probe.
|
|
484
|
+
expect(readFileCalls).toBe(0);
|
|
485
|
+
} finally {
|
|
486
|
+
jest.useRealTimers();
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
it('bounds execute_code temp-dir setup RPCs (mkdir/writeFile) that stall', async () => {
|
|
491
|
+
jest.useFakeTimers();
|
|
492
|
+
try {
|
|
493
|
+
const sandbox = createRuntime({
|
|
494
|
+
// exec would resolve fine; the stall is in the pre-exec mkdir setup.
|
|
495
|
+
mkdir: () => new Promise<{ ok: true }>(() => undefined),
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
const promise = executeCloudflareCode(
|
|
499
|
+
{ lang: 'py', code: 'print("hi")' },
|
|
500
|
+
{ sandbox, workspaceRoot: '/workspace', timeoutMs: 1000 }
|
|
501
|
+
);
|
|
502
|
+
const assertion = expect(promise).rejects.toThrow(/client-side timeout/);
|
|
503
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
504
|
+
await assertion;
|
|
505
|
+
} finally {
|
|
506
|
+
jest.useRealTimers();
|
|
507
|
+
}
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it('still cleans up the temp dir when execute_code setup (writeFile) times out', async () => {
|
|
511
|
+
// The setup RPCs are inside the try, so a stalled writeFile still triggers
|
|
512
|
+
// the finally cleanup — otherwise the late (uncancellable) write leaves an
|
|
513
|
+
// orphaned .lc-exec/<uuid> dir behind on every cold-container failure.
|
|
514
|
+
jest.useFakeTimers();
|
|
515
|
+
try {
|
|
516
|
+
const execCommands: string[] = [];
|
|
517
|
+
const sandbox = createRuntime({
|
|
518
|
+
mkdir: async () => ({ ok: true }),
|
|
519
|
+
writeFile: () => new Promise<{ ok: true }>(() => undefined), // stalls
|
|
520
|
+
exec: async (command) => {
|
|
521
|
+
execCommands.push(command);
|
|
522
|
+
return { exitCode: 0, stdout: '', stderr: '' };
|
|
523
|
+
},
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
const promise = executeCloudflareCode(
|
|
527
|
+
{ lang: 'py', code: 'print("hi")' },
|
|
528
|
+
{ sandbox, workspaceRoot: '/workspace', timeoutMs: 1000 }
|
|
529
|
+
).catch((e: unknown) => e);
|
|
530
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
531
|
+
const settled = await promise;
|
|
532
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
533
|
+
// Cleanup must have been issued despite the setup failure.
|
|
534
|
+
expect(execCommands.some((c) => c.startsWith('rm -rf'))).toBe(true);
|
|
535
|
+
} finally {
|
|
536
|
+
jest.useRealTimers();
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
|
|
540
|
+
it('rejects with a client-side timeout when sandbox listFiles stalls', async () => {
|
|
541
|
+
jest.useFakeTimers();
|
|
542
|
+
try {
|
|
543
|
+
const fs = createCloudflareWorkspaceFS({
|
|
544
|
+
workspaceRoot: '/workspace',
|
|
545
|
+
timeoutMs: 1000,
|
|
546
|
+
sandbox: createRuntime({
|
|
547
|
+
listFiles: () =>
|
|
548
|
+
new Promise<t.CloudflareSandboxFileInfo[]>(() => undefined),
|
|
549
|
+
}),
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
const promise = (fs.readdir as (p: string) => Promise<unknown>)(
|
|
553
|
+
'/workspace/sub'
|
|
554
|
+
);
|
|
555
|
+
const assertion = expect(promise).rejects.toThrow(/client-side timeout/);
|
|
556
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
557
|
+
await assertion;
|
|
558
|
+
} finally {
|
|
559
|
+
jest.useRealTimers();
|
|
560
|
+
}
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
it('does not time out a native FS RPC that returns in time', async () => {
|
|
564
|
+
jest.useFakeTimers();
|
|
565
|
+
try {
|
|
566
|
+
const fs = createCloudflareWorkspaceFS({
|
|
567
|
+
workspaceRoot: '/workspace',
|
|
568
|
+
timeoutMs: 1000,
|
|
569
|
+
sandbox: createRuntime({ readFile: async () => 'contents' }),
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
const result = await (
|
|
573
|
+
fs.readFile as (p: string, e: 'utf8') => Promise<string>
|
|
574
|
+
)('/workspace/a.txt', 'utf8');
|
|
575
|
+
expect(result).toBe('contents');
|
|
576
|
+
// The backstop timer must have been cleared, not left dangling.
|
|
577
|
+
expect(jest.getTimerCount()).toBe(0);
|
|
578
|
+
} finally {
|
|
579
|
+
jest.useRealTimers();
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
|
|
391
583
|
it('passes call-specific timeouts to the Cloudflare spawn wrapper', async () => {
|
|
392
584
|
let execCommand = '';
|
|
393
585
|
let execTimeout: number | undefined;
|