@librechat/agents 3.2.33 → 3.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/llm/bedrock/index.cjs +21 -2
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/utils/message_outputs.cjs +38 -2
- package/dist/cjs/llm/bedrock/utils/message_outputs.cjs.map +1 -1
- package/dist/cjs/llm/google/utils/common.cjs +6 -0
- package/dist/cjs/llm/google/utils/common.cjs.map +1 -1
- package/dist/cjs/llm/openai/index.cjs +48 -1
- package/dist/cjs/llm/openai/index.cjs.map +1 -1
- package/dist/cjs/llm/vertexai/index.cjs +19 -0
- package/dist/cjs/llm/vertexai/index.cjs.map +1 -1
- package/dist/cjs/stream.cjs +20 -2
- package/dist/cjs/stream.cjs.map +1 -1
- package/dist/cjs/tools/ToolNode.cjs +41 -4
- package/dist/cjs/tools/ToolNode.cjs.map +1 -1
- package/dist/cjs/tools/streamedToolCallSeals.cjs +30 -1
- package/dist/cjs/tools/streamedToolCallSeals.cjs.map +1 -1
- package/dist/esm/llm/bedrock/index.mjs +22 -3
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/llm/bedrock/utils/message_outputs.mjs +38 -3
- package/dist/esm/llm/bedrock/utils/message_outputs.mjs.map +1 -1
- package/dist/esm/llm/google/utils/common.mjs +6 -0
- package/dist/esm/llm/google/utils/common.mjs.map +1 -1
- package/dist/esm/llm/openai/index.mjs +48 -1
- package/dist/esm/llm/openai/index.mjs.map +1 -1
- package/dist/esm/llm/vertexai/index.mjs +19 -0
- package/dist/esm/llm/vertexai/index.mjs.map +1 -1
- package/dist/esm/stream.mjs +21 -3
- package/dist/esm/stream.mjs.map +1 -1
- package/dist/esm/tools/ToolNode.mjs +41 -4
- package/dist/esm/tools/ToolNode.mjs.map +1 -1
- package/dist/esm/tools/streamedToolCallSeals.mjs +25 -2
- package/dist/esm/tools/streamedToolCallSeals.mjs.map +1 -1
- package/dist/types/llm/bedrock/utils/index.d.ts +1 -1
- package/dist/types/llm/bedrock/utils/message_outputs.d.ts +9 -0
- package/dist/types/llm/vertexai/index.d.ts +10 -0
- package/dist/types/tools/ToolNode.d.ts +8 -0
- package/dist/types/tools/streamedToolCallSeals.d.ts +5 -1
- package/dist/types/types/tools.d.ts +10 -0
- package/package.json +1 -1
- package/src/__tests__/stream.eagerEventExecution.test.ts +703 -0
- package/src/llm/bedrock/index.ts +40 -0
- package/src/llm/bedrock/streamSealDispatch.test.ts +158 -0
- package/src/llm/bedrock/utils/index.ts +1 -0
- package/src/llm/bedrock/utils/message_outputs.test.ts +85 -0
- package/src/llm/bedrock/utils/message_outputs.ts +43 -0
- package/src/llm/google/utils/common.test.ts +64 -0
- package/src/llm/google/utils/common.ts +18 -0
- package/src/llm/openai/index.ts +95 -1
- package/src/llm/openai/sequentialToolCallSeals.test.ts +199 -0
- package/src/llm/vertexai/index.ts +31 -0
- package/src/llm/vertexai/sealStreamedToolCalls.test.ts +88 -0
- package/src/llm/vertexai/streamSealDispatch.test.ts +148 -0
- package/src/stream.ts +40 -6
- package/src/tools/ToolNode.ts +85 -3
- package/src/tools/__tests__/ToolNode.onResultCompletion.test.ts +368 -0
- package/src/tools/streamedToolCallSeals.ts +37 -9
- package/src/types/tools.ts +10 -0
package/src/llm/bedrock/index.ts
CHANGED
|
@@ -34,6 +34,7 @@ import type { BaseMessage, ResponseMetadata } from '@langchain/core/messages';
|
|
|
34
34
|
import type { ChatBedrockConverseInput } from '@langchain/aws';
|
|
35
35
|
import {
|
|
36
36
|
convertToConverseMessages,
|
|
37
|
+
createConverseToolUseStopChunk,
|
|
37
38
|
handleConverseStreamContentBlockStart,
|
|
38
39
|
handleConverseStreamContentBlockDelta,
|
|
39
40
|
handleConverseStreamMetadata,
|
|
@@ -224,6 +225,15 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
|
|
|
224
225
|
}
|
|
225
226
|
|
|
226
227
|
const seenBlockIndices = new Set<number>();
|
|
228
|
+
const toolUseBlockIndices = new Set<number>();
|
|
229
|
+
/**
|
|
230
|
+
* Guardrails can reject an already-streamed toolUse block at
|
|
231
|
+
* `messageStop` (`guardrail_intervened`), after `contentBlockStop` has
|
|
232
|
+
* passed. Only emit eager-execution seals when no guardrails are
|
|
233
|
+
* configured, so a later intervention can't race an eagerly started tool.
|
|
234
|
+
*/
|
|
235
|
+
const sealToolUseOnStop =
|
|
236
|
+
options.guardrailConfig == null && this.guardrailConfig == null;
|
|
227
237
|
|
|
228
238
|
for await (const event of response.stream) {
|
|
229
239
|
if (event.contentBlockStart != null) {
|
|
@@ -234,8 +244,23 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
|
|
|
234
244
|
const idx = event.contentBlockStart.contentBlockIndex;
|
|
235
245
|
if (idx != null) {
|
|
236
246
|
seenBlockIndices.add(idx);
|
|
247
|
+
if (event.contentBlockStart.start?.toolUse != null) {
|
|
248
|
+
toolUseBlockIndices.add(idx);
|
|
249
|
+
}
|
|
237
250
|
}
|
|
238
251
|
yield this.enrichChunk(startChunk, seenBlockIndices);
|
|
252
|
+
|
|
253
|
+
// Registered stream handlers receive chunks through callback
|
|
254
|
+
// events, not the yielded generator — dispatch the start chunk so
|
|
255
|
+
// they see the tool call's id/name (eager chunk state needs both).
|
|
256
|
+
await runManager?.handleLLMNewToken(
|
|
257
|
+
startChunk.text,
|
|
258
|
+
undefined,
|
|
259
|
+
undefined,
|
|
260
|
+
undefined,
|
|
261
|
+
undefined,
|
|
262
|
+
{ chunk: startChunk }
|
|
263
|
+
);
|
|
239
264
|
}
|
|
240
265
|
} else if (event.contentBlockDelta != null) {
|
|
241
266
|
const deltaChunk = handleConverseStreamContentBlockDelta(
|
|
@@ -263,6 +288,21 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
|
|
|
263
288
|
const stopIdx = event.contentBlockStop.contentBlockIndex;
|
|
264
289
|
if (stopIdx != null) {
|
|
265
290
|
seenBlockIndices.add(stopIdx);
|
|
291
|
+
if (sealToolUseOnStop && toolUseBlockIndices.has(stopIdx)) {
|
|
292
|
+
// Converse guarantees the block's input is complete at stop, so
|
|
293
|
+
// emit an explicit seal chunk for eager tool execution — through
|
|
294
|
+
// the callback path too, for registered stream handlers.
|
|
295
|
+
const sealChunk = createConverseToolUseStopChunk(stopIdx);
|
|
296
|
+
yield sealChunk;
|
|
297
|
+
await runManager?.handleLLMNewToken(
|
|
298
|
+
sealChunk.text,
|
|
299
|
+
undefined,
|
|
300
|
+
undefined,
|
|
301
|
+
undefined,
|
|
302
|
+
undefined,
|
|
303
|
+
{ chunk: sealChunk }
|
|
304
|
+
);
|
|
305
|
+
}
|
|
266
306
|
}
|
|
267
307
|
} else {
|
|
268
308
|
yield new ChatGenerationChunk({
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { expect, test, describe, jest } from '@jest/globals';
|
|
2
|
+
import { HumanMessage, AIMessageChunk } from '@langchain/core/messages';
|
|
3
|
+
import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
|
|
4
|
+
import type { ChatGenerationChunk } from '@langchain/core/outputs';
|
|
5
|
+
import {
|
|
6
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY,
|
|
7
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
|
|
8
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
9
|
+
} from '@/tools/streamedToolCallSeals';
|
|
10
|
+
import { CustomChatBedrockConverse } from './index';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Registered stream handlers consume chunks through `handleLLMNewToken`
|
|
14
|
+
* callback events, not the yielded generator (`attemptInvoke` skips manual
|
|
15
|
+
* dispatch when a handler is registered). These tests drive the Converse
|
|
16
|
+
* stream loop with a stubbed client and assert that toolUse start and stop
|
|
17
|
+
* seal chunks reach BOTH paths.
|
|
18
|
+
*/
|
|
19
|
+
describe('Converse stream seal dispatch', () => {
|
|
20
|
+
async function runStream(
|
|
21
|
+
events: Array<Record<string, unknown>>,
|
|
22
|
+
modelFields: Record<string, unknown> = {}
|
|
23
|
+
): Promise<{
|
|
24
|
+
yielded: AIMessageChunk[];
|
|
25
|
+
dispatched: AIMessageChunk[];
|
|
26
|
+
}> {
|
|
27
|
+
const model = new CustomChatBedrockConverse({
|
|
28
|
+
model: 'anthropic.claude-3-5-sonnet-20240620-v1:0',
|
|
29
|
+
region: 'us-east-1',
|
|
30
|
+
credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
|
|
31
|
+
...modelFields,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
(model as unknown as { client: { send: unknown } }).client.send = jest.fn(
|
|
35
|
+
async () => ({
|
|
36
|
+
stream: (async function* () {
|
|
37
|
+
yield* events;
|
|
38
|
+
})(),
|
|
39
|
+
})
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
const dispatched: AIMessageChunk[] = [];
|
|
43
|
+
const runManager = {
|
|
44
|
+
handleLLMNewToken: jest.fn(
|
|
45
|
+
async (
|
|
46
|
+
_token: string,
|
|
47
|
+
_idx?: unknown,
|
|
48
|
+
_runId?: unknown,
|
|
49
|
+
_parentRunId?: unknown,
|
|
50
|
+
_tags?: unknown,
|
|
51
|
+
fields?: { chunk?: ChatGenerationChunk }
|
|
52
|
+
) => {
|
|
53
|
+
const message = fields?.chunk?.message;
|
|
54
|
+
if (message instanceof AIMessageChunk) {
|
|
55
|
+
dispatched.push(message);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
),
|
|
59
|
+
} as unknown as CallbackManagerForLLMRun;
|
|
60
|
+
|
|
61
|
+
const yielded: AIMessageChunk[] = [];
|
|
62
|
+
for await (const chunk of model._streamResponseChunks(
|
|
63
|
+
[new HumanMessage('hi')],
|
|
64
|
+
{} as Parameters<CustomChatBedrockConverse['_streamResponseChunks']>[1],
|
|
65
|
+
runManager
|
|
66
|
+
)) {
|
|
67
|
+
if (chunk.message instanceof AIMessageChunk) {
|
|
68
|
+
yielded.push(chunk.message);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return { yielded, dispatched };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const toolUseEvents = [
|
|
75
|
+
{
|
|
76
|
+
contentBlockStart: {
|
|
77
|
+
contentBlockIndex: 1,
|
|
78
|
+
start: { toolUse: { toolUseId: 'call_1', name: 'weather' } },
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
contentBlockDelta: {
|
|
83
|
+
contentBlockIndex: 1,
|
|
84
|
+
delta: { toolUse: { input: '{"city":"NYC"}' } },
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
{ contentBlockStop: { contentBlockIndex: 1 } },
|
|
88
|
+
];
|
|
89
|
+
|
|
90
|
+
test('dispatches toolUse start and stop seal chunks to callbacks', async () => {
|
|
91
|
+
const { yielded, dispatched } = await runStream(toolUseEvents);
|
|
92
|
+
|
|
93
|
+
const sealOf = (m: AIMessageChunk): unknown =>
|
|
94
|
+
(m.response_metadata as Record<string, unknown>)[
|
|
95
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY
|
|
96
|
+
];
|
|
97
|
+
|
|
98
|
+
expect(yielded.some((m) => sealOf(m) != null)).toBe(true);
|
|
99
|
+
|
|
100
|
+
expect(dispatched).toHaveLength(3);
|
|
101
|
+
expect(dispatched[0].tool_call_chunks).toMatchObject([
|
|
102
|
+
{ id: 'call_1', name: 'weather', index: 1 },
|
|
103
|
+
]);
|
|
104
|
+
expect(dispatched[1].tool_call_chunks).toMatchObject([
|
|
105
|
+
{ args: '{"city":"NYC"}', index: 1 },
|
|
106
|
+
]);
|
|
107
|
+
expect(sealOf(dispatched[2])).toEqual({ kind: 'single', index: 1 });
|
|
108
|
+
expect(
|
|
109
|
+
(dispatched[2].response_metadata as Record<string, unknown>)[
|
|
110
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY
|
|
111
|
+
]
|
|
112
|
+
).toBe(BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('does not emit seal chunks when guardrails are configured', async () => {
|
|
116
|
+
const { yielded, dispatched } = await runStream(toolUseEvents, {
|
|
117
|
+
guardrailConfig: {
|
|
118
|
+
guardrailIdentifier: 'guardrail_1',
|
|
119
|
+
guardrailVersion: '1',
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const hasSeal = (m: AIMessageChunk): boolean =>
|
|
124
|
+
(m.response_metadata as Record<string, unknown>)[
|
|
125
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY
|
|
126
|
+
] != null;
|
|
127
|
+
|
|
128
|
+
// Guardrails can reject the turn at messageStop after contentBlockStop,
|
|
129
|
+
// so no eager seal may be emitted — but tool chunks still stream.
|
|
130
|
+
expect(yielded.some(hasSeal)).toBe(false);
|
|
131
|
+
expect(dispatched.some(hasSeal)).toBe(false);
|
|
132
|
+
expect(dispatched).toHaveLength(2);
|
|
133
|
+
expect(dispatched[0].tool_call_chunks).toMatchObject([
|
|
134
|
+
{ id: 'call_1', name: 'weather', index: 1 },
|
|
135
|
+
]);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test('does not emit seal chunks for non-toolUse block stops', async () => {
|
|
139
|
+
const { yielded, dispatched } = await runStream([
|
|
140
|
+
{
|
|
141
|
+
contentBlockDelta: {
|
|
142
|
+
contentBlockIndex: 0,
|
|
143
|
+
delta: { text: 'hello' },
|
|
144
|
+
},
|
|
145
|
+
},
|
|
146
|
+
{ contentBlockStop: { contentBlockIndex: 0 } },
|
|
147
|
+
]);
|
|
148
|
+
|
|
149
|
+
const hasSeal = (m: AIMessageChunk): boolean =>
|
|
150
|
+
(m.response_metadata as Record<string, unknown>)[
|
|
151
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY
|
|
152
|
+
] != null;
|
|
153
|
+
|
|
154
|
+
expect(yielded.some(hasSeal)).toBe(false);
|
|
155
|
+
expect(dispatched.some(hasSeal)).toBe(false);
|
|
156
|
+
expect(dispatched).toHaveLength(1);
|
|
157
|
+
});
|
|
158
|
+
});
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { expect, test, describe } from '@jest/globals';
|
|
2
|
+
import { AIMessageChunk } from '@langchain/core/messages';
|
|
3
|
+
import type { ContentBlockDeltaEvent, ContentBlockStartEvent } from '../types';
|
|
4
|
+
import {
|
|
5
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY,
|
|
6
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
|
|
7
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
8
|
+
} from '@/tools/streamedToolCallSeals';
|
|
9
|
+
import {
|
|
10
|
+
createConverseToolUseStopChunk,
|
|
11
|
+
handleConverseStreamContentBlockStart,
|
|
12
|
+
handleConverseStreamContentBlockDelta,
|
|
13
|
+
} from './message_outputs';
|
|
14
|
+
|
|
15
|
+
function asAIMessageChunk(message: unknown): AIMessageChunk {
|
|
16
|
+
expect(message).toBeInstanceOf(AIMessageChunk);
|
|
17
|
+
return message as AIMessageChunk;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
describe('Converse streamed tool-call seal metadata', () => {
|
|
21
|
+
test('stamps the adapter on toolUse content block starts', () => {
|
|
22
|
+
const chunk = handleConverseStreamContentBlockStart({
|
|
23
|
+
contentBlockIndex: 1,
|
|
24
|
+
start: {
|
|
25
|
+
toolUse: { toolUseId: 'call_1', name: 'weather' },
|
|
26
|
+
},
|
|
27
|
+
} as ContentBlockStartEvent);
|
|
28
|
+
|
|
29
|
+
const message = asAIMessageChunk(chunk?.message);
|
|
30
|
+
expect(message.response_metadata).toMatchObject({
|
|
31
|
+
contentBlockIndex: 1,
|
|
32
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
33
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
34
|
+
});
|
|
35
|
+
expect(message.tool_call_chunks).toEqual([
|
|
36
|
+
{
|
|
37
|
+
id: 'call_1',
|
|
38
|
+
name: 'weather',
|
|
39
|
+
index: 1,
|
|
40
|
+
type: 'tool_call_chunk',
|
|
41
|
+
},
|
|
42
|
+
]);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test('stamps the adapter on toolUse deltas but not text deltas', () => {
|
|
46
|
+
const toolChunk = handleConverseStreamContentBlockDelta({
|
|
47
|
+
contentBlockIndex: 1,
|
|
48
|
+
delta: { toolUse: { input: '{"city":' } },
|
|
49
|
+
} as ContentBlockDeltaEvent);
|
|
50
|
+
const toolMetadata = asAIMessageChunk(toolChunk.message)
|
|
51
|
+
.response_metadata as Record<string, unknown>;
|
|
52
|
+
expect(toolMetadata[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]).toBe(
|
|
53
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
const textChunk = handleConverseStreamContentBlockDelta({
|
|
57
|
+
contentBlockIndex: 0,
|
|
58
|
+
delta: { text: 'hello' },
|
|
59
|
+
} as ContentBlockDeltaEvent);
|
|
60
|
+
const textMetadata = asAIMessageChunk(textChunk.message)
|
|
61
|
+
.response_metadata as Record<string, unknown>;
|
|
62
|
+
expect(
|
|
63
|
+
textMetadata[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]
|
|
64
|
+
).toBeUndefined();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('builds an explicit single seal chunk for a stopped toolUse block', () => {
|
|
68
|
+
const chunk = createConverseToolUseStopChunk(2);
|
|
69
|
+
|
|
70
|
+
const message = asAIMessageChunk(chunk.message);
|
|
71
|
+
expect(message.response_metadata).toEqual({
|
|
72
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
73
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
74
|
+
[STREAMED_TOOL_CALL_SEAL_METADATA_KEY]: { kind: 'single', index: 2 },
|
|
75
|
+
});
|
|
76
|
+
expect(message.tool_call_chunks).toEqual([
|
|
77
|
+
{
|
|
78
|
+
args: '',
|
|
79
|
+
index: 2,
|
|
80
|
+
type: 'tool_call_chunk',
|
|
81
|
+
},
|
|
82
|
+
]);
|
|
83
|
+
expect(message.content).toBe('');
|
|
84
|
+
});
|
|
85
|
+
});
|
|
@@ -17,6 +17,11 @@ import type {
|
|
|
17
17
|
MessageContentReasoningBlockReasoningTextPartial,
|
|
18
18
|
MessageContentReasoningBlockRedacted,
|
|
19
19
|
} from '../types';
|
|
20
|
+
import {
|
|
21
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY,
|
|
22
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
|
|
23
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
24
|
+
} from '@/tools/streamedToolCallSeals';
|
|
20
25
|
import { toLangChainContent } from '@/messages/langchain';
|
|
21
26
|
|
|
22
27
|
/**
|
|
@@ -235,6 +240,8 @@ export function handleConverseStreamContentBlockDelta(
|
|
|
235
240
|
],
|
|
236
241
|
response_metadata: {
|
|
237
242
|
contentBlockIndex: contentBlockDelta.contentBlockIndex,
|
|
243
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
244
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
238
245
|
},
|
|
239
246
|
}),
|
|
240
247
|
});
|
|
@@ -292,6 +299,8 @@ export function handleConverseStreamContentBlockStart(
|
|
|
292
299
|
],
|
|
293
300
|
response_metadata: {
|
|
294
301
|
contentBlockIndex: index,
|
|
302
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
303
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
295
304
|
},
|
|
296
305
|
}),
|
|
297
306
|
});
|
|
@@ -301,6 +310,40 @@ export function handleConverseStreamContentBlockStart(
|
|
|
301
310
|
return null;
|
|
302
311
|
}
|
|
303
312
|
|
|
313
|
+
/**
|
|
314
|
+
* Build the chunk emitted when a Converse `contentBlockStop` event closes a
|
|
315
|
+
* toolUse block. The Converse protocol guarantees a block's input is complete
|
|
316
|
+
* at `contentBlockStop`, so this chunk carries an explicit streamed tool-call
|
|
317
|
+
* seal for that block index. The empty `args` delta merges as a no-op into the
|
|
318
|
+
* accumulated tool call; id/name are omitted so the chunk matches the existing
|
|
319
|
+
* entry purely by index.
|
|
320
|
+
*/
|
|
321
|
+
export function createConverseToolUseStopChunk(
|
|
322
|
+
contentBlockIndex: number
|
|
323
|
+
): ChatGenerationChunk {
|
|
324
|
+
return new ChatGenerationChunk({
|
|
325
|
+
text: '',
|
|
326
|
+
message: new AIMessageChunk({
|
|
327
|
+
content: '',
|
|
328
|
+
tool_call_chunks: [
|
|
329
|
+
{
|
|
330
|
+
args: '',
|
|
331
|
+
index: contentBlockIndex,
|
|
332
|
+
type: 'tool_call_chunk',
|
|
333
|
+
},
|
|
334
|
+
],
|
|
335
|
+
response_metadata: {
|
|
336
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
337
|
+
BEDROCK_CONVERSE_STREAMED_TOOL_CALL_ADAPTER,
|
|
338
|
+
[STREAMED_TOOL_CALL_SEAL_METADATA_KEY]: {
|
|
339
|
+
kind: 'single',
|
|
340
|
+
index: contentBlockIndex,
|
|
341
|
+
},
|
|
342
|
+
},
|
|
343
|
+
}),
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
|
|
304
347
|
/**
|
|
305
348
|
* Handle a metadata event from Bedrock Converse stream.
|
|
306
349
|
*/
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { expect, test, describe } from '@jest/globals';
|
|
2
|
+
import { AIMessageChunk } from '@langchain/core/messages';
|
|
3
|
+
import type { EnhancedGenerateContentResponse } from '@google/generative-ai';
|
|
4
|
+
import {
|
|
5
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY,
|
|
6
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
|
|
7
|
+
GOOGLE_STREAMED_TOOL_CALL_ADAPTER,
|
|
8
|
+
} from '@/tools/streamedToolCallSeals';
|
|
9
|
+
import { convertResponseContentToChatGenerationChunk } from './common';
|
|
10
|
+
|
|
11
|
+
function buildResponse(
|
|
12
|
+
parts: Array<Record<string, unknown>>
|
|
13
|
+
): EnhancedGenerateContentResponse {
|
|
14
|
+
return {
|
|
15
|
+
candidates: [
|
|
16
|
+
{
|
|
17
|
+
content: { role: 'model', parts },
|
|
18
|
+
index: 0,
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
} as unknown as EnhancedGenerateContentResponse;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function asAIMessageChunk(message: unknown): AIMessageChunk {
|
|
25
|
+
expect(message).toBeInstanceOf(AIMessageChunk);
|
|
26
|
+
return message as AIMessageChunk;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe('convertResponseContentToChatGenerationChunk seal metadata', () => {
|
|
30
|
+
test('stamps an on-arrival seal on function call chunks', () => {
|
|
31
|
+
const chunk = convertResponseContentToChatGenerationChunk(
|
|
32
|
+
buildResponse([
|
|
33
|
+
{
|
|
34
|
+
functionCall: { name: 'weather', args: { city: 'NYC' } },
|
|
35
|
+
},
|
|
36
|
+
]),
|
|
37
|
+
{ usageMetadata: undefined, index: 0 }
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
const message = asAIMessageChunk(chunk?.message);
|
|
41
|
+
expect(message.response_metadata).toMatchObject({
|
|
42
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
43
|
+
GOOGLE_STREAMED_TOOL_CALL_ADAPTER,
|
|
44
|
+
[STREAMED_TOOL_CALL_SEAL_METADATA_KEY]: { kind: 'all' },
|
|
45
|
+
});
|
|
46
|
+
expect(message.tool_call_chunks).toHaveLength(1);
|
|
47
|
+
expect(message.tool_calls?.[0]).toMatchObject({
|
|
48
|
+
name: 'weather',
|
|
49
|
+
args: { city: 'NYC' },
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test('does not stamp seal metadata on text-only chunks', () => {
|
|
54
|
+
const chunk = convertResponseContentToChatGenerationChunk(
|
|
55
|
+
buildResponse([{ text: 'hello' }]),
|
|
56
|
+
{ usageMetadata: undefined, index: 0 }
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
const metadata = asAIMessageChunk(chunk?.message)
|
|
60
|
+
.response_metadata as Record<string, unknown>;
|
|
61
|
+
expect(metadata[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]).toBeUndefined();
|
|
62
|
+
expect(metadata[STREAMED_TOOL_CALL_SEAL_METADATA_KEY]).toBeUndefined();
|
|
63
|
+
});
|
|
64
|
+
});
|
|
@@ -35,6 +35,11 @@ import {
|
|
|
35
35
|
type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
|
|
36
36
|
} from '@google/generative-ai';
|
|
37
37
|
import type { ChatGeneration, ChatResult } from '@langchain/core/outputs';
|
|
38
|
+
import {
|
|
39
|
+
STREAMED_TOOL_CALL_SEAL_METADATA_KEY,
|
|
40
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
|
|
41
|
+
GOOGLE_STREAMED_TOOL_CALL_ADAPTER,
|
|
42
|
+
} from '@/tools/streamedToolCallSeals';
|
|
38
43
|
import {
|
|
39
44
|
jsonSchemaToGeminiParameters,
|
|
40
45
|
schemaToGenerativeAIParameters,
|
|
@@ -770,6 +775,18 @@ export function convertResponseContentToChatGenerationChunk(
|
|
|
770
775
|
response.candidates[0]?.finishReason === 'MAX_TOKENS' ||
|
|
771
776
|
response.candidates[0]?.finishReason === 'SAFETY';
|
|
772
777
|
|
|
778
|
+
// The GenAI API delivers function calls as complete objects (never partial
|
|
779
|
+
// arg deltas), so every call on this chunk is sealed on arrival for eager
|
|
780
|
+
// tool execution.
|
|
781
|
+
const response_metadata: Record<string, unknown> | undefined =
|
|
782
|
+
toolCallChunks.length > 0
|
|
783
|
+
? {
|
|
784
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
785
|
+
GOOGLE_STREAMED_TOOL_CALL_ADAPTER,
|
|
786
|
+
[STREAMED_TOOL_CALL_SEAL_METADATA_KEY]: { kind: 'all' },
|
|
787
|
+
}
|
|
788
|
+
: undefined;
|
|
789
|
+
|
|
773
790
|
return new ChatGenerationChunk({
|
|
774
791
|
text,
|
|
775
792
|
message: new AIMessageChunk({
|
|
@@ -779,6 +796,7 @@ export function convertResponseContentToChatGenerationChunk(
|
|
|
779
796
|
// Each chunk can have unique "generationInfo", and merging strategy is unclear,
|
|
780
797
|
// so leave blank for now.
|
|
781
798
|
additional_kwargs,
|
|
799
|
+
response_metadata,
|
|
782
800
|
usage_metadata: isFinalChunk ? extra.usageMetadata : undefined,
|
|
783
801
|
}),
|
|
784
802
|
generationInfo,
|
package/src/llm/openai/index.ts
CHANGED
|
@@ -34,6 +34,10 @@ import type { ChatGeneration, ChatResult } from '@langchain/core/outputs';
|
|
|
34
34
|
import type { ChatXAIInput } from '@langchain/xai';
|
|
35
35
|
import type * as t from '@langchain/openai';
|
|
36
36
|
import type { HeaderValue, HeadersLike } from './types';
|
|
37
|
+
import {
|
|
38
|
+
STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
|
|
39
|
+
OPENAI_CHAT_SEQUENTIAL_STREAMED_TOOL_CALL_ADAPTER,
|
|
40
|
+
} from '@/tools/streamedToolCallSeals';
|
|
37
41
|
import { isReasoningModel, _convertMessagesToOpenAIParams } from './utils';
|
|
38
42
|
|
|
39
43
|
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
|
|
@@ -670,6 +674,69 @@ export class CustomAzureOpenAIClient extends AzureOpenAIClient {
|
|
|
670
674
|
}
|
|
671
675
|
}
|
|
672
676
|
|
|
677
|
+
const OFFICIAL_OPENAI_BASE_URL_PATTERN = /^https:\/\/api\.openai\.com(\/|$)/;
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* Official OpenAI (api.openai.com) and Azure OpenAI Chat Completions streams
|
|
681
|
+
* emit tool-call deltas strictly sequentially by index: once a delta for a
|
|
682
|
+
* later index appears, a prior index's arguments never change. Stamping this
|
|
683
|
+
* adapter lets the stream handler seal a prior call for eager execution the
|
|
684
|
+
* moment the next call begins. OpenAI-compatible endpoints (custom baseURL)
|
|
685
|
+
* must NOT be stamped — e.g. live Kimi/Moonshot streams revise prior-index
|
|
686
|
+
* args after advancing — so callers gate on the wire endpoint, not the class.
|
|
687
|
+
*/
|
|
688
|
+
function stampSequentialStreamedToolCallAdapter(
|
|
689
|
+
message: BaseMessageChunk
|
|
690
|
+
): BaseMessageChunk {
|
|
691
|
+
if (
|
|
692
|
+
message instanceof AIMessageChunk &&
|
|
693
|
+
(message.tool_call_chunks?.length ?? 0) > 0
|
|
694
|
+
) {
|
|
695
|
+
message.response_metadata = {
|
|
696
|
+
...message.response_metadata,
|
|
697
|
+
[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:
|
|
698
|
+
OPENAI_CHAT_SEQUENTIAL_STREAMED_TOOL_CALL_ADAPTER,
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
return message;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
function isOfficialOpenAIBaseURL(baseURL: string | null | undefined): boolean {
|
|
705
|
+
// The OpenAI SDK falls back to OPENAI_BASE_URL when the client has no
|
|
706
|
+
// explicit baseURL, so an unset constructor value can still route to an
|
|
707
|
+
// OpenAI-compatible endpoint.
|
|
708
|
+
const effectiveBaseURL =
|
|
709
|
+
baseURL != null && baseURL !== '' ? baseURL : process.env.OPENAI_BASE_URL;
|
|
710
|
+
if (effectiveBaseURL == null || effectiveBaseURL === '') {
|
|
711
|
+
return true;
|
|
712
|
+
}
|
|
713
|
+
return OFFICIAL_OPENAI_BASE_URL_PATTERN.test(effectiveBaseURL);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
const AZURE_FIRST_PARTY_BASE_PATH_PATTERN =
|
|
717
|
+
/^https:\/\/[^/]+\.(openai\.azure\.com|cognitiveservices\.azure\.com|api\.cognitive\.microsoft\.com)(:\d+)?(\/|$)/;
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* Azure OpenAI is first-party when requests resolve to an instance-name
|
|
721
|
+
* endpoint or an *.openai.azure.com / *.cognitiveservices.azure.com /
|
|
722
|
+
* regional *.api.cognitive.microsoft.com base path. A custom
|
|
723
|
+
* `clientConfig.baseURL` or a non-Azure `azureOpenAIBasePath` routes through
|
|
724
|
+
* a proxy or Azure-compatible endpoint whose stream contract is unknown, so
|
|
725
|
+
* those are not stamped.
|
|
726
|
+
*/
|
|
727
|
+
function isFirstPartyAzureEndpoint(args: {
|
|
728
|
+
baseURL: string | null | undefined;
|
|
729
|
+
azureOpenAIBasePath: string | undefined;
|
|
730
|
+
}): boolean {
|
|
731
|
+
if (args.baseURL != null && args.baseURL !== '') {
|
|
732
|
+
return false;
|
|
733
|
+
}
|
|
734
|
+
if (args.azureOpenAIBasePath == null || args.azureOpenAIBasePath === '') {
|
|
735
|
+
return true;
|
|
736
|
+
}
|
|
737
|
+
return AZURE_FIRST_PARTY_BASE_PATH_PATTERN.test(args.azureOpenAIBasePath);
|
|
738
|
+
}
|
|
739
|
+
|
|
673
740
|
class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
|
|
674
741
|
private includeReasoningContent?: boolean;
|
|
675
742
|
private includeReasoningDetails?: boolean;
|
|
@@ -721,7 +788,7 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
|
|
|
721
788
|
rawResponse: OpenAIClient.Chat.Completions.ChatCompletionChunk,
|
|
722
789
|
defaultRole?: OpenAIClient.Chat.ChatCompletionRole
|
|
723
790
|
): BaseMessageChunk {
|
|
724
|
-
|
|
791
|
+
const message = attachLibreChatDeltaFields(
|
|
725
792
|
super._convertCompletionsDeltaToBaseMessageChunk(
|
|
726
793
|
delta,
|
|
727
794
|
rawResponse,
|
|
@@ -729,6 +796,10 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
|
|
|
729
796
|
),
|
|
730
797
|
delta
|
|
731
798
|
);
|
|
799
|
+
if (isOfficialOpenAIBaseURL(this.clientConfig.baseURL)) {
|
|
800
|
+
return stampSequentialStreamedToolCallAdapter(message);
|
|
801
|
+
}
|
|
802
|
+
return message;
|
|
732
803
|
}
|
|
733
804
|
|
|
734
805
|
protected _convertCompletionsMessageToBaseMessage(
|
|
@@ -1090,6 +1161,29 @@ class LibreChatAzureOpenAICompletions extends OriginalAzureChatOpenAICompletions
|
|
|
1090
1161
|
return getGatedReasoningParams(this.model, this.reasoning, options);
|
|
1091
1162
|
}
|
|
1092
1163
|
|
|
1164
|
+
protected _convertCompletionsDeltaToBaseMessageChunk(
|
|
1165
|
+
delta: Record<string, unknown>,
|
|
1166
|
+
rawResponse: OpenAIClient.Chat.Completions.ChatCompletionChunk,
|
|
1167
|
+
defaultRole?: OpenAIClient.Chat.ChatCompletionRole
|
|
1168
|
+
): BaseMessageChunk {
|
|
1169
|
+
const message = super._convertCompletionsDeltaToBaseMessageChunk(
|
|
1170
|
+
delta,
|
|
1171
|
+
rawResponse,
|
|
1172
|
+
defaultRole
|
|
1173
|
+
);
|
|
1174
|
+
if (
|
|
1175
|
+
isFirstPartyAzureEndpoint({
|
|
1176
|
+
baseURL: this.clientConfig.baseURL,
|
|
1177
|
+
azureOpenAIBasePath: this.azureOpenAIBasePath,
|
|
1178
|
+
})
|
|
1179
|
+
) {
|
|
1180
|
+
// First-party Azure OpenAI: same sequential-by-index stream contract
|
|
1181
|
+
// as api.openai.com.
|
|
1182
|
+
return stampSequentialStreamedToolCallAdapter(message);
|
|
1183
|
+
}
|
|
1184
|
+
return message;
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1093
1187
|
_getClientOptions(
|
|
1094
1188
|
options: OpenAICoreRequestOptions | undefined
|
|
1095
1189
|
): OpenAICoreRequestOptions {
|