@librechat/agents 3.3.12 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/graphs/Graph.cjs +10 -0
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/instrumentation.cjs +1 -0
- package/dist/cjs/instrumentation.cjs.map +1 -1
- package/dist/cjs/langfuseSpanRegistry.cjs +6 -3
- package/dist/cjs/langfuseSpanRegistry.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/index.cjs +35 -206
- package/dist/cjs/llm/anthropic/index.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/index.cjs +121 -241
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/llm/google/index.cjs +19 -12
- package/dist/cjs/llm/google/index.cjs.map +1 -1
- package/dist/cjs/llm/mistral/index.cjs +26 -0
- package/dist/cjs/llm/mistral/index.cjs.map +1 -0
- package/dist/cjs/llm/openai/index.cjs +82 -80
- package/dist/cjs/llm/openai/index.cjs.map +1 -1
- package/dist/cjs/llm/openrouter/index.cjs.map +1 -1
- package/dist/cjs/llm/providers.cjs +19 -19
- package/dist/cjs/llm/providers.cjs.map +1 -1
- package/dist/cjs/llm/stream/chunkAdapters.cjs +198 -0
- package/dist/cjs/llm/stream/chunkAdapters.cjs.map +1 -0
- package/dist/cjs/llm/stream/smoother.cjs +369 -0
- package/dist/cjs/llm/stream/smoother.cjs.map +1 -0
- package/dist/cjs/llm/vertexai/index.cjs +13 -1
- package/dist/cjs/llm/vertexai/index.cjs.map +1 -1
- package/dist/cjs/main.cjs +18 -10
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs +34 -2
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -1
- package/dist/cjs/utils/tokens.cjs +76 -29
- package/dist/cjs/utils/tokens.cjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +10 -0
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/instrumentation.mjs +1 -0
- package/dist/esm/instrumentation.mjs.map +1 -1
- package/dist/esm/langfuseSpanRegistry.mjs +6 -3
- package/dist/esm/langfuseSpanRegistry.mjs.map +1 -1
- package/dist/esm/llm/anthropic/index.mjs +34 -205
- package/dist/esm/llm/anthropic/index.mjs.map +1 -1
- package/dist/esm/llm/bedrock/index.mjs +120 -240
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/llm/google/index.mjs +19 -12
- package/dist/esm/llm/google/index.mjs.map +1 -1
- package/dist/esm/llm/mistral/index.mjs +26 -0
- package/dist/esm/llm/mistral/index.mjs.map +1 -0
- package/dist/esm/llm/openai/index.mjs +82 -80
- package/dist/esm/llm/openai/index.mjs.map +1 -1
- package/dist/esm/llm/openrouter/index.mjs.map +1 -1
- package/dist/esm/llm/providers.mjs +3 -3
- package/dist/esm/llm/providers.mjs.map +1 -1
- package/dist/esm/llm/stream/chunkAdapters.mjs +195 -0
- package/dist/esm/llm/stream/chunkAdapters.mjs.map +1 -0
- package/dist/esm/llm/stream/smoother.mjs +365 -0
- package/dist/esm/llm/stream/smoother.mjs.map +1 -0
- package/dist/esm/llm/vertexai/index.mjs +13 -1
- package/dist/esm/llm/vertexai/index.mjs.map +1 -1
- package/dist/esm/main.mjs +4 -2
- package/dist/esm/tools/subagent/SubagentExecutor.mjs +34 -2
- package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -1
- package/dist/esm/utils/tokens.mjs +76 -30
- package/dist/esm/utils/tokens.mjs.map +1 -1
- package/dist/types/graphs/Graph.d.ts +3 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/types/llm/google/index.d.ts +2 -0
- package/dist/types/llm/mistral/index.d.ts +11 -0
- package/dist/types/llm/openai/index.d.ts +20 -4
- package/dist/types/llm/openrouter/index.d.ts +4 -1
- package/dist/types/llm/stream/chunkAdapters.d.ts +48 -0
- package/dist/types/llm/stream/smoother.d.ts +95 -0
- package/dist/types/llm/vertexai/index.d.ts +2 -0
- package/dist/types/tools/subagent/SubagentExecutor.d.ts +3 -0
- package/dist/types/types/graph.d.ts +5 -0
- package/dist/types/types/llm.d.ts +21 -14
- package/dist/types/utils/tokens.d.ts +10 -0
- package/package.json +2 -2
- package/src/graphs/Graph.ts +11 -0
- package/src/index.ts +8 -0
- package/src/instrumentation.ts +1 -0
- package/src/langfuseSpanRegistry.ts +9 -0
- package/src/llm/anthropic/index.ts +85 -354
- package/src/llm/bedrock/index.ts +174 -390
- package/src/llm/bedrock/llm.spec.ts +2 -0
- package/src/llm/bedrock/streamSealDispatch.test.ts +76 -0
- package/src/llm/custom-chat-models.smoke.test.ts +16 -1
- package/src/llm/google/index.ts +17 -18
- package/src/llm/google/streamSmoothing.test.ts +121 -0
- package/src/llm/mistral/index.ts +33 -0
- package/src/llm/mistral/streamSmoothing.test.ts +97 -0
- package/src/llm/openai/deepseek.test.ts +56 -0
- package/src/llm/openai/index.ts +119 -126
- package/src/llm/openrouter/index.ts +4 -1
- package/src/llm/providers.ts +3 -3
- package/src/llm/stream/chunkAdapters.test.ts +202 -0
- package/src/llm/stream/chunkAdapters.ts +317 -0
- package/src/llm/stream/reassembly.test.ts +241 -0
- package/src/llm/stream/smoother.bench.test.ts +155 -0
- package/src/llm/stream/smoother.test.ts +519 -0
- package/src/llm/stream/smoother.ts +574 -0
- package/src/llm/vertexai/index.ts +17 -1
- package/src/llm/vertexai/streamSmoothing.test.ts +109 -0
- package/src/specs/langfuse-instrumentation.test.ts +26 -0
- package/src/specs/langfuse-span-registry.test.ts +17 -0
- package/src/specs/subagent.test.ts +44 -0
- package/src/specs/summarization.test.ts +5 -2
- package/src/specs/tokens.test.ts +159 -14
- package/src/tools/__tests__/SubagentExecutor.test.ts +48 -1
- package/src/tools/subagent/SubagentExecutor.ts +69 -5
- package/src/types/graph.ts +5 -0
- package/src/types/llm.ts +53 -36
- package/src/utils/tokens.ts +115 -30
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import { AIMessageChunk } from '@langchain/core/messages';
|
|
2
|
+
import { ChatGenerationChunk } from '@langchain/core/outputs';
|
|
3
|
+
import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
|
|
4
|
+
import type { SmoothItem, SmoothPiece } from '@/llm/stream/smoother';
|
|
5
|
+
import { smoothStream } from '@/llm/stream/smoother';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Rebuilds a generation chunk carrying one piece of a split plain-text chunk.
|
|
9
|
+
* Unsplit pieces return the original chunk untouched, so disabled smoothing is
|
|
10
|
+
* byte-identical to no smoothing. Usage metadata, additional kwargs and
|
|
11
|
+
* response metadata survive only on the first piece: the aggregator merges
|
|
12
|
+
* dicts by concatenating string fields, so replicating them across pieces
|
|
13
|
+
* would duplicate reasoning text and scalar metadata once per piece.
|
|
14
|
+
*/
|
|
15
|
+
export function cloneGenerationChunkPiece(
|
|
16
|
+
chunk: ChatGenerationChunk,
|
|
17
|
+
piece: SmoothPiece
|
|
18
|
+
): ChatGenerationChunk {
|
|
19
|
+
if (piece.isFirst && piece.isLast) {
|
|
20
|
+
return chunk;
|
|
21
|
+
}
|
|
22
|
+
const message = chunk.message as AIMessageChunk;
|
|
23
|
+
return new ChatGenerationChunk({
|
|
24
|
+
text: piece.text,
|
|
25
|
+
generationInfo: piece.isFirst ? chunk.generationInfo : undefined,
|
|
26
|
+
message: new AIMessageChunk(
|
|
27
|
+
Object.assign({}, message, {
|
|
28
|
+
content: piece.text,
|
|
29
|
+
usage_metadata: piece.isFirst ? message.usage_metadata : undefined,
|
|
30
|
+
additional_kwargs: piece.isFirst ? message.additional_kwargs : {},
|
|
31
|
+
response_metadata: piece.isFirst ? message.response_metadata : {},
|
|
32
|
+
})
|
|
33
|
+
),
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const SPLITTABLE_TEXT_PART_KEYS: ReadonlySet<string> = new Set([
|
|
38
|
+
'type',
|
|
39
|
+
'text',
|
|
40
|
+
'index',
|
|
41
|
+
]);
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Returns the sole plain text part of a single-element complex content array
|
|
45
|
+
* (the shape google-common emits for text deltas), or null when the part
|
|
46
|
+
* carries anything beyond `type`/`text`/`index` (thought signatures, media,
|
|
47
|
+
* function calls) and must not be sliced. The part must carry a numeric
|
|
48
|
+
* `index`: it is the aggregator's list-merge key, and without it each split
|
|
49
|
+
* piece would append as a separate content part instead of merging back
|
|
50
|
+
* into one — index-less parts pace whole instead.
|
|
51
|
+
*/
|
|
52
|
+
function getSplittableTextPart(
|
|
53
|
+
message: AIMessageChunk,
|
|
54
|
+
chunkText: string
|
|
55
|
+
): Record<string, unknown> | null {
|
|
56
|
+
const content = message.content;
|
|
57
|
+
if (!Array.isArray(content) || content.length !== 1) {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
const part: unknown = content[0];
|
|
61
|
+
if (part == null || typeof part !== 'object') {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const record = part as Record<string, unknown>;
|
|
65
|
+
if (typeof record.text !== 'string' || record.text !== chunkText) {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
if (record.type != null && record.type !== 'text') {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
if (typeof record.index !== 'number') {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
if (!Object.keys(record).every((key) => SPLITTABLE_TEXT_PART_KEYS.has(key))) {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
return record;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function clonePartGenerationChunkPiece(
|
|
81
|
+
chunk: ChatGenerationChunk,
|
|
82
|
+
part: Record<string, unknown>,
|
|
83
|
+
piece: SmoothPiece
|
|
84
|
+
): ChatGenerationChunk {
|
|
85
|
+
if (piece.isFirst && piece.isLast) {
|
|
86
|
+
return chunk;
|
|
87
|
+
}
|
|
88
|
+
const message = chunk.message as AIMessageChunk;
|
|
89
|
+
return new ChatGenerationChunk({
|
|
90
|
+
text: piece.text,
|
|
91
|
+
generationInfo: piece.isFirst ? chunk.generationInfo : undefined,
|
|
92
|
+
message: new AIMessageChunk(
|
|
93
|
+
Object.assign({}, message, {
|
|
94
|
+
content: [Object.assign({}, part, { text: piece.text })],
|
|
95
|
+
usage_metadata: piece.isFirst ? message.usage_metadata : undefined,
|
|
96
|
+
additional_kwargs: piece.isFirst ? message.additional_kwargs : {},
|
|
97
|
+
response_metadata: piece.isFirst ? message.response_metadata : {},
|
|
98
|
+
})
|
|
99
|
+
),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Provider-agnostic classification of a `ChatGenerationChunk` for the
|
|
105
|
+
* smoothing engine:
|
|
106
|
+
* - splittable: plain visible text — string content equal to `chunk.text`, or
|
|
107
|
+
* a single plain text part (google-common's delta shape) — with no
|
|
108
|
+
* logprobs / finish_reason; sliced adaptively at the pacing cadence.
|
|
109
|
+
* - atomic: any other text-bearing chunk (complex content arrays, logprobs,
|
|
110
|
+
* finish_reason, provider-specific reasoning payloads surfaced via
|
|
111
|
+
* `getAtomicText`) — paced as one piece, never split.
|
|
112
|
+
* - passthrough: tool-call deltas, usage-only and metadata chunks — strict
|
|
113
|
+
* FIFO, zero delay.
|
|
114
|
+
*/
|
|
115
|
+
/**
|
|
116
|
+
* google-common stamps `logprobs: { content: [] }` on every chunk, so only
|
|
117
|
+
* logprobs that actually carry data may block splitting here. This is looser
|
|
118
|
+
* than the OpenAI-family adapter in `llm/openai/index.ts`, which treats ANY
|
|
119
|
+
* logprobs as atomic (its providers only attach logprobs when requested, and
|
|
120
|
+
* the DeepSeek suite pins that contract) — do not unify the two predicates.
|
|
121
|
+
*/
|
|
122
|
+
function hasMeaningfulLogprobs(
|
|
123
|
+
generationInfo: ChatGenerationChunk['generationInfo']
|
|
124
|
+
): boolean {
|
|
125
|
+
const logprobs = generationInfo?.logprobs;
|
|
126
|
+
if (logprobs == null) {
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
const content = (logprobs as { content?: unknown }).content;
|
|
130
|
+
if (Array.isArray(content)) {
|
|
131
|
+
return content.length > 0;
|
|
132
|
+
}
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** google-common reports the terminal reason camelCase (`finishReason`). */
|
|
137
|
+
function hasFinishReason(
|
|
138
|
+
generationInfo: ChatGenerationChunk['generationInfo']
|
|
139
|
+
): boolean {
|
|
140
|
+
return (
|
|
141
|
+
generationInfo?.finish_reason != null || generationInfo?.finishReason != null
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Chunks that pair visible text with reasoning payloads in
|
|
147
|
+
* `additional_kwargs` (Gemini thought summaries, reasoning_content deltas,
|
|
148
|
+
* OpenRouter reasoning_details) must pace whole: split pieces would each
|
|
149
|
+
* carry the same kwargs and downstream merging — the aggregator's dict merge
|
|
150
|
+
* or OpenRouter's reasoning_details accumulation — duplicates them once per
|
|
151
|
+
* piece.
|
|
152
|
+
*/
|
|
153
|
+
/**
|
|
154
|
+
* Mixed text/tool-call deltas must never split: cloned pieces would each
|
|
155
|
+
* carry the same tool_call_chunks and downstream accumulation would corrupt
|
|
156
|
+
* the assembled tool arguments.
|
|
157
|
+
*/
|
|
158
|
+
export function hasToolCallChunks(message: AIMessageChunk): boolean {
|
|
159
|
+
return (message.tool_call_chunks?.length ?? 0) > 0;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function hasReasoningKwargs(message: AIMessageChunk): boolean {
|
|
163
|
+
const kwargs = message.additional_kwargs;
|
|
164
|
+
if (
|
|
165
|
+
typeof kwargs.reasoning_content === 'string' &&
|
|
166
|
+
kwargs.reasoning_content !== ''
|
|
167
|
+
) {
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
if (
|
|
171
|
+
Array.isArray(kwargs.reasoning_details) &&
|
|
172
|
+
kwargs.reasoning_details.length > 0
|
|
173
|
+
) {
|
|
174
|
+
return true;
|
|
175
|
+
}
|
|
176
|
+
return kwargs.reasoning != null;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Extracts the visible reasoning text a kwargs-borne delta contributes, so
|
|
181
|
+
* reasoning-only chunks (Gemini thoughts, DeepSeek reasoning_content,
|
|
182
|
+
* OpenAI reasoning summaries, OpenRouter reasoning_details) pace atomically
|
|
183
|
+
* at the cadence instead of passing through unsmoothed.
|
|
184
|
+
*/
|
|
185
|
+
export function getReasoningKwargsText(message: AIMessageChunk): string {
|
|
186
|
+
const kwargs = message.additional_kwargs;
|
|
187
|
+
if (
|
|
188
|
+
typeof kwargs.reasoning_content === 'string' &&
|
|
189
|
+
kwargs.reasoning_content !== ''
|
|
190
|
+
) {
|
|
191
|
+
return kwargs.reasoning_content;
|
|
192
|
+
}
|
|
193
|
+
const reasoning = kwargs.reasoning;
|
|
194
|
+
if (typeof reasoning === 'string' && reasoning !== '') {
|
|
195
|
+
return reasoning;
|
|
196
|
+
}
|
|
197
|
+
if (reasoning != null && typeof reasoning === 'object') {
|
|
198
|
+
const summaryText = (
|
|
199
|
+
reasoning as { summary?: { text?: unknown }[] }
|
|
200
|
+
).summary?.[0]?.text;
|
|
201
|
+
if (typeof summaryText === 'string' && summaryText !== '') {
|
|
202
|
+
return summaryText;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
const details = kwargs.reasoning_details;
|
|
206
|
+
if (Array.isArray(details)) {
|
|
207
|
+
let text = '';
|
|
208
|
+
for (const detail of details) {
|
|
209
|
+
if (detail != null && typeof detail === 'object') {
|
|
210
|
+
const detailText = (detail as { text?: unknown }).text;
|
|
211
|
+
if (typeof detailText === 'string') {
|
|
212
|
+
text += detailText;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
if (text !== '') {
|
|
217
|
+
return text;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return '';
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function toGenerationSmoothItem(
|
|
224
|
+
chunk: ChatGenerationChunk,
|
|
225
|
+
getAtomicText?: (message: AIMessageChunk) => string
|
|
226
|
+
): SmoothItem<ChatGenerationChunk> {
|
|
227
|
+
const { message } = chunk;
|
|
228
|
+
const isMessageChunk = message instanceof AIMessageChunk;
|
|
229
|
+
const cleanGenerationInfo =
|
|
230
|
+
!hasMeaningfulLogprobs(chunk.generationInfo) &&
|
|
231
|
+
!hasFinishReason(chunk.generationInfo);
|
|
232
|
+
const splittable =
|
|
233
|
+
Boolean(chunk.text) &&
|
|
234
|
+
isMessageChunk &&
|
|
235
|
+
typeof message.content === 'string' &&
|
|
236
|
+
message.content === chunk.text &&
|
|
237
|
+
cleanGenerationInfo &&
|
|
238
|
+
!hasReasoningKwargs(message) &&
|
|
239
|
+
!hasToolCallChunks(message);
|
|
240
|
+
|
|
241
|
+
if (splittable) {
|
|
242
|
+
return {
|
|
243
|
+
text: chunk.text,
|
|
244
|
+
smooth: true,
|
|
245
|
+
emit: (piece) => cloneGenerationChunkPiece(chunk, piece),
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const splittablePart =
|
|
250
|
+
Boolean(chunk.text) &&
|
|
251
|
+
isMessageChunk &&
|
|
252
|
+
cleanGenerationInfo &&
|
|
253
|
+
!hasReasoningKwargs(message) &&
|
|
254
|
+
!hasToolCallChunks(message)
|
|
255
|
+
? getSplittableTextPart(message, chunk.text)
|
|
256
|
+
: null;
|
|
257
|
+
if (splittablePart != null) {
|
|
258
|
+
return {
|
|
259
|
+
text: chunk.text,
|
|
260
|
+
smooth: true,
|
|
261
|
+
emit: (piece) =>
|
|
262
|
+
clonePartGenerationChunkPiece(chunk, splittablePart, piece),
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const pacedText =
|
|
267
|
+
chunk.text ||
|
|
268
|
+
(isMessageChunk && getAtomicText != null ? getAtomicText(message) : '');
|
|
269
|
+
if (pacedText !== '') {
|
|
270
|
+
return {
|
|
271
|
+
text: pacedText,
|
|
272
|
+
smooth: true,
|
|
273
|
+
atomic: true,
|
|
274
|
+
emit: () => chunk,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
return { text: '', smooth: false, emit: () => chunk };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Wraps a provider's raw chunk stream with adaptive smoothing and per-piece
|
|
283
|
+
* `handleLLMNewToken` dispatch. The raw stream must NOT dispatch runManager
|
|
284
|
+
* callbacks itself — callback-echo consumers would otherwise observe the
|
|
285
|
+
* unsmoothed deltas.
|
|
286
|
+
*/
|
|
287
|
+
export async function* smoothGenerationChunks({
|
|
288
|
+
chunks,
|
|
289
|
+
delayMs,
|
|
290
|
+
signal,
|
|
291
|
+
runManager,
|
|
292
|
+
}: {
|
|
293
|
+
chunks: AsyncGenerator<ChatGenerationChunk>;
|
|
294
|
+
delayMs: number;
|
|
295
|
+
signal?: AbortSignal;
|
|
296
|
+
runManager?: CallbackManagerForLLMRun;
|
|
297
|
+
}): AsyncGenerator<ChatGenerationChunk> {
|
|
298
|
+
const source = (async function* (): AsyncGenerator<
|
|
299
|
+
SmoothItem<ChatGenerationChunk>
|
|
300
|
+
> {
|
|
301
|
+
for await (const chunk of chunks) {
|
|
302
|
+
yield toGenerationSmoothItem(chunk, getReasoningKwargsText);
|
|
303
|
+
}
|
|
304
|
+
})();
|
|
305
|
+
|
|
306
|
+
for await (const outputChunk of smoothStream({ source, delayMs, signal })) {
|
|
307
|
+
yield outputChunk;
|
|
308
|
+
await runManager?.handleLLMNewToken(
|
|
309
|
+
outputChunk.text || '',
|
|
310
|
+
undefined,
|
|
311
|
+
undefined,
|
|
312
|
+
undefined,
|
|
313
|
+
undefined,
|
|
314
|
+
{ chunk: outputChunk }
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { concat } from '@langchain/core/utils/stream';
|
|
2
|
+
import { AIMessageChunk } from '@langchain/core/messages';
|
|
3
|
+
import { ChatGenerationChunk } from '@langchain/core/outputs';
|
|
4
|
+
import type { SmoothItem } from './smoother';
|
|
5
|
+
import { toSmoothItem } from '@/llm/openai';
|
|
6
|
+
import {
|
|
7
|
+
toGenerationSmoothItem,
|
|
8
|
+
getReasoningKwargsText,
|
|
9
|
+
} from './chunkAdapters';
|
|
10
|
+
import { findStreamChunkBoundary } from './smoother';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Lossless-reassembly property: for ANY chunk shape an adapter classifies,
|
|
14
|
+
* aggregating the emitted pieces must reproduce the original chunk's
|
|
15
|
+
* observable payload — text, content, kwargs, response metadata, usage,
|
|
16
|
+
* tool calls. This is the invariant behind every split-corruption bug class
|
|
17
|
+
* (duplicated reasoning/kwargs, summed usage, cloned tool args): if a field
|
|
18
|
+
* cannot survive slicing, the adapter must classify the chunk atomic or
|
|
19
|
+
* scope the field to a single piece, and this property proves it did.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
type AdapterName = 'openai-family' | 'generic';
|
|
23
|
+
|
|
24
|
+
const ADAPTERS: Record<
|
|
25
|
+
AdapterName,
|
|
26
|
+
(chunk: ChatGenerationChunk) => SmoothItem<ChatGenerationChunk>
|
|
27
|
+
> = {
|
|
28
|
+
'openai-family': (chunk) => toSmoothItem(chunk),
|
|
29
|
+
generic: (chunk) => toGenerationSmoothItem(chunk, getReasoningKwargsText),
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** Emits the item the way the engine would: split smooth+splittable items
|
|
33
|
+
* into ≥2 boundary-aligned pieces; atomic and passthrough emit whole. */
|
|
34
|
+
function emitPieces(item: SmoothItem<ChatGenerationChunk>): ChatGenerationChunk[] {
|
|
35
|
+
if (!item.smooth || item.atomic === true || item.text.length < 8) {
|
|
36
|
+
return [item.emit({ text: item.text, isFirst: true, isLast: true })];
|
|
37
|
+
}
|
|
38
|
+
const pieces: ChatGenerationChunk[] = [];
|
|
39
|
+
let offset = 0;
|
|
40
|
+
while (offset < item.text.length) {
|
|
41
|
+
const end = offset + findStreamChunkBoundary(item.text.slice(offset), 4);
|
|
42
|
+
pieces.push(
|
|
43
|
+
item.emit({
|
|
44
|
+
text: item.text.slice(offset, end),
|
|
45
|
+
isFirst: offset === 0,
|
|
46
|
+
isLast: end === item.text.length,
|
|
47
|
+
})
|
|
48
|
+
);
|
|
49
|
+
offset = end;
|
|
50
|
+
}
|
|
51
|
+
return pieces;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function aggregate(pieces: ChatGenerationChunk[]): AIMessageChunk {
|
|
55
|
+
let merged = pieces[0].message as AIMessageChunk;
|
|
56
|
+
for (const piece of pieces.slice(1)) {
|
|
57
|
+
merged = concat(merged, piece.message as AIMessageChunk);
|
|
58
|
+
}
|
|
59
|
+
return merged;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function assertLossless(
|
|
63
|
+
adapter: AdapterName,
|
|
64
|
+
chunk: ChatGenerationChunk
|
|
65
|
+
): void {
|
|
66
|
+
const item = ADAPTERS[adapter](chunk);
|
|
67
|
+
const pieces = emitPieces(item);
|
|
68
|
+
const original = chunk.message as AIMessageChunk;
|
|
69
|
+
|
|
70
|
+
const joinedText = pieces.map((p) => p.text).join('');
|
|
71
|
+
if (item.smooth) {
|
|
72
|
+
expect(joinedText === chunk.text || joinedText === item.text).toBe(true);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const merged = aggregate(pieces);
|
|
76
|
+
expect(merged.content).toEqual(original.content);
|
|
77
|
+
expect(merged.additional_kwargs).toEqual(original.additional_kwargs);
|
|
78
|
+
expect(merged.response_metadata).toEqual(original.response_metadata);
|
|
79
|
+
/** concat() adds empty *_token_details objects during merge; token COUNTS
|
|
80
|
+
* are the corruption signal (replicated usage would sum). */
|
|
81
|
+
expect(merged.usage_metadata?.input_tokens).toBe(
|
|
82
|
+
original.usage_metadata?.input_tokens
|
|
83
|
+
);
|
|
84
|
+
expect(merged.usage_metadata?.output_tokens).toBe(
|
|
85
|
+
original.usage_metadata?.output_tokens
|
|
86
|
+
);
|
|
87
|
+
expect(merged.usage_metadata?.total_tokens).toBe(
|
|
88
|
+
original.usage_metadata?.total_tokens
|
|
89
|
+
);
|
|
90
|
+
expect(merged.tool_call_chunks ?? []).toEqual(
|
|
91
|
+
original.tool_call_chunks ?? []
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
type MatrixCase = {
|
|
96
|
+
name: string;
|
|
97
|
+
chunk: () => ChatGenerationChunk;
|
|
98
|
+
adapters: AdapterName[];
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const TEXT = 'the quick brown fox jumps over the lazy dog again and again ';
|
|
102
|
+
|
|
103
|
+
const MATRIX: MatrixCase[] = [
|
|
104
|
+
{
|
|
105
|
+
name: 'plain string text',
|
|
106
|
+
adapters: ['openai-family', 'generic'],
|
|
107
|
+
chunk: () =>
|
|
108
|
+
new ChatGenerationChunk({
|
|
109
|
+
text: TEXT,
|
|
110
|
+
message: new AIMessageChunk({ content: TEXT }),
|
|
111
|
+
}),
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
name: 'text with usage metadata',
|
|
115
|
+
adapters: ['openai-family', 'generic'],
|
|
116
|
+
chunk: () =>
|
|
117
|
+
new ChatGenerationChunk({
|
|
118
|
+
text: TEXT,
|
|
119
|
+
message: new AIMessageChunk({
|
|
120
|
+
content: TEXT,
|
|
121
|
+
usage_metadata: { input_tokens: 7, output_tokens: 11, total_tokens: 18 },
|
|
122
|
+
}),
|
|
123
|
+
}),
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
name: 'text with an unknown string kwargs field',
|
|
127
|
+
adapters: ['openai-family', 'generic'],
|
|
128
|
+
chunk: () =>
|
|
129
|
+
new ChatGenerationChunk({
|
|
130
|
+
text: TEXT,
|
|
131
|
+
message: new AIMessageChunk({
|
|
132
|
+
content: TEXT,
|
|
133
|
+
additional_kwargs: { annotation: 'gateway-added-value' },
|
|
134
|
+
}),
|
|
135
|
+
}),
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
name: 'text with scalar response metadata',
|
|
139
|
+
adapters: ['openai-family', 'generic'],
|
|
140
|
+
chunk: () =>
|
|
141
|
+
new ChatGenerationChunk({
|
|
142
|
+
text: TEXT,
|
|
143
|
+
message: new AIMessageChunk({
|
|
144
|
+
content: TEXT,
|
|
145
|
+
response_metadata: { model_name: 'test-model', system_fingerprint: 'fp_1' },
|
|
146
|
+
}),
|
|
147
|
+
}),
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
name: 'text with reasoning_content kwargs',
|
|
151
|
+
adapters: ['openai-family', 'generic'],
|
|
152
|
+
chunk: () =>
|
|
153
|
+
new ChatGenerationChunk({
|
|
154
|
+
text: TEXT,
|
|
155
|
+
message: new AIMessageChunk({
|
|
156
|
+
content: TEXT,
|
|
157
|
+
additional_kwargs: { reasoning_content: 'a hidden thought' },
|
|
158
|
+
}),
|
|
159
|
+
}),
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
name: 'text with reasoning_details kwargs',
|
|
163
|
+
adapters: ['openai-family', 'generic'],
|
|
164
|
+
chunk: () =>
|
|
165
|
+
new ChatGenerationChunk({
|
|
166
|
+
text: TEXT,
|
|
167
|
+
message: new AIMessageChunk({
|
|
168
|
+
content: TEXT,
|
|
169
|
+
additional_kwargs: {
|
|
170
|
+
reasoning_details: [{ type: 'reasoning.text', text: 'thought' }],
|
|
171
|
+
},
|
|
172
|
+
}),
|
|
173
|
+
}),
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
name: 'text alongside tool_call_chunks',
|
|
177
|
+
adapters: ['openai-family', 'generic'],
|
|
178
|
+
chunk: () =>
|
|
179
|
+
new ChatGenerationChunk({
|
|
180
|
+
text: TEXT,
|
|
181
|
+
message: new AIMessageChunk({
|
|
182
|
+
content: TEXT,
|
|
183
|
+
tool_call_chunks: [
|
|
184
|
+
{ name: 'lookup', args: '{"q":1}', id: 'call_1', index: 0, type: 'tool_call_chunk' },
|
|
185
|
+
],
|
|
186
|
+
}),
|
|
187
|
+
}),
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
name: 'indexed text-part array content (google shape)',
|
|
191
|
+
adapters: ['generic'],
|
|
192
|
+
chunk: () =>
|
|
193
|
+
new ChatGenerationChunk({
|
|
194
|
+
text: TEXT,
|
|
195
|
+
message: new AIMessageChunk({
|
|
196
|
+
content: [{ type: 'text', text: TEXT, index: 0 }],
|
|
197
|
+
}),
|
|
198
|
+
}),
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
name: 'index-less text-part array content (no merge key)',
|
|
202
|
+
adapters: ['generic'],
|
|
203
|
+
chunk: () =>
|
|
204
|
+
new ChatGenerationChunk({
|
|
205
|
+
text: TEXT,
|
|
206
|
+
message: new AIMessageChunk({
|
|
207
|
+
content: [{ type: 'text', text: TEXT }],
|
|
208
|
+
}),
|
|
209
|
+
}),
|
|
210
|
+
},
|
|
211
|
+
{
|
|
212
|
+
name: 'reasoning-only delta',
|
|
213
|
+
adapters: ['openai-family', 'generic'],
|
|
214
|
+
chunk: () =>
|
|
215
|
+
new ChatGenerationChunk({
|
|
216
|
+
text: '',
|
|
217
|
+
message: new AIMessageChunk({
|
|
218
|
+
content: '',
|
|
219
|
+
additional_kwargs: { reasoning_content: 'thought only delta here' },
|
|
220
|
+
}),
|
|
221
|
+
}),
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
name: 'usage-only delta',
|
|
225
|
+
adapters: ['openai-family', 'generic'],
|
|
226
|
+
chunk: () =>
|
|
227
|
+
new ChatGenerationChunk({
|
|
228
|
+
text: '',
|
|
229
|
+
message: new AIMessageChunk({
|
|
230
|
+
content: '',
|
|
231
|
+
usage_metadata: { input_tokens: 1, output_tokens: 2, total_tokens: 3 },
|
|
232
|
+
}),
|
|
233
|
+
}),
|
|
234
|
+
},
|
|
235
|
+
];
|
|
236
|
+
|
|
237
|
+
describe.each(MATRIX)('lossless reassembly: $name', ({ chunk, adapters }) => {
|
|
238
|
+
it.each(adapters)('%s adapter', (adapter) => {
|
|
239
|
+
assertLossless(adapter, chunk());
|
|
240
|
+
});
|
|
241
|
+
});
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import { HumanMessage } from '@langchain/core/messages';
|
|
3
|
+
import type { AddressInfo } from 'node:net';
|
|
4
|
+
import { ChatOpenAI } from '@/llm/openai';
|
|
5
|
+
import { SMOOTH_TARGET_LATENCY_MS } from './smoother';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* End-to-end cadence/lag benchmark: drives the real ChatOpenAI client against
|
|
9
|
+
* a local SSE server that emits large chunks fast (the big-chunk gateway
|
|
10
|
+
* profile smoothing exists for), asserting the properties that define the
|
|
11
|
+
* feature: even cadence at the configured tick, and render lag pinned near
|
|
12
|
+
* the target latency instead of growing with reply length.
|
|
13
|
+
*/
|
|
14
|
+
describe('adaptive smoothing benchmark (big-chunk provider)', () => {
|
|
15
|
+
const WORDS =
|
|
16
|
+
'ClickHouse stores data in columns so queries read only what they touch and compress well '.split(
|
|
17
|
+
' '
|
|
18
|
+
);
|
|
19
|
+
const CHUNK_CHARS = 110;
|
|
20
|
+
const PROVIDER_INTERVAL_MS = 20;
|
|
21
|
+
|
|
22
|
+
function buildChunks(totalWords: number): string[] {
|
|
23
|
+
const out: string[] = [];
|
|
24
|
+
let current = '';
|
|
25
|
+
for (let i = 0; i < totalWords; i++) {
|
|
26
|
+
current += WORDS[i % WORDS.length] + ' ';
|
|
27
|
+
if (current.length >= CHUNK_CHARS) {
|
|
28
|
+
out.push(current);
|
|
29
|
+
current = '';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (current) {
|
|
33
|
+
out.push(current);
|
|
34
|
+
}
|
|
35
|
+
return out;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function startServer(chunks: string[], stamp: { doneAt: number }): Promise<{
|
|
39
|
+
server: http.Server;
|
|
40
|
+
port: number;
|
|
41
|
+
}> {
|
|
42
|
+
return new Promise((resolve) => {
|
|
43
|
+
const server = http.createServer((req, res) => {
|
|
44
|
+
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
|
45
|
+
const frame = (
|
|
46
|
+
delta: Record<string, unknown>,
|
|
47
|
+
finish: string | null
|
|
48
|
+
): string =>
|
|
49
|
+
`data: ${JSON.stringify({
|
|
50
|
+
id: 'bench',
|
|
51
|
+
object: 'chat.completion.chunk',
|
|
52
|
+
created: 1,
|
|
53
|
+
model: 'bench-model',
|
|
54
|
+
choices: [{ index: 0, delta, finish_reason: finish }],
|
|
55
|
+
})}\n\n`;
|
|
56
|
+
res.write(frame({ role: 'assistant', content: '' }, null));
|
|
57
|
+
let i = 0;
|
|
58
|
+
const send = (): void => {
|
|
59
|
+
if (i < chunks.length) {
|
|
60
|
+
res.write(frame({ content: chunks[i] }, null));
|
|
61
|
+
i += 1;
|
|
62
|
+
setTimeout(send, PROVIDER_INTERVAL_MS);
|
|
63
|
+
} else {
|
|
64
|
+
stamp.doneAt = Date.now();
|
|
65
|
+
res.write(frame({}, 'stop'));
|
|
66
|
+
res.write('data: [DONE]\n\n');
|
|
67
|
+
res.end();
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
send();
|
|
71
|
+
});
|
|
72
|
+
server.listen(0, '127.0.0.1', () => {
|
|
73
|
+
resolve({ server, port: (server.address() as AddressInfo).port });
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async function measure(
|
|
79
|
+
totalWords: number,
|
|
80
|
+
tick: number
|
|
81
|
+
): Promise<{
|
|
82
|
+
meanGap: number;
|
|
83
|
+
jitter: number;
|
|
84
|
+
lag: number;
|
|
85
|
+
totalText: string;
|
|
86
|
+
expected: string;
|
|
87
|
+
}> {
|
|
88
|
+
const chunks = buildChunks(totalWords);
|
|
89
|
+
const stamp = { doneAt: 0 };
|
|
90
|
+
const { server, port } = await startServer(chunks, stamp);
|
|
91
|
+
try {
|
|
92
|
+
const model = new ChatOpenAI({
|
|
93
|
+
model: 'bench-model',
|
|
94
|
+
apiKey: 'bench',
|
|
95
|
+
streaming: true,
|
|
96
|
+
configuration: { baseURL: `http://127.0.0.1:${port}/v1` },
|
|
97
|
+
_lc_stream_delay: tick,
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
const gaps: number[] = [];
|
|
101
|
+
let totalText = '';
|
|
102
|
+
let last = Date.now();
|
|
103
|
+
for await (const piece of await model.stream([new HumanMessage('go')])) {
|
|
104
|
+
if (typeof piece.content !== 'string' || piece.content === '') {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
const now = Date.now();
|
|
108
|
+
gaps.push(now - last);
|
|
109
|
+
last = now;
|
|
110
|
+
totalText += piece.content;
|
|
111
|
+
}
|
|
112
|
+
const renderDoneAt = Date.now();
|
|
113
|
+
|
|
114
|
+
const paint = gaps.slice(1);
|
|
115
|
+
const mean = paint.reduce((a, b) => a + b, 0) / paint.length;
|
|
116
|
+
const jitter = Math.sqrt(
|
|
117
|
+
paint.reduce((a, b) => a + (b - mean) ** 2, 0) / paint.length
|
|
118
|
+
);
|
|
119
|
+
return {
|
|
120
|
+
meanGap: mean,
|
|
121
|
+
jitter,
|
|
122
|
+
lag: renderDoneAt - stamp.doneAt,
|
|
123
|
+
totalText,
|
|
124
|
+
expected: chunks.join(''),
|
|
125
|
+
};
|
|
126
|
+
} finally {
|
|
127
|
+
server.close();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Bounds are deliberately loose on absolutes (CI machines and parallel
|
|
132
|
+
* jest workers add scheduling noise); the discriminating assertion is the
|
|
133
|
+
* last one — lag must stay FLAT as reply length grows, which is the
|
|
134
|
+
* property fixed-rate smoothing lacks (it lagged +31s at 4x length). */
|
|
135
|
+
test('holds cadence at the tick with bounded lag on a medium reply', async () => {
|
|
136
|
+
const tick = 15;
|
|
137
|
+
const result = await measure(500, tick);
|
|
138
|
+
|
|
139
|
+
expect(result.totalText).toBe(result.expected);
|
|
140
|
+
expect(result.meanGap).toBeGreaterThan(tick * 0.5);
|
|
141
|
+
expect(result.meanGap).toBeLessThan(tick * 2.5);
|
|
142
|
+
expect(result.jitter).toBeLessThan(25);
|
|
143
|
+
expect(result.lag).toBeLessThan(SMOOTH_TARGET_LATENCY_MS * 4);
|
|
144
|
+
}, 60000);
|
|
145
|
+
|
|
146
|
+
test('lag does not grow with reply length', async () => {
|
|
147
|
+
const tick = 15;
|
|
148
|
+
const short = await measure(250, tick);
|
|
149
|
+
const long = await measure(1000, tick);
|
|
150
|
+
|
|
151
|
+
expect(long.totalText).toBe(long.expected);
|
|
152
|
+
expect(long.lag).toBeLessThan(SMOOTH_TARGET_LATENCY_MS * 4);
|
|
153
|
+
expect(long.lag).toBeLessThan(short.lag + SMOOTH_TARGET_LATENCY_MS * 2);
|
|
154
|
+
}, 60000);
|
|
155
|
+
});
|