@librechat/agents 3.3.13 → 3.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/dist/cjs/graphs/Graph.cjs +121 -27
  2. package/dist/cjs/graphs/Graph.cjs.map +1 -1
  3. package/dist/cjs/hooks/HookRegistry.cjs +83 -0
  4. package/dist/cjs/hooks/HookRegistry.cjs.map +1 -1
  5. package/dist/cjs/hooks/executeHooks.cjs +60 -14
  6. package/dist/cjs/hooks/executeHooks.cjs.map +1 -1
  7. package/dist/cjs/hooks/index.cjs.map +1 -1
  8. package/dist/cjs/hooks/types.cjs +2 -0
  9. package/dist/cjs/hooks/types.cjs.map +1 -1
  10. package/dist/cjs/llm/anthropic/index.cjs +35 -206
  11. package/dist/cjs/llm/anthropic/index.cjs.map +1 -1
  12. package/dist/cjs/llm/bedrock/index.cjs +121 -241
  13. package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
  14. package/dist/cjs/llm/google/index.cjs +19 -12
  15. package/dist/cjs/llm/google/index.cjs.map +1 -1
  16. package/dist/cjs/llm/init.cjs +3 -3
  17. package/dist/cjs/llm/invoke.cjs +2 -2
  18. package/dist/cjs/llm/mistral/index.cjs +26 -0
  19. package/dist/cjs/llm/mistral/index.cjs.map +1 -0
  20. package/dist/cjs/llm/openai/index.cjs +82 -80
  21. package/dist/cjs/llm/openai/index.cjs.map +1 -1
  22. package/dist/cjs/llm/openrouter/index.cjs.map +1 -1
  23. package/dist/cjs/llm/providers.cjs +19 -19
  24. package/dist/cjs/llm/providers.cjs.map +1 -1
  25. package/dist/cjs/llm/stream/chunkAdapters.cjs +198 -0
  26. package/dist/cjs/llm/stream/chunkAdapters.cjs.map +1 -0
  27. package/dist/cjs/llm/stream/smoother.cjs +369 -0
  28. package/dist/cjs/llm/stream/smoother.cjs.map +1 -0
  29. package/dist/cjs/llm/vertexai/index.cjs +13 -1
  30. package/dist/cjs/llm/vertexai/index.cjs.map +1 -1
  31. package/dist/cjs/main.cjs +25 -16
  32. package/dist/cjs/messages/format.cjs +40 -23
  33. package/dist/cjs/messages/format.cjs.map +1 -1
  34. package/dist/cjs/run.cjs +82 -15
  35. package/dist/cjs/run.cjs.map +1 -1
  36. package/dist/cjs/session/AgentSession.cjs +37 -2
  37. package/dist/cjs/session/AgentSession.cjs.map +1 -1
  38. package/dist/cjs/stream.cjs +1 -1
  39. package/dist/cjs/summarization/node.cjs +6 -3
  40. package/dist/cjs/summarization/node.cjs.map +1 -1
  41. package/dist/cjs/tools/BashExecutor.cjs +1 -1
  42. package/dist/cjs/tools/CodeExecutor.cjs +1 -1
  43. package/dist/cjs/tools/ProgrammaticToolCalling.cjs +1 -1
  44. package/dist/cjs/tools/ToolNode.cjs +316 -94
  45. package/dist/cjs/tools/ToolNode.cjs.map +1 -1
  46. package/dist/cjs/tools/ToolSearch.cjs +1 -1
  47. package/dist/cjs/tools/subagent/SubagentExecutor.cjs +664 -80
  48. package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -1
  49. package/dist/cjs/tools/subagent/SubagentReplay.cjs +175 -0
  50. package/dist/cjs/tools/subagent/SubagentReplay.cjs.map +1 -0
  51. package/dist/cjs/tools/toolOutputReferences.cjs +20 -0
  52. package/dist/cjs/tools/toolOutputReferences.cjs.map +1 -1
  53. package/dist/cjs/utils/index.cjs +2 -2
  54. package/dist/cjs/utils/tokens.cjs +76 -29
  55. package/dist/cjs/utils/tokens.cjs.map +1 -1
  56. package/dist/esm/graphs/Graph.mjs +121 -27
  57. package/dist/esm/graphs/Graph.mjs.map +1 -1
  58. package/dist/esm/hooks/HookRegistry.mjs +83 -0
  59. package/dist/esm/hooks/HookRegistry.mjs.map +1 -1
  60. package/dist/esm/hooks/executeHooks.mjs +60 -14
  61. package/dist/esm/hooks/executeHooks.mjs.map +1 -1
  62. package/dist/esm/hooks/index.mjs.map +1 -1
  63. package/dist/esm/hooks/types.mjs +2 -1
  64. package/dist/esm/hooks/types.mjs.map +1 -1
  65. package/dist/esm/llm/anthropic/index.mjs +34 -205
  66. package/dist/esm/llm/anthropic/index.mjs.map +1 -1
  67. package/dist/esm/llm/bedrock/index.mjs +120 -240
  68. package/dist/esm/llm/bedrock/index.mjs.map +1 -1
  69. package/dist/esm/llm/google/index.mjs +19 -12
  70. package/dist/esm/llm/google/index.mjs.map +1 -1
  71. package/dist/esm/llm/init.mjs +1 -1
  72. package/dist/esm/llm/invoke.mjs +2 -2
  73. package/dist/esm/llm/mistral/index.mjs +26 -0
  74. package/dist/esm/llm/mistral/index.mjs.map +1 -0
  75. package/dist/esm/llm/openai/index.mjs +82 -80
  76. package/dist/esm/llm/openai/index.mjs.map +1 -1
  77. package/dist/esm/llm/openrouter/index.mjs.map +1 -1
  78. package/dist/esm/llm/providers.mjs +3 -3
  79. package/dist/esm/llm/providers.mjs.map +1 -1
  80. package/dist/esm/llm/stream/chunkAdapters.mjs +195 -0
  81. package/dist/esm/llm/stream/chunkAdapters.mjs.map +1 -0
  82. package/dist/esm/llm/stream/smoother.mjs +365 -0
  83. package/dist/esm/llm/stream/smoother.mjs.map +1 -0
  84. package/dist/esm/llm/vertexai/index.mjs +13 -1
  85. package/dist/esm/llm/vertexai/index.mjs.map +1 -1
  86. package/dist/esm/main.mjs +13 -11
  87. package/dist/esm/messages/format.mjs +40 -23
  88. package/dist/esm/messages/format.mjs.map +1 -1
  89. package/dist/esm/run.mjs +83 -16
  90. package/dist/esm/run.mjs.map +1 -1
  91. package/dist/esm/session/AgentSession.mjs +37 -2
  92. package/dist/esm/session/AgentSession.mjs.map +1 -1
  93. package/dist/esm/stream.mjs +1 -1
  94. package/dist/esm/summarization/node.mjs +6 -3
  95. package/dist/esm/summarization/node.mjs.map +1 -1
  96. package/dist/esm/tools/BashExecutor.mjs +1 -1
  97. package/dist/esm/tools/CodeExecutor.mjs +1 -1
  98. package/dist/esm/tools/ProgrammaticToolCalling.mjs +1 -1
  99. package/dist/esm/tools/ToolNode.mjs +317 -95
  100. package/dist/esm/tools/ToolNode.mjs.map +1 -1
  101. package/dist/esm/tools/ToolSearch.mjs +1 -1
  102. package/dist/esm/tools/subagent/SubagentExecutor.mjs +665 -81
  103. package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -1
  104. package/dist/esm/tools/subagent/SubagentReplay.mjs +168 -0
  105. package/dist/esm/tools/subagent/SubagentReplay.mjs.map +1 -0
  106. package/dist/esm/tools/toolOutputReferences.mjs +20 -0
  107. package/dist/esm/tools/toolOutputReferences.mjs.map +1 -1
  108. package/dist/esm/utils/index.mjs +2 -2
  109. package/dist/esm/utils/tokens.mjs +76 -30
  110. package/dist/esm/utils/tokens.mjs.map +1 -1
  111. package/dist/types/graphs/Graph.d.ts +19 -1
  112. package/dist/types/hooks/HookRegistry.d.ts +10 -1
  113. package/dist/types/hooks/executeHooks.d.ts +5 -1
  114. package/dist/types/hooks/index.d.ts +2 -2
  115. package/dist/types/hooks/types.d.ts +12 -0
  116. package/dist/types/index.d.ts +3 -0
  117. package/dist/types/llm/google/index.d.ts +2 -0
  118. package/dist/types/llm/mistral/index.d.ts +11 -0
  119. package/dist/types/llm/openai/index.d.ts +20 -4
  120. package/dist/types/llm/openrouter/index.d.ts +4 -1
  121. package/dist/types/llm/stream/chunkAdapters.d.ts +48 -0
  122. package/dist/types/llm/stream/smoother.d.ts +95 -0
  123. package/dist/types/llm/vertexai/index.d.ts +2 -0
  124. package/dist/types/run.d.ts +4 -1
  125. package/dist/types/session/AgentSession.d.ts +1 -0
  126. package/dist/types/tools/ToolNode.d.ts +23 -15
  127. package/dist/types/tools/subagent/SubagentExecutor.d.ts +49 -7
  128. package/dist/types/tools/subagent/SubagentReplay.d.ts +81 -0
  129. package/dist/types/tools/toolOutputReferences.d.ts +12 -0
  130. package/dist/types/types/graph.d.ts +5 -5
  131. package/dist/types/types/hitl.d.ts +15 -0
  132. package/dist/types/types/llm.d.ts +21 -14
  133. package/dist/types/utils/tokens.d.ts +10 -0
  134. package/package.json +6 -2
  135. package/src/graphs/Graph.ts +257 -52
  136. package/src/graphs/__tests__/Graph.breakerLifecycle.test.ts +48 -9
  137. package/src/graphs/__tests__/Graph.contextOverflow.test.ts +1 -1
  138. package/src/graphs/__tests__/Graph.subagentResumeState.test.ts +80 -0
  139. package/src/hooks/HookRegistry.ts +151 -1
  140. package/src/hooks/__tests__/HookRegistry.test.ts +96 -0
  141. package/src/hooks/__tests__/executeHooks.test.ts +160 -0
  142. package/src/hooks/executeHooks.ts +133 -15
  143. package/src/hooks/index.ts +3 -1
  144. package/src/hooks/types.ts +16 -0
  145. package/src/index.ts +8 -0
  146. package/src/llm/anthropic/index.ts +85 -354
  147. package/src/llm/bedrock/index.ts +174 -390
  148. package/src/llm/bedrock/llm.spec.ts +2 -0
  149. package/src/llm/bedrock/streamSealDispatch.test.ts +76 -0
  150. package/src/llm/custom-chat-models.smoke.test.ts +16 -1
  151. package/src/llm/google/index.ts +17 -18
  152. package/src/llm/google/streamSmoothing.test.ts +121 -0
  153. package/src/llm/mistral/index.ts +33 -0
  154. package/src/llm/mistral/streamSmoothing.test.ts +97 -0
  155. package/src/llm/openai/deepseek.test.ts +56 -0
  156. package/src/llm/openai/index.ts +119 -126
  157. package/src/llm/openrouter/index.ts +4 -1
  158. package/src/llm/providers.ts +3 -3
  159. package/src/llm/stream/chunkAdapters.test.ts +202 -0
  160. package/src/llm/stream/chunkAdapters.ts +317 -0
  161. package/src/llm/stream/reassembly.test.ts +241 -0
  162. package/src/llm/stream/smoother.bench.test.ts +155 -0
  163. package/src/llm/stream/smoother.test.ts +519 -0
  164. package/src/llm/stream/smoother.ts +574 -0
  165. package/src/llm/vertexai/index.ts +17 -1
  166. package/src/llm/vertexai/streamSmoothing.test.ts +109 -0
  167. package/src/messages/format.ts +54 -26
  168. package/src/messages/formatAgentMessages.reducer.test.ts +162 -0
  169. package/src/messages/formatAgentMessages.steer.test.ts +16 -11
  170. package/src/messages/formatAgentMessages.test.ts +7 -2
  171. package/src/run.ts +164 -18
  172. package/src/session/AgentSession.ts +52 -2
  173. package/src/session/__tests__/JsonlSessionStore.test.ts +53 -0
  174. package/src/specs/subagent.test.ts +44 -0
  175. package/src/specs/tokens.test.ts +159 -14
  176. package/src/summarization/__tests__/node.test.ts +32 -14
  177. package/src/summarization/node.ts +15 -11
  178. package/src/tools/ToolNode.ts +631 -157
  179. package/src/tools/__tests__/SubagentExecutor.test.ts +590 -7
  180. package/src/tools/__tests__/SubagentReplay.test.ts +300 -0
  181. package/src/tools/__tests__/ToolNode.breakerSignal.test.ts +208 -2
  182. package/src/tools/__tests__/directToolHITLResumeScope.test.ts +486 -2
  183. package/src/tools/__tests__/hitl.test.ts +234 -5
  184. package/src/tools/__tests__/subagentHooks.test.ts +1040 -62
  185. package/src/tools/__tests__/toolOutputReferences.test.ts +19 -2
  186. package/src/tools/subagent/SubagentExecutor.ts +1438 -136
  187. package/src/tools/subagent/SubagentReplay.ts +575 -0
  188. package/src/tools/toolOutputReferences.ts +40 -1
  189. package/src/types/graph.ts +5 -5
  190. package/src/types/hitl.ts +16 -0
  191. package/src/types/llm.ts +53 -36
  192. package/src/utils/tokens.ts +115 -30
@@ -0,0 +1,317 @@
1
+ import { AIMessageChunk } from '@langchain/core/messages';
2
+ import { ChatGenerationChunk } from '@langchain/core/outputs';
3
+ import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
4
+ import type { SmoothItem, SmoothPiece } from '@/llm/stream/smoother';
5
+ import { smoothStream } from '@/llm/stream/smoother';
6
+
7
+ /**
8
+ * Rebuilds a generation chunk carrying one piece of a split plain-text chunk.
9
+ * Unsplit pieces return the original chunk untouched, so disabled smoothing is
10
+ * byte-identical to no smoothing. Usage metadata, additional kwargs and
11
+ * response metadata survive only on the first piece: the aggregator merges
12
+ * dicts by concatenating string fields, so replicating them across pieces
13
+ * would duplicate reasoning text and scalar metadata once per piece.
14
+ */
15
+ export function cloneGenerationChunkPiece(
16
+ chunk: ChatGenerationChunk,
17
+ piece: SmoothPiece
18
+ ): ChatGenerationChunk {
19
+ if (piece.isFirst && piece.isLast) {
20
+ return chunk;
21
+ }
22
+ const message = chunk.message as AIMessageChunk;
23
+ return new ChatGenerationChunk({
24
+ text: piece.text,
25
+ generationInfo: piece.isFirst ? chunk.generationInfo : undefined,
26
+ message: new AIMessageChunk(
27
+ Object.assign({}, message, {
28
+ content: piece.text,
29
+ usage_metadata: piece.isFirst ? message.usage_metadata : undefined,
30
+ additional_kwargs: piece.isFirst ? message.additional_kwargs : {},
31
+ response_metadata: piece.isFirst ? message.response_metadata : {},
32
+ })
33
+ ),
34
+ });
35
+ }
36
+
37
+ const SPLITTABLE_TEXT_PART_KEYS: ReadonlySet<string> = new Set([
38
+ 'type',
39
+ 'text',
40
+ 'index',
41
+ ]);
42
+
43
+ /**
44
+ * Returns the sole plain text part of a single-element complex content array
45
+ * (the shape google-common emits for text deltas), or null when the part
46
+ * carries anything beyond `type`/`text`/`index` (thought signatures, media,
47
+ * function calls) and must not be sliced. The part must carry a numeric
48
+ * `index`: it is the aggregator's list-merge key, and without it each split
49
+ * piece would append as a separate content part instead of merging back
50
+ * into one — index-less parts pace whole instead.
51
+ */
52
+ function getSplittableTextPart(
53
+ message: AIMessageChunk,
54
+ chunkText: string
55
+ ): Record<string, unknown> | null {
56
+ const content = message.content;
57
+ if (!Array.isArray(content) || content.length !== 1) {
58
+ return null;
59
+ }
60
+ const part: unknown = content[0];
61
+ if (part == null || typeof part !== 'object') {
62
+ return null;
63
+ }
64
+ const record = part as Record<string, unknown>;
65
+ if (typeof record.text !== 'string' || record.text !== chunkText) {
66
+ return null;
67
+ }
68
+ if (record.type != null && record.type !== 'text') {
69
+ return null;
70
+ }
71
+ if (typeof record.index !== 'number') {
72
+ return null;
73
+ }
74
+ if (!Object.keys(record).every((key) => SPLITTABLE_TEXT_PART_KEYS.has(key))) {
75
+ return null;
76
+ }
77
+ return record;
78
+ }
79
+
80
+ function clonePartGenerationChunkPiece(
81
+ chunk: ChatGenerationChunk,
82
+ part: Record<string, unknown>,
83
+ piece: SmoothPiece
84
+ ): ChatGenerationChunk {
85
+ if (piece.isFirst && piece.isLast) {
86
+ return chunk;
87
+ }
88
+ const message = chunk.message as AIMessageChunk;
89
+ return new ChatGenerationChunk({
90
+ text: piece.text,
91
+ generationInfo: piece.isFirst ? chunk.generationInfo : undefined,
92
+ message: new AIMessageChunk(
93
+ Object.assign({}, message, {
94
+ content: [Object.assign({}, part, { text: piece.text })],
95
+ usage_metadata: piece.isFirst ? message.usage_metadata : undefined,
96
+ additional_kwargs: piece.isFirst ? message.additional_kwargs : {},
97
+ response_metadata: piece.isFirst ? message.response_metadata : {},
98
+ })
99
+ ),
100
+ });
101
+ }
102
+
103
+ /**
104
+ * Provider-agnostic classification of a `ChatGenerationChunk` for the
105
+ * smoothing engine:
106
+ * - splittable: plain visible text — string content equal to `chunk.text`, or
107
+ * a single plain text part (google-common's delta shape) — with no
108
+ * logprobs / finish_reason; sliced adaptively at the pacing cadence.
109
+ * - atomic: any other text-bearing chunk (complex content arrays, logprobs,
110
+ * finish_reason, provider-specific reasoning payloads surfaced via
111
+ * `getAtomicText`) — paced as one piece, never split.
112
+ * - passthrough: tool-call deltas, usage-only and metadata chunks — strict
113
+ * FIFO, zero delay.
114
+ */
115
+ /**
116
+ * google-common stamps `logprobs: { content: [] }` on every chunk, so only
117
+ * logprobs that actually carry data may block splitting here. This is looser
118
+ * than the OpenAI-family adapter in `llm/openai/index.ts`, which treats ANY
119
+ * logprobs as atomic (its providers only attach logprobs when requested, and
120
+ * the DeepSeek suite pins that contract) — do not unify the two predicates.
121
+ */
122
+ function hasMeaningfulLogprobs(
123
+ generationInfo: ChatGenerationChunk['generationInfo']
124
+ ): boolean {
125
+ const logprobs = generationInfo?.logprobs;
126
+ if (logprobs == null) {
127
+ return false;
128
+ }
129
+ const content = (logprobs as { content?: unknown }).content;
130
+ if (Array.isArray(content)) {
131
+ return content.length > 0;
132
+ }
133
+ return true;
134
+ }
135
+
136
+ /** google-common reports the terminal reason camelCase (`finishReason`). */
137
+ function hasFinishReason(
138
+ generationInfo: ChatGenerationChunk['generationInfo']
139
+ ): boolean {
140
+ return (
141
+ generationInfo?.finish_reason != null || generationInfo?.finishReason != null
142
+ );
143
+ }
144
+
145
+ /**
146
+ * Chunks that pair visible text with reasoning payloads in
147
+ * `additional_kwargs` (Gemini thought summaries, reasoning_content deltas,
148
+ * OpenRouter reasoning_details) must pace whole: split pieces would each
149
+ * carry the same kwargs and downstream merging — the aggregator's dict merge
150
+ * or OpenRouter's reasoning_details accumulation — duplicates them once per
151
+ * piece.
152
+ */
153
+ /**
154
+ * Mixed text/tool-call deltas must never split: cloned pieces would each
155
+ * carry the same tool_call_chunks and downstream accumulation would corrupt
156
+ * the assembled tool arguments.
157
+ */
158
+ export function hasToolCallChunks(message: AIMessageChunk): boolean {
159
+ return (message.tool_call_chunks?.length ?? 0) > 0;
160
+ }
161
+
162
+ export function hasReasoningKwargs(message: AIMessageChunk): boolean {
163
+ const kwargs = message.additional_kwargs;
164
+ if (
165
+ typeof kwargs.reasoning_content === 'string' &&
166
+ kwargs.reasoning_content !== ''
167
+ ) {
168
+ return true;
169
+ }
170
+ if (
171
+ Array.isArray(kwargs.reasoning_details) &&
172
+ kwargs.reasoning_details.length > 0
173
+ ) {
174
+ return true;
175
+ }
176
+ return kwargs.reasoning != null;
177
+ }
178
+
179
+ /**
180
+ * Extracts the visible reasoning text a kwargs-borne delta contributes, so
181
+ * reasoning-only chunks (Gemini thoughts, DeepSeek reasoning_content,
182
+ * OpenAI reasoning summaries, OpenRouter reasoning_details) pace atomically
183
+ * at the cadence instead of passing through unsmoothed.
184
+ */
185
+ export function getReasoningKwargsText(message: AIMessageChunk): string {
186
+ const kwargs = message.additional_kwargs;
187
+ if (
188
+ typeof kwargs.reasoning_content === 'string' &&
189
+ kwargs.reasoning_content !== ''
190
+ ) {
191
+ return kwargs.reasoning_content;
192
+ }
193
+ const reasoning = kwargs.reasoning;
194
+ if (typeof reasoning === 'string' && reasoning !== '') {
195
+ return reasoning;
196
+ }
197
+ if (reasoning != null && typeof reasoning === 'object') {
198
+ const summaryText = (
199
+ reasoning as { summary?: { text?: unknown }[] }
200
+ ).summary?.[0]?.text;
201
+ if (typeof summaryText === 'string' && summaryText !== '') {
202
+ return summaryText;
203
+ }
204
+ }
205
+ const details = kwargs.reasoning_details;
206
+ if (Array.isArray(details)) {
207
+ let text = '';
208
+ for (const detail of details) {
209
+ if (detail != null && typeof detail === 'object') {
210
+ const detailText = (detail as { text?: unknown }).text;
211
+ if (typeof detailText === 'string') {
212
+ text += detailText;
213
+ }
214
+ }
215
+ }
216
+ if (text !== '') {
217
+ return text;
218
+ }
219
+ }
220
+ return '';
221
+ }
222
+
223
+ export function toGenerationSmoothItem(
224
+ chunk: ChatGenerationChunk,
225
+ getAtomicText?: (message: AIMessageChunk) => string
226
+ ): SmoothItem<ChatGenerationChunk> {
227
+ const { message } = chunk;
228
+ const isMessageChunk = message instanceof AIMessageChunk;
229
+ const cleanGenerationInfo =
230
+ !hasMeaningfulLogprobs(chunk.generationInfo) &&
231
+ !hasFinishReason(chunk.generationInfo);
232
+ const splittable =
233
+ Boolean(chunk.text) &&
234
+ isMessageChunk &&
235
+ typeof message.content === 'string' &&
236
+ message.content === chunk.text &&
237
+ cleanGenerationInfo &&
238
+ !hasReasoningKwargs(message) &&
239
+ !hasToolCallChunks(message);
240
+
241
+ if (splittable) {
242
+ return {
243
+ text: chunk.text,
244
+ smooth: true,
245
+ emit: (piece) => cloneGenerationChunkPiece(chunk, piece),
246
+ };
247
+ }
248
+
249
+ const splittablePart =
250
+ Boolean(chunk.text) &&
251
+ isMessageChunk &&
252
+ cleanGenerationInfo &&
253
+ !hasReasoningKwargs(message) &&
254
+ !hasToolCallChunks(message)
255
+ ? getSplittableTextPart(message, chunk.text)
256
+ : null;
257
+ if (splittablePart != null) {
258
+ return {
259
+ text: chunk.text,
260
+ smooth: true,
261
+ emit: (piece) =>
262
+ clonePartGenerationChunkPiece(chunk, splittablePart, piece),
263
+ };
264
+ }
265
+
266
+ const pacedText =
267
+ chunk.text ||
268
+ (isMessageChunk && getAtomicText != null ? getAtomicText(message) : '');
269
+ if (pacedText !== '') {
270
+ return {
271
+ text: pacedText,
272
+ smooth: true,
273
+ atomic: true,
274
+ emit: () => chunk,
275
+ };
276
+ }
277
+
278
+ return { text: '', smooth: false, emit: () => chunk };
279
+ }
280
+
281
+ /**
282
+ * Wraps a provider's raw chunk stream with adaptive smoothing and per-piece
283
+ * `handleLLMNewToken` dispatch. The raw stream must NOT dispatch runManager
284
+ * callbacks itself — callback-echo consumers would otherwise observe the
285
+ * unsmoothed deltas.
286
+ */
287
+ export async function* smoothGenerationChunks({
288
+ chunks,
289
+ delayMs,
290
+ signal,
291
+ runManager,
292
+ }: {
293
+ chunks: AsyncGenerator<ChatGenerationChunk>;
294
+ delayMs: number;
295
+ signal?: AbortSignal;
296
+ runManager?: CallbackManagerForLLMRun;
297
+ }): AsyncGenerator<ChatGenerationChunk> {
298
+ const source = (async function* (): AsyncGenerator<
299
+ SmoothItem<ChatGenerationChunk>
300
+ > {
301
+ for await (const chunk of chunks) {
302
+ yield toGenerationSmoothItem(chunk, getReasoningKwargsText);
303
+ }
304
+ })();
305
+
306
+ for await (const outputChunk of smoothStream({ source, delayMs, signal })) {
307
+ yield outputChunk;
308
+ await runManager?.handleLLMNewToken(
309
+ outputChunk.text || '',
310
+ undefined,
311
+ undefined,
312
+ undefined,
313
+ undefined,
314
+ { chunk: outputChunk }
315
+ );
316
+ }
317
+ }
@@ -0,0 +1,241 @@
1
+ import { concat } from '@langchain/core/utils/stream';
2
+ import { AIMessageChunk } from '@langchain/core/messages';
3
+ import { ChatGenerationChunk } from '@langchain/core/outputs';
4
+ import type { SmoothItem } from './smoother';
5
+ import { toSmoothItem } from '@/llm/openai';
6
+ import {
7
+ toGenerationSmoothItem,
8
+ getReasoningKwargsText,
9
+ } from './chunkAdapters';
10
+ import { findStreamChunkBoundary } from './smoother';
11
+
12
+ /**
13
+ * Lossless-reassembly property: for ANY chunk shape an adapter classifies,
14
+ * aggregating the emitted pieces must reproduce the original chunk's
15
+ * observable payload — text, content, kwargs, response metadata, usage,
16
+ * tool calls. This is the invariant behind every split-corruption bug class
17
+ * (duplicated reasoning/kwargs, summed usage, cloned tool args): if a field
18
+ * cannot survive slicing, the adapter must classify the chunk atomic or
19
+ * scope the field to a single piece, and this property proves it did.
20
+ */
21
+
22
+ type AdapterName = 'openai-family' | 'generic';
23
+
24
+ const ADAPTERS: Record<
25
+ AdapterName,
26
+ (chunk: ChatGenerationChunk) => SmoothItem<ChatGenerationChunk>
27
+ > = {
28
+ 'openai-family': (chunk) => toSmoothItem(chunk),
29
+ generic: (chunk) => toGenerationSmoothItem(chunk, getReasoningKwargsText),
30
+ };
31
+
32
+ /** Emits the item the way the engine would: split smooth+splittable items
33
+ * into ≥2 boundary-aligned pieces; atomic and passthrough emit whole. */
34
+ function emitPieces(item: SmoothItem<ChatGenerationChunk>): ChatGenerationChunk[] {
35
+ if (!item.smooth || item.atomic === true || item.text.length < 8) {
36
+ return [item.emit({ text: item.text, isFirst: true, isLast: true })];
37
+ }
38
+ const pieces: ChatGenerationChunk[] = [];
39
+ let offset = 0;
40
+ while (offset < item.text.length) {
41
+ const end = offset + findStreamChunkBoundary(item.text.slice(offset), 4);
42
+ pieces.push(
43
+ item.emit({
44
+ text: item.text.slice(offset, end),
45
+ isFirst: offset === 0,
46
+ isLast: end === item.text.length,
47
+ })
48
+ );
49
+ offset = end;
50
+ }
51
+ return pieces;
52
+ }
53
+
54
+ function aggregate(pieces: ChatGenerationChunk[]): AIMessageChunk {
55
+ let merged = pieces[0].message as AIMessageChunk;
56
+ for (const piece of pieces.slice(1)) {
57
+ merged = concat(merged, piece.message as AIMessageChunk);
58
+ }
59
+ return merged;
60
+ }
61
+
62
+ function assertLossless(
63
+ adapter: AdapterName,
64
+ chunk: ChatGenerationChunk
65
+ ): void {
66
+ const item = ADAPTERS[adapter](chunk);
67
+ const pieces = emitPieces(item);
68
+ const original = chunk.message as AIMessageChunk;
69
+
70
+ const joinedText = pieces.map((p) => p.text).join('');
71
+ if (item.smooth) {
72
+ expect(joinedText === chunk.text || joinedText === item.text).toBe(true);
73
+ }
74
+
75
+ const merged = aggregate(pieces);
76
+ expect(merged.content).toEqual(original.content);
77
+ expect(merged.additional_kwargs).toEqual(original.additional_kwargs);
78
+ expect(merged.response_metadata).toEqual(original.response_metadata);
79
+ /** concat() adds empty *_token_details objects during merge; token COUNTS
80
+ * are the corruption signal (replicated usage would sum). */
81
+ expect(merged.usage_metadata?.input_tokens).toBe(
82
+ original.usage_metadata?.input_tokens
83
+ );
84
+ expect(merged.usage_metadata?.output_tokens).toBe(
85
+ original.usage_metadata?.output_tokens
86
+ );
87
+ expect(merged.usage_metadata?.total_tokens).toBe(
88
+ original.usage_metadata?.total_tokens
89
+ );
90
+ expect(merged.tool_call_chunks ?? []).toEqual(
91
+ original.tool_call_chunks ?? []
92
+ );
93
+ }
94
+
95
+ type MatrixCase = {
96
+ name: string;
97
+ chunk: () => ChatGenerationChunk;
98
+ adapters: AdapterName[];
99
+ };
100
+
101
+ const TEXT = 'the quick brown fox jumps over the lazy dog again and again ';
102
+
103
+ const MATRIX: MatrixCase[] = [
104
+ {
105
+ name: 'plain string text',
106
+ adapters: ['openai-family', 'generic'],
107
+ chunk: () =>
108
+ new ChatGenerationChunk({
109
+ text: TEXT,
110
+ message: new AIMessageChunk({ content: TEXT }),
111
+ }),
112
+ },
113
+ {
114
+ name: 'text with usage metadata',
115
+ adapters: ['openai-family', 'generic'],
116
+ chunk: () =>
117
+ new ChatGenerationChunk({
118
+ text: TEXT,
119
+ message: new AIMessageChunk({
120
+ content: TEXT,
121
+ usage_metadata: { input_tokens: 7, output_tokens: 11, total_tokens: 18 },
122
+ }),
123
+ }),
124
+ },
125
+ {
126
+ name: 'text with an unknown string kwargs field',
127
+ adapters: ['openai-family', 'generic'],
128
+ chunk: () =>
129
+ new ChatGenerationChunk({
130
+ text: TEXT,
131
+ message: new AIMessageChunk({
132
+ content: TEXT,
133
+ additional_kwargs: { annotation: 'gateway-added-value' },
134
+ }),
135
+ }),
136
+ },
137
+ {
138
+ name: 'text with scalar response metadata',
139
+ adapters: ['openai-family', 'generic'],
140
+ chunk: () =>
141
+ new ChatGenerationChunk({
142
+ text: TEXT,
143
+ message: new AIMessageChunk({
144
+ content: TEXT,
145
+ response_metadata: { model_name: 'test-model', system_fingerprint: 'fp_1' },
146
+ }),
147
+ }),
148
+ },
149
+ {
150
+ name: 'text with reasoning_content kwargs',
151
+ adapters: ['openai-family', 'generic'],
152
+ chunk: () =>
153
+ new ChatGenerationChunk({
154
+ text: TEXT,
155
+ message: new AIMessageChunk({
156
+ content: TEXT,
157
+ additional_kwargs: { reasoning_content: 'a hidden thought' },
158
+ }),
159
+ }),
160
+ },
161
+ {
162
+ name: 'text with reasoning_details kwargs',
163
+ adapters: ['openai-family', 'generic'],
164
+ chunk: () =>
165
+ new ChatGenerationChunk({
166
+ text: TEXT,
167
+ message: new AIMessageChunk({
168
+ content: TEXT,
169
+ additional_kwargs: {
170
+ reasoning_details: [{ type: 'reasoning.text', text: 'thought' }],
171
+ },
172
+ }),
173
+ }),
174
+ },
175
+ {
176
+ name: 'text alongside tool_call_chunks',
177
+ adapters: ['openai-family', 'generic'],
178
+ chunk: () =>
179
+ new ChatGenerationChunk({
180
+ text: TEXT,
181
+ message: new AIMessageChunk({
182
+ content: TEXT,
183
+ tool_call_chunks: [
184
+ { name: 'lookup', args: '{"q":1}', id: 'call_1', index: 0, type: 'tool_call_chunk' },
185
+ ],
186
+ }),
187
+ }),
188
+ },
189
+ {
190
+ name: 'indexed text-part array content (google shape)',
191
+ adapters: ['generic'],
192
+ chunk: () =>
193
+ new ChatGenerationChunk({
194
+ text: TEXT,
195
+ message: new AIMessageChunk({
196
+ content: [{ type: 'text', text: TEXT, index: 0 }],
197
+ }),
198
+ }),
199
+ },
200
+ {
201
+ name: 'index-less text-part array content (no merge key)',
202
+ adapters: ['generic'],
203
+ chunk: () =>
204
+ new ChatGenerationChunk({
205
+ text: TEXT,
206
+ message: new AIMessageChunk({
207
+ content: [{ type: 'text', text: TEXT }],
208
+ }),
209
+ }),
210
+ },
211
+ {
212
+ name: 'reasoning-only delta',
213
+ adapters: ['openai-family', 'generic'],
214
+ chunk: () =>
215
+ new ChatGenerationChunk({
216
+ text: '',
217
+ message: new AIMessageChunk({
218
+ content: '',
219
+ additional_kwargs: { reasoning_content: 'thought only delta here' },
220
+ }),
221
+ }),
222
+ },
223
+ {
224
+ name: 'usage-only delta',
225
+ adapters: ['openai-family', 'generic'],
226
+ chunk: () =>
227
+ new ChatGenerationChunk({
228
+ text: '',
229
+ message: new AIMessageChunk({
230
+ content: '',
231
+ usage_metadata: { input_tokens: 1, output_tokens: 2, total_tokens: 3 },
232
+ }),
233
+ }),
234
+ },
235
+ ];
236
+
237
+ describe.each(MATRIX)('lossless reassembly: $name', ({ chunk, adapters }) => {
238
+ it.each(adapters)('%s adapter', (adapter) => {
239
+ assertLossless(adapter, chunk());
240
+ });
241
+ });
@@ -0,0 +1,155 @@
1
+ import http from 'node:http';
2
+ import { HumanMessage } from '@langchain/core/messages';
3
+ import type { AddressInfo } from 'node:net';
4
+ import { ChatOpenAI } from '@/llm/openai';
5
+ import { SMOOTH_TARGET_LATENCY_MS } from './smoother';
6
+
7
+ /**
8
+ * End-to-end cadence/lag benchmark: drives the real ChatOpenAI client against
9
+ * a local SSE server that emits large chunks fast (the big-chunk gateway
10
+ * profile smoothing exists for), asserting the properties that define the
11
+ * feature: even cadence at the configured tick, and render lag pinned near
12
+ * the target latency instead of growing with reply length.
13
+ */
14
+ describe('adaptive smoothing benchmark (big-chunk provider)', () => {
15
+ const WORDS =
16
+ 'ClickHouse stores data in columns so queries read only what they touch and compress well '.split(
17
+ ' '
18
+ );
19
+ const CHUNK_CHARS = 110;
20
+ const PROVIDER_INTERVAL_MS = 20;
21
+
22
+ function buildChunks(totalWords: number): string[] {
23
+ const out: string[] = [];
24
+ let current = '';
25
+ for (let i = 0; i < totalWords; i++) {
26
+ current += WORDS[i % WORDS.length] + ' ';
27
+ if (current.length >= CHUNK_CHARS) {
28
+ out.push(current);
29
+ current = '';
30
+ }
31
+ }
32
+ if (current) {
33
+ out.push(current);
34
+ }
35
+ return out;
36
+ }
37
+
38
+ function startServer(chunks: string[], stamp: { doneAt: number }): Promise<{
39
+ server: http.Server;
40
+ port: number;
41
+ }> {
42
+ return new Promise((resolve) => {
43
+ const server = http.createServer((req, res) => {
44
+ res.writeHead(200, { 'Content-Type': 'text/event-stream' });
45
+ const frame = (
46
+ delta: Record<string, unknown>,
47
+ finish: string | null
48
+ ): string =>
49
+ `data: ${JSON.stringify({
50
+ id: 'bench',
51
+ object: 'chat.completion.chunk',
52
+ created: 1,
53
+ model: 'bench-model',
54
+ choices: [{ index: 0, delta, finish_reason: finish }],
55
+ })}\n\n`;
56
+ res.write(frame({ role: 'assistant', content: '' }, null));
57
+ let i = 0;
58
+ const send = (): void => {
59
+ if (i < chunks.length) {
60
+ res.write(frame({ content: chunks[i] }, null));
61
+ i += 1;
62
+ setTimeout(send, PROVIDER_INTERVAL_MS);
63
+ } else {
64
+ stamp.doneAt = Date.now();
65
+ res.write(frame({}, 'stop'));
66
+ res.write('data: [DONE]\n\n');
67
+ res.end();
68
+ }
69
+ };
70
+ send();
71
+ });
72
+ server.listen(0, '127.0.0.1', () => {
73
+ resolve({ server, port: (server.address() as AddressInfo).port });
74
+ });
75
+ });
76
+ }
77
+
78
+ async function measure(
79
+ totalWords: number,
80
+ tick: number
81
+ ): Promise<{
82
+ meanGap: number;
83
+ jitter: number;
84
+ lag: number;
85
+ totalText: string;
86
+ expected: string;
87
+ }> {
88
+ const chunks = buildChunks(totalWords);
89
+ const stamp = { doneAt: 0 };
90
+ const { server, port } = await startServer(chunks, stamp);
91
+ try {
92
+ const model = new ChatOpenAI({
93
+ model: 'bench-model',
94
+ apiKey: 'bench',
95
+ streaming: true,
96
+ configuration: { baseURL: `http://127.0.0.1:${port}/v1` },
97
+ _lc_stream_delay: tick,
98
+ });
99
+
100
+ const gaps: number[] = [];
101
+ let totalText = '';
102
+ let last = Date.now();
103
+ for await (const piece of await model.stream([new HumanMessage('go')])) {
104
+ if (typeof piece.content !== 'string' || piece.content === '') {
105
+ continue;
106
+ }
107
+ const now = Date.now();
108
+ gaps.push(now - last);
109
+ last = now;
110
+ totalText += piece.content;
111
+ }
112
+ const renderDoneAt = Date.now();
113
+
114
+ const paint = gaps.slice(1);
115
+ const mean = paint.reduce((a, b) => a + b, 0) / paint.length;
116
+ const jitter = Math.sqrt(
117
+ paint.reduce((a, b) => a + (b - mean) ** 2, 0) / paint.length
118
+ );
119
+ return {
120
+ meanGap: mean,
121
+ jitter,
122
+ lag: renderDoneAt - stamp.doneAt,
123
+ totalText,
124
+ expected: chunks.join(''),
125
+ };
126
+ } finally {
127
+ server.close();
128
+ }
129
+ }
130
+
131
+ /** Bounds are deliberately loose on absolutes (CI machines and parallel
132
+ * jest workers add scheduling noise); the discriminating assertion is the
133
+ * last one — lag must stay FLAT as reply length grows, which is the
134
+ * property fixed-rate smoothing lacks (it lagged +31s at 4x length). */
135
+ test('holds cadence at the tick with bounded lag on a medium reply', async () => {
136
+ const tick = 15;
137
+ const result = await measure(500, tick);
138
+
139
+ expect(result.totalText).toBe(result.expected);
140
+ expect(result.meanGap).toBeGreaterThan(tick * 0.5);
141
+ expect(result.meanGap).toBeLessThan(tick * 2.5);
142
+ expect(result.jitter).toBeLessThan(25);
143
+ expect(result.lag).toBeLessThan(SMOOTH_TARGET_LATENCY_MS * 4);
144
+ }, 60000);
145
+
146
+ test('lag does not grow with reply length', async () => {
147
+ const tick = 15;
148
+ const short = await measure(250, tick);
149
+ const long = await measure(1000, tick);
150
+
151
+ expect(long.totalText).toBe(long.expected);
152
+ expect(long.lag).toBeLessThan(SMOOTH_TARGET_LATENCY_MS * 4);
153
+ expect(long.lag).toBeLessThan(short.lag + SMOOTH_TARGET_LATENCY_MS * 2);
154
+ }, 60000);
155
+ });