@librechat/agents 3.3.13 → 3.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/graphs/Graph.cjs +121 -27
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/hooks/HookRegistry.cjs +83 -0
- package/dist/cjs/hooks/HookRegistry.cjs.map +1 -1
- package/dist/cjs/hooks/executeHooks.cjs +60 -14
- package/dist/cjs/hooks/executeHooks.cjs.map +1 -1
- package/dist/cjs/hooks/index.cjs.map +1 -1
- package/dist/cjs/hooks/types.cjs +2 -0
- package/dist/cjs/hooks/types.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/index.cjs +35 -206
- package/dist/cjs/llm/anthropic/index.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/index.cjs +121 -241
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/llm/google/index.cjs +19 -12
- package/dist/cjs/llm/google/index.cjs.map +1 -1
- package/dist/cjs/llm/init.cjs +3 -3
- package/dist/cjs/llm/invoke.cjs +2 -2
- package/dist/cjs/llm/mistral/index.cjs +26 -0
- package/dist/cjs/llm/mistral/index.cjs.map +1 -0
- package/dist/cjs/llm/openai/index.cjs +82 -80
- package/dist/cjs/llm/openai/index.cjs.map +1 -1
- package/dist/cjs/llm/openrouter/index.cjs.map +1 -1
- package/dist/cjs/llm/providers.cjs +19 -19
- package/dist/cjs/llm/providers.cjs.map +1 -1
- package/dist/cjs/llm/stream/chunkAdapters.cjs +198 -0
- package/dist/cjs/llm/stream/chunkAdapters.cjs.map +1 -0
- package/dist/cjs/llm/stream/smoother.cjs +369 -0
- package/dist/cjs/llm/stream/smoother.cjs.map +1 -0
- package/dist/cjs/llm/vertexai/index.cjs +13 -1
- package/dist/cjs/llm/vertexai/index.cjs.map +1 -1
- package/dist/cjs/main.cjs +25 -16
- package/dist/cjs/messages/format.cjs +40 -23
- package/dist/cjs/messages/format.cjs.map +1 -1
- package/dist/cjs/run.cjs +82 -15
- package/dist/cjs/run.cjs.map +1 -1
- package/dist/cjs/session/AgentSession.cjs +37 -2
- package/dist/cjs/session/AgentSession.cjs.map +1 -1
- package/dist/cjs/stream.cjs +1 -1
- package/dist/cjs/summarization/node.cjs +6 -3
- package/dist/cjs/summarization/node.cjs.map +1 -1
- package/dist/cjs/tools/BashExecutor.cjs +1 -1
- package/dist/cjs/tools/CodeExecutor.cjs +1 -1
- package/dist/cjs/tools/ProgrammaticToolCalling.cjs +1 -1
- package/dist/cjs/tools/ToolNode.cjs +316 -94
- package/dist/cjs/tools/ToolNode.cjs.map +1 -1
- package/dist/cjs/tools/ToolSearch.cjs +1 -1
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs +664 -80
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -1
- package/dist/cjs/tools/subagent/SubagentReplay.cjs +175 -0
- package/dist/cjs/tools/subagent/SubagentReplay.cjs.map +1 -0
- package/dist/cjs/tools/toolOutputReferences.cjs +20 -0
- package/dist/cjs/tools/toolOutputReferences.cjs.map +1 -1
- package/dist/cjs/utils/index.cjs +2 -2
- package/dist/cjs/utils/tokens.cjs +76 -29
- package/dist/cjs/utils/tokens.cjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +121 -27
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/hooks/HookRegistry.mjs +83 -0
- package/dist/esm/hooks/HookRegistry.mjs.map +1 -1
- package/dist/esm/hooks/executeHooks.mjs +60 -14
- package/dist/esm/hooks/executeHooks.mjs.map +1 -1
- package/dist/esm/hooks/index.mjs.map +1 -1
- package/dist/esm/hooks/types.mjs +2 -1
- package/dist/esm/hooks/types.mjs.map +1 -1
- package/dist/esm/llm/anthropic/index.mjs +34 -205
- package/dist/esm/llm/anthropic/index.mjs.map +1 -1
- package/dist/esm/llm/bedrock/index.mjs +120 -240
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/llm/google/index.mjs +19 -12
- package/dist/esm/llm/google/index.mjs.map +1 -1
- package/dist/esm/llm/init.mjs +1 -1
- package/dist/esm/llm/invoke.mjs +2 -2
- package/dist/esm/llm/mistral/index.mjs +26 -0
- package/dist/esm/llm/mistral/index.mjs.map +1 -0
- package/dist/esm/llm/openai/index.mjs +82 -80
- package/dist/esm/llm/openai/index.mjs.map +1 -1
- package/dist/esm/llm/openrouter/index.mjs.map +1 -1
- package/dist/esm/llm/providers.mjs +3 -3
- package/dist/esm/llm/providers.mjs.map +1 -1
- package/dist/esm/llm/stream/chunkAdapters.mjs +195 -0
- package/dist/esm/llm/stream/chunkAdapters.mjs.map +1 -0
- package/dist/esm/llm/stream/smoother.mjs +365 -0
- package/dist/esm/llm/stream/smoother.mjs.map +1 -0
- package/dist/esm/llm/vertexai/index.mjs +13 -1
- package/dist/esm/llm/vertexai/index.mjs.map +1 -1
- package/dist/esm/main.mjs +13 -11
- package/dist/esm/messages/format.mjs +40 -23
- package/dist/esm/messages/format.mjs.map +1 -1
- package/dist/esm/run.mjs +83 -16
- package/dist/esm/run.mjs.map +1 -1
- package/dist/esm/session/AgentSession.mjs +37 -2
- package/dist/esm/session/AgentSession.mjs.map +1 -1
- package/dist/esm/stream.mjs +1 -1
- package/dist/esm/summarization/node.mjs +6 -3
- package/dist/esm/summarization/node.mjs.map +1 -1
- package/dist/esm/tools/BashExecutor.mjs +1 -1
- package/dist/esm/tools/CodeExecutor.mjs +1 -1
- package/dist/esm/tools/ProgrammaticToolCalling.mjs +1 -1
- package/dist/esm/tools/ToolNode.mjs +317 -95
- package/dist/esm/tools/ToolNode.mjs.map +1 -1
- package/dist/esm/tools/ToolSearch.mjs +1 -1
- package/dist/esm/tools/subagent/SubagentExecutor.mjs +665 -81
- package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -1
- package/dist/esm/tools/subagent/SubagentReplay.mjs +168 -0
- package/dist/esm/tools/subagent/SubagentReplay.mjs.map +1 -0
- package/dist/esm/tools/toolOutputReferences.mjs +20 -0
- package/dist/esm/tools/toolOutputReferences.mjs.map +1 -1
- package/dist/esm/utils/index.mjs +2 -2
- package/dist/esm/utils/tokens.mjs +76 -30
- package/dist/esm/utils/tokens.mjs.map +1 -1
- package/dist/types/graphs/Graph.d.ts +19 -1
- package/dist/types/hooks/HookRegistry.d.ts +10 -1
- package/dist/types/hooks/executeHooks.d.ts +5 -1
- package/dist/types/hooks/index.d.ts +2 -2
- package/dist/types/hooks/types.d.ts +12 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/types/llm/google/index.d.ts +2 -0
- package/dist/types/llm/mistral/index.d.ts +11 -0
- package/dist/types/llm/openai/index.d.ts +20 -4
- package/dist/types/llm/openrouter/index.d.ts +4 -1
- package/dist/types/llm/stream/chunkAdapters.d.ts +48 -0
- package/dist/types/llm/stream/smoother.d.ts +95 -0
- package/dist/types/llm/vertexai/index.d.ts +2 -0
- package/dist/types/run.d.ts +4 -1
- package/dist/types/session/AgentSession.d.ts +1 -0
- package/dist/types/tools/ToolNode.d.ts +23 -15
- package/dist/types/tools/subagent/SubagentExecutor.d.ts +49 -7
- package/dist/types/tools/subagent/SubagentReplay.d.ts +81 -0
- package/dist/types/tools/toolOutputReferences.d.ts +12 -0
- package/dist/types/types/graph.d.ts +5 -5
- package/dist/types/types/hitl.d.ts +15 -0
- package/dist/types/types/llm.d.ts +21 -14
- package/dist/types/utils/tokens.d.ts +10 -0
- package/package.json +6 -2
- package/src/graphs/Graph.ts +257 -52
- package/src/graphs/__tests__/Graph.breakerLifecycle.test.ts +48 -9
- package/src/graphs/__tests__/Graph.contextOverflow.test.ts +1 -1
- package/src/graphs/__tests__/Graph.subagentResumeState.test.ts +80 -0
- package/src/hooks/HookRegistry.ts +151 -1
- package/src/hooks/__tests__/HookRegistry.test.ts +96 -0
- package/src/hooks/__tests__/executeHooks.test.ts +160 -0
- package/src/hooks/executeHooks.ts +133 -15
- package/src/hooks/index.ts +3 -1
- package/src/hooks/types.ts +16 -0
- package/src/index.ts +8 -0
- package/src/llm/anthropic/index.ts +85 -354
- package/src/llm/bedrock/index.ts +174 -390
- package/src/llm/bedrock/llm.spec.ts +2 -0
- package/src/llm/bedrock/streamSealDispatch.test.ts +76 -0
- package/src/llm/custom-chat-models.smoke.test.ts +16 -1
- package/src/llm/google/index.ts +17 -18
- package/src/llm/google/streamSmoothing.test.ts +121 -0
- package/src/llm/mistral/index.ts +33 -0
- package/src/llm/mistral/streamSmoothing.test.ts +97 -0
- package/src/llm/openai/deepseek.test.ts +56 -0
- package/src/llm/openai/index.ts +119 -126
- package/src/llm/openrouter/index.ts +4 -1
- package/src/llm/providers.ts +3 -3
- package/src/llm/stream/chunkAdapters.test.ts +202 -0
- package/src/llm/stream/chunkAdapters.ts +317 -0
- package/src/llm/stream/reassembly.test.ts +241 -0
- package/src/llm/stream/smoother.bench.test.ts +155 -0
- package/src/llm/stream/smoother.test.ts +519 -0
- package/src/llm/stream/smoother.ts +574 -0
- package/src/llm/vertexai/index.ts +17 -1
- package/src/llm/vertexai/streamSmoothing.test.ts +109 -0
- package/src/messages/format.ts +54 -26
- package/src/messages/formatAgentMessages.reducer.test.ts +162 -0
- package/src/messages/formatAgentMessages.steer.test.ts +16 -11
- package/src/messages/formatAgentMessages.test.ts +7 -2
- package/src/run.ts +164 -18
- package/src/session/AgentSession.ts +52 -2
- package/src/session/__tests__/JsonlSessionStore.test.ts +53 -0
- package/src/specs/subagent.test.ts +44 -0
- package/src/specs/tokens.test.ts +159 -14
- package/src/summarization/__tests__/node.test.ts +32 -14
- package/src/summarization/node.ts +15 -11
- package/src/tools/ToolNode.ts +631 -157
- package/src/tools/__tests__/SubagentExecutor.test.ts +590 -7
- package/src/tools/__tests__/SubagentReplay.test.ts +300 -0
- package/src/tools/__tests__/ToolNode.breakerSignal.test.ts +208 -2
- package/src/tools/__tests__/directToolHITLResumeScope.test.ts +486 -2
- package/src/tools/__tests__/hitl.test.ts +234 -5
- package/src/tools/__tests__/subagentHooks.test.ts +1040 -62
- package/src/tools/__tests__/toolOutputReferences.test.ts +19 -2
- package/src/tools/subagent/SubagentExecutor.ts +1438 -136
- package/src/tools/subagent/SubagentReplay.ts +575 -0
- package/src/tools/toolOutputReferences.ts +40 -1
- package/src/types/graph.ts +5 -5
- package/src/types/hitl.ts +16 -0
- package/src/types/llm.ts +53 -36
- package/src/utils/tokens.ts +115 -30
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
//#region src/llm/stream/smoother.ts
|
|
2
|
+
const DEFAULT_STREAM_DELAY = 25;
|
|
3
|
+
const MAX_SMOOTH_ITEM_SEGMENT_CHARS = 4096;
|
|
4
|
+
const STREAM_BOUNDARIES = new Set([
|
|
5
|
+
" ",
|
|
6
|
+
".",
|
|
7
|
+
",",
|
|
8
|
+
"!",
|
|
9
|
+
"?",
|
|
10
|
+
";",
|
|
11
|
+
":"
|
|
12
|
+
]);
|
|
13
|
+
const STREAM_ABORT_MESSAGE = "AbortError: User aborted the request.";
|
|
14
|
+
/**
|
|
15
|
+
* How long generator teardown waits for the background producer to observe a
|
|
16
|
+
* consumer close before abandoning it. Well-behaved streams settle in
|
|
17
|
+
* microseconds (the next enqueue throws); a stalled provider that ignores
|
|
18
|
+
* aborts otherwise blocks teardown — and abort propagation — indefinitely.
|
|
19
|
+
* An abandoned producer still self-terminates on its next enqueue attempt.
|
|
20
|
+
*/
|
|
21
|
+
const PRODUCER_CLOSE_GRACE_MS = 1e3;
|
|
22
|
+
/**
|
|
23
|
+
* Resolves a configured stream delay to its effective value (default 25ms;
|
|
24
|
+
* 0 disables smoothing). Non-finite inputs (NaN from a malformed config
|
|
25
|
+
* value, ±Infinity) normalize to the default rather than poisoning piece
|
|
26
|
+
* arithmetic downstream.
|
|
27
|
+
*/
|
|
28
|
+
function resolveStreamDelay(delay) {
|
|
29
|
+
if (delay == null || !Number.isFinite(delay)) return 25;
|
|
30
|
+
return Math.max(0, delay);
|
|
31
|
+
}
|
|
32
|
+
function isSignalAborted(signal) {
|
|
33
|
+
return signal?.aborted === true;
|
|
34
|
+
}
|
|
35
|
+
function findStreamChunkBoundary(text, minSize) {
|
|
36
|
+
if (minSize >= text.length) return text.length;
|
|
37
|
+
const scanEnd = Math.min(text.length, minSize + 64);
|
|
38
|
+
for (let position = minSize; position < scanEnd; position++) if (STREAM_BOUNDARIES.has(text[position])) return position + 1;
|
|
39
|
+
return scanEnd;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Backlog-proportional piece sizing: emit enough per tick that the current
|
|
43
|
+
* backlog drains in ~`targetLatencyMs`, so render lag stays pinned near the
|
|
44
|
+
* target regardless of how fast the provider streams. Token-sized arrivals
|
|
45
|
+
* never exceed the minimum piece, matching the legacy fixed-size splitter.
|
|
46
|
+
*/
|
|
47
|
+
function computeAdaptivePieceSize(bufferedTextLength, tickMs, targetLatencyMs = 250) {
|
|
48
|
+
if (bufferedTextLength <= 0) return 4;
|
|
49
|
+
if (tickMs <= 0 || targetLatencyMs <= 0) return bufferedTextLength;
|
|
50
|
+
return Math.max(4, Math.ceil(bufferedTextLength * tickMs / targetLatencyMs));
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A cadence, not an additive sleep: time the consumer already spent since the
|
|
54
|
+
* last visible emission counts against the target delay, so slow downstream
|
|
55
|
+
* handlers never compound latency.
|
|
56
|
+
*/
|
|
57
|
+
function getCadencedStreamDelay({ targetDelay, lastVisibleTextAt, now }) {
|
|
58
|
+
if (targetDelay <= 0 || lastVisibleTextAt == null) return 0;
|
|
59
|
+
return Math.max(0, targetDelay - (now - lastVisibleTextAt));
|
|
60
|
+
}
|
|
61
|
+
/** Abort-aware sleep that resolves (never rejects) on abort; callers re-check the signal. */
|
|
62
|
+
async function waitForStreamDelay(delay, signal) {
|
|
63
|
+
if (delay <= 0 || isSignalAborted(signal)) return;
|
|
64
|
+
await new Promise((resolve) => {
|
|
65
|
+
const timeoutRef = {};
|
|
66
|
+
const onAbort = () => {
|
|
67
|
+
if (timeoutRef.current) clearTimeout(timeoutRef.current);
|
|
68
|
+
signal?.removeEventListener("abort", onAbort);
|
|
69
|
+
resolve();
|
|
70
|
+
};
|
|
71
|
+
timeoutRef.current = setTimeout(() => {
|
|
72
|
+
signal?.removeEventListener("abort", onAbort);
|
|
73
|
+
resolve();
|
|
74
|
+
}, delay);
|
|
75
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
76
|
+
if (isSignalAborted(signal)) onAbort();
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Bounded producer/consumer smoothing engine.
|
|
81
|
+
*
|
|
82
|
+
* The producer drains `source` eagerly into a bounded queue (the buffer is the
|
|
83
|
+
* backlog measurement adaptive sizing needs); at capacity it parks, applying
|
|
84
|
+
* backpressure to the underlying stream. The consumer emits paced pieces,
|
|
85
|
+
* decrementing the text budget and waking the producer *before* each cadenced
|
|
86
|
+
* sleep so the provider stream keeps being read during pacing.
|
|
87
|
+
*
|
|
88
|
+
* `delayMs <= 0` disables smoothing entirely: every item passes through FIFO,
|
|
89
|
+
* unsplit and undelayed.
|
|
90
|
+
*/
|
|
91
|
+
async function* smoothStream({ source, delayMs, signal, abortUpstream }) {
|
|
92
|
+
if (!(delayMs > 0)) {
|
|
93
|
+
/** Disabled smoothing preserves fully lazy streaming: no background
|
|
94
|
+
* producer, no read-ahead — each provider chunk is pulled only when the
|
|
95
|
+
* consumer asks, exactly like the pre-engine pass-through paths. */
|
|
96
|
+
for await (const item of source) {
|
|
97
|
+
if (isSignalAborted(signal)) {
|
|
98
|
+
abortUpstream?.();
|
|
99
|
+
throw new Error(STREAM_ABORT_MESSAGE);
|
|
100
|
+
}
|
|
101
|
+
yield item.emit({
|
|
102
|
+
text: item.text,
|
|
103
|
+
isFirst: true,
|
|
104
|
+
isLast: true
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
const queuedItems = [];
|
|
110
|
+
const producerState = {
|
|
111
|
+
done: false,
|
|
112
|
+
failed: false
|
|
113
|
+
};
|
|
114
|
+
let queuedItemIndex = 0;
|
|
115
|
+
let bufferedTextLength = 0;
|
|
116
|
+
let consumerClosed = false;
|
|
117
|
+
let notifyConsumer;
|
|
118
|
+
let notifyProducer;
|
|
119
|
+
const notifyConsumerForItem = () => {
|
|
120
|
+
notifyConsumer?.();
|
|
121
|
+
notifyConsumer = void 0;
|
|
122
|
+
};
|
|
123
|
+
const notifyProducerForSpace = () => {
|
|
124
|
+
notifyProducer?.();
|
|
125
|
+
notifyProducer = void 0;
|
|
126
|
+
};
|
|
127
|
+
const hasQueuedItems = () => queuedItemIndex < queuedItems.length;
|
|
128
|
+
const getQueuedItemCount = () => queuedItems.length - queuedItemIndex;
|
|
129
|
+
const isQueueAtCapacity = () => getQueuedItemCount() >= 256 || bufferedTextLength >= 8192;
|
|
130
|
+
/** Abort-aware: a consumer parked on an empty queue must wake when the
|
|
131
|
+
* signal fires even if the provider stream never honors the abort — the
|
|
132
|
+
* loop's top-of-iteration check then throws the canonical error. */
|
|
133
|
+
const waitForNextItem = async () => {
|
|
134
|
+
if (hasQueuedItems() || producerState.done || producerState.failed || isSignalAborted(signal)) return;
|
|
135
|
+
await new Promise((resolve) => {
|
|
136
|
+
const onAbort = () => {
|
|
137
|
+
signal?.removeEventListener("abort", onAbort);
|
|
138
|
+
resolve();
|
|
139
|
+
};
|
|
140
|
+
notifyConsumer = () => {
|
|
141
|
+
signal?.removeEventListener("abort", onAbort);
|
|
142
|
+
resolve();
|
|
143
|
+
};
|
|
144
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
145
|
+
if (isSignalAborted(signal)) onAbort();
|
|
146
|
+
});
|
|
147
|
+
};
|
|
148
|
+
const waitForQueueSpace = async () => {
|
|
149
|
+
while (isQueueAtCapacity() && !consumerClosed && !isSignalAborted(signal)) await new Promise((resolve) => {
|
|
150
|
+
const onAbort = () => {
|
|
151
|
+
signal?.removeEventListener("abort", onAbort);
|
|
152
|
+
resolve();
|
|
153
|
+
};
|
|
154
|
+
const onSpace = () => {
|
|
155
|
+
signal?.removeEventListener("abort", onAbort);
|
|
156
|
+
resolve();
|
|
157
|
+
};
|
|
158
|
+
notifyProducer = onSpace;
|
|
159
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
160
|
+
if (isSignalAborted(signal)) onAbort();
|
|
161
|
+
});
|
|
162
|
+
};
|
|
163
|
+
const dequeue = () => {
|
|
164
|
+
if (!hasQueuedItems()) return;
|
|
165
|
+
const queuedItem = queuedItems[queuedItemIndex];
|
|
166
|
+
queuedItemIndex++;
|
|
167
|
+
if (queuedItemIndex > 128 && queuedItemIndex * 2 >= queuedItems.length) {
|
|
168
|
+
queuedItems.splice(0, queuedItemIndex);
|
|
169
|
+
queuedItemIndex = 0;
|
|
170
|
+
}
|
|
171
|
+
return queuedItem;
|
|
172
|
+
};
|
|
173
|
+
const throwAborted = () => {
|
|
174
|
+
abortUpstream?.();
|
|
175
|
+
throw new Error(STREAM_ABORT_MESSAGE);
|
|
176
|
+
};
|
|
177
|
+
const enqueue = async (item) => {
|
|
178
|
+
await waitForQueueSpace();
|
|
179
|
+
if (consumerClosed || isSignalAborted(signal)) throwAborted();
|
|
180
|
+
const textLength = item.smooth ? item.text.length : 0;
|
|
181
|
+
queuedItems.push({
|
|
182
|
+
item,
|
|
183
|
+
textLength
|
|
184
|
+
});
|
|
185
|
+
bufferedTextLength += textLength;
|
|
186
|
+
notifyConsumerForItem();
|
|
187
|
+
};
|
|
188
|
+
/**
|
|
189
|
+
* Oversized splittable items are segmented at admission so a single giant
|
|
190
|
+
* provider chunk cannot blow past the text budget: each segment re-checks
|
|
191
|
+
* capacity, so the producer parks mid-chunk once the buffer fills — the
|
|
192
|
+
* same bound the legacy split-before-enqueue queues enforced. The wrapped
|
|
193
|
+
* emit maps segment-local pieces back to chunk-global isFirst/isLast so
|
|
194
|
+
* provider clone contracts are unaffected.
|
|
195
|
+
*/
|
|
196
|
+
const enqueueSegmented = async (item) => {
|
|
197
|
+
if (!item.smooth || item.atomic === true || item.text.length <= 4096) {
|
|
198
|
+
await enqueue(item);
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
const segments = [];
|
|
202
|
+
let offset = 0;
|
|
203
|
+
while (offset < item.text.length) {
|
|
204
|
+
const end = offset + findStreamChunkBoundary(item.text.slice(offset), MAX_SMOOTH_ITEM_SEGMENT_CHARS);
|
|
205
|
+
segments.push({
|
|
206
|
+
start: offset,
|
|
207
|
+
end
|
|
208
|
+
});
|
|
209
|
+
offset = end;
|
|
210
|
+
}
|
|
211
|
+
for (let i = 0; i < segments.length; i++) {
|
|
212
|
+
const isFirstSegment = i === 0;
|
|
213
|
+
const isLastSegment = i === segments.length - 1;
|
|
214
|
+
await enqueue({
|
|
215
|
+
text: item.text.slice(segments[i].start, segments[i].end),
|
|
216
|
+
smooth: true,
|
|
217
|
+
emit: (piece) => item.emit({
|
|
218
|
+
text: piece.text,
|
|
219
|
+
isFirst: isFirstSegment && piece.isFirst,
|
|
220
|
+
isLast: isLastSegment && piece.isLast
|
|
221
|
+
})
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
const producer = (async () => {
|
|
226
|
+
try {
|
|
227
|
+
for await (const item of source) {
|
|
228
|
+
if (isSignalAborted(signal)) throwAborted();
|
|
229
|
+
await enqueueSegmented(item);
|
|
230
|
+
}
|
|
231
|
+
} catch (error) {
|
|
232
|
+
producerState.failed = true;
|
|
233
|
+
producerState.error = error;
|
|
234
|
+
} finally {
|
|
235
|
+
producerState.done = true;
|
|
236
|
+
notifyConsumerForItem();
|
|
237
|
+
}
|
|
238
|
+
})();
|
|
239
|
+
let hasEmittedText = false;
|
|
240
|
+
let lastVisibleTextAt;
|
|
241
|
+
let drainTicksRemaining;
|
|
242
|
+
let current;
|
|
243
|
+
let headOffset = 0;
|
|
244
|
+
let keepStreaming = true;
|
|
245
|
+
try {
|
|
246
|
+
while (keepStreaming) {
|
|
247
|
+
if (isSignalAborted(signal)) throwAborted();
|
|
248
|
+
if (current == null) {
|
|
249
|
+
await waitForNextItem();
|
|
250
|
+
current = dequeue();
|
|
251
|
+
headOffset = 0;
|
|
252
|
+
}
|
|
253
|
+
if (current == null) {
|
|
254
|
+
if (producerState.failed) throw producerState.error ?? /* @__PURE__ */ new Error("Stream producer failed.");
|
|
255
|
+
if (producerState.done) keepStreaming = false;
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
const { item } = current;
|
|
259
|
+
if (!item.smooth) {
|
|
260
|
+
notifyProducerForSpace();
|
|
261
|
+
current = void 0;
|
|
262
|
+
yield item.emit({
|
|
263
|
+
text: item.text,
|
|
264
|
+
isFirst: true,
|
|
265
|
+
isLast: true
|
|
266
|
+
});
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
if (item.text === "") {
|
|
270
|
+
bufferedTextLength = Math.max(0, bufferedTextLength - current.textLength);
|
|
271
|
+
notifyProducerForSpace();
|
|
272
|
+
current = void 0;
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
/** Once the producer is done the backlog is final: drain it linearly
|
|
276
|
+
* across the remaining target window instead of letting the
|
|
277
|
+
* proportional formula decay geometrically and stretch the tail. */
|
|
278
|
+
if (producerState.done && drainTicksRemaining == null) drainTicksRemaining = Math.max(1, Math.floor(250 / delayMs));
|
|
279
|
+
const tickBudget = drainTicksRemaining != null ? Math.max(4, Math.ceil(bufferedTextLength / drainTicksRemaining)) : computeAdaptivePieceSize(bufferedTextLength, delayMs);
|
|
280
|
+
if (drainTicksRemaining != null && drainTicksRemaining > 1) drainTicksRemaining -= 1;
|
|
281
|
+
await waitForStreamDelay(getCadencedStreamDelay({
|
|
282
|
+
targetDelay: hasEmittedText ? delayMs : 0,
|
|
283
|
+
lastVisibleTextAt,
|
|
284
|
+
now: Date.now()
|
|
285
|
+
}), signal);
|
|
286
|
+
if (isSignalAborted(signal)) throwAborted();
|
|
287
|
+
hasEmittedText = true;
|
|
288
|
+
lastVisibleTextAt = Date.now();
|
|
289
|
+
if (item.atomic === true) {
|
|
290
|
+
bufferedTextLength = Math.max(0, bufferedTextLength - current.textLength);
|
|
291
|
+
notifyProducerForSpace();
|
|
292
|
+
current = void 0;
|
|
293
|
+
yield item.emit({
|
|
294
|
+
text: item.text,
|
|
295
|
+
isFirst: true,
|
|
296
|
+
isLast: true
|
|
297
|
+
});
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
/** One cadence tick drains up to the adaptive budget ACROSS queued
|
|
301
|
+
* items, so token-sized provider deltas coalesce instead of costing a
|
|
302
|
+
* full tick each; passthrough items flush free mid-batch (FIFO), and
|
|
303
|
+
* atomic items end the batch to take their own tick. */
|
|
304
|
+
let consumed = 0;
|
|
305
|
+
while (consumed < tickBudget) {
|
|
306
|
+
if (isSignalAborted(signal)) throwAborted();
|
|
307
|
+
if (current == null) {
|
|
308
|
+
if (!hasQueuedItems()) break;
|
|
309
|
+
current = dequeue();
|
|
310
|
+
headOffset = 0;
|
|
311
|
+
if (current == null) break;
|
|
312
|
+
}
|
|
313
|
+
const batchItem = current.item;
|
|
314
|
+
if (!batchItem.smooth) {
|
|
315
|
+
notifyProducerForSpace();
|
|
316
|
+
current = void 0;
|
|
317
|
+
yield batchItem.emit({
|
|
318
|
+
text: batchItem.text,
|
|
319
|
+
isFirst: true,
|
|
320
|
+
isLast: true
|
|
321
|
+
});
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
if (batchItem.text === "") {
|
|
325
|
+
bufferedTextLength = Math.max(0, bufferedTextLength - current.textLength);
|
|
326
|
+
notifyProducerForSpace();
|
|
327
|
+
current = void 0;
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
if (batchItem.atomic === true) break;
|
|
331
|
+
const pieceLength = findStreamChunkBoundary(batchItem.text.slice(headOffset), tickBudget - consumed);
|
|
332
|
+
const pieceEnd = headOffset + pieceLength;
|
|
333
|
+
const piece = batchItem.text.slice(headOffset, pieceEnd);
|
|
334
|
+
const isFirst = headOffset === 0;
|
|
335
|
+
const isLast = pieceEnd === batchItem.text.length;
|
|
336
|
+
bufferedTextLength = Math.max(0, bufferedTextLength - piece.length);
|
|
337
|
+
notifyProducerForSpace();
|
|
338
|
+
if (isLast) current = void 0;
|
|
339
|
+
else headOffset = pieceEnd;
|
|
340
|
+
consumed += piece.length;
|
|
341
|
+
yield batchItem.emit({
|
|
342
|
+
text: piece,
|
|
343
|
+
isFirst,
|
|
344
|
+
isLast
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
} finally {
|
|
349
|
+
consumerClosed = true;
|
|
350
|
+
if (producerState.done) await producer;
|
|
351
|
+
else {
|
|
352
|
+
abortUpstream?.();
|
|
353
|
+
notifyProducerForSpace();
|
|
354
|
+
const closing = source.return?.call(source, void 0);
|
|
355
|
+
if (closing != null) closing.then(() => void 0, () => void 0);
|
|
356
|
+
await Promise.race([producer, new Promise((resolve) => {
|
|
357
|
+
setTimeout(resolve, PRODUCER_CLOSE_GRACE_MS).unref();
|
|
358
|
+
})]);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
//#endregion
|
|
363
|
+
exports.DEFAULT_STREAM_DELAY = DEFAULT_STREAM_DELAY;
|
|
364
|
+
exports.computeAdaptivePieceSize = computeAdaptivePieceSize;
|
|
365
|
+
exports.isSignalAborted = isSignalAborted;
|
|
366
|
+
exports.resolveStreamDelay = resolveStreamDelay;
|
|
367
|
+
exports.smoothStream = smoothStream;
|
|
368
|
+
|
|
369
|
+
//# sourceMappingURL=smoother.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"smoother.cjs","names":["iterator"],"sources":["../../../../src/llm/stream/smoother.ts"],"sourcesContent":["export const DEFAULT_STREAM_DELAY = 25;\nexport const SMOOTH_TARGET_LATENCY_MS = 250;\nexport const MAX_STREAM_QUEUE_CHUNKS = 256;\nexport const MAX_STREAM_QUEUE_TEXT_CHARS = 8192;\nexport const MAX_SMOOTH_ITEM_SEGMENT_CHARS = 4096;\nexport const STREAM_CHUNK_MIN_SIZE = 4;\nexport const STREAM_BOUNDARIES: ReadonlySet<string> = new Set([\n ' ',\n '.',\n ',',\n '!',\n '?',\n ';',\n ':',\n]);\n\nexport const STREAM_ABORT_MESSAGE = 'AbortError: User aborted the request.';\nexport const STREAM_PRODUCER_FAILURE = 'Stream producer failed.';\n\n/**\n * How long generator teardown waits for the background producer to observe a\n * consumer close before abandoning it. Well-behaved streams settle in\n * microseconds (the next enqueue throws); a stalled provider that ignores\n * aborts otherwise blocks teardown — and abort propagation — indefinitely.\n * An abandoned producer still self-terminates on its next enqueue attempt.\n */\nexport const PRODUCER_CLOSE_GRACE_MS = 1000;\n\n/**\n * Resolves a configured stream delay to its effective value (default 25ms;\n * 0 disables smoothing). Non-finite inputs (NaN from a malformed config\n * value, ±Infinity) normalize to the default rather than poisoning piece\n * arithmetic downstream.\n */\nexport function resolveStreamDelay(delay?: number): number {\n if (delay == null || !Number.isFinite(delay)) {\n return DEFAULT_STREAM_DELAY;\n }\n return Math.max(0, delay);\n}\n\nexport function isSignalAborted(signal?: AbortSignal): boolean {\n return signal?.aborted === true;\n}\n\n/**\n * How far past the target size the word-boundary search may extend before\n * hard-cutting. Natural language hits a boundary within a few characters;\n * boundary-free runs (base64, minified data, long identifiers) must not\n * stretch a piece — or an admission segment — arbitrarily far past its\n * budget.\n */\nexport const STREAM_BOUNDARY_LOOKAHEAD_CHARS = 64;\n\nexport function findStreamChunkBoundary(\n text: string,\n minSize: number\n): number {\n if (minSize >= text.length) {\n return text.length;\n }\n\n const scanEnd = Math.min(\n text.length,\n minSize + STREAM_BOUNDARY_LOOKAHEAD_CHARS\n );\n for (let position = minSize; position < scanEnd; position++) {\n if (STREAM_BOUNDARIES.has(text[position])) {\n return position + 1;\n }\n }\n\n return scanEnd;\n}\n\n/**\n * Backlog-proportional piece sizing: emit enough per tick that the current\n * backlog drains in ~`targetLatencyMs`, so render lag stays pinned near the\n * target regardless of how fast the provider streams. Token-sized arrivals\n * never exceed the minimum piece, matching the legacy fixed-size splitter.\n */\nexport function computeAdaptivePieceSize(\n bufferedTextLength: number,\n tickMs: number,\n targetLatencyMs: number = SMOOTH_TARGET_LATENCY_MS\n): number {\n if (bufferedTextLength <= 0) {\n return STREAM_CHUNK_MIN_SIZE;\n }\n if (tickMs <= 0 || targetLatencyMs <= 0) {\n return bufferedTextLength;\n }\n return Math.max(\n STREAM_CHUNK_MIN_SIZE,\n Math.ceil((bufferedTextLength * tickMs) / targetLatencyMs)\n );\n}\n\n/**\n * A cadence, not an additive sleep: time the consumer already spent since the\n * last visible emission counts against the target delay, so slow downstream\n * handlers never compound latency.\n */\nexport function getCadencedStreamDelay({\n targetDelay,\n lastVisibleTextAt,\n now,\n}: {\n targetDelay: number;\n lastVisibleTextAt?: number;\n now: number;\n}): number {\n if (targetDelay <= 0 || lastVisibleTextAt == null) {\n return 0;\n }\n return Math.max(0, targetDelay - (now - lastVisibleTextAt));\n}\n\n/** Abort-aware sleep that resolves (never rejects) on abort; callers re-check the signal. */\nexport async function waitForStreamDelay(\n delay: number,\n signal?: AbortSignal\n): Promise<void> {\n if (delay <= 0 || isSignalAborted(signal)) {\n return;\n }\n await new Promise<void>((resolve) => {\n const timeoutRef: { current?: ReturnType<typeof setTimeout> } = {};\n const onAbort = (): void => {\n if (timeoutRef.current) {\n clearTimeout(timeoutRef.current);\n }\n signal?.removeEventListener('abort', onAbort);\n resolve();\n };\n timeoutRef.current = setTimeout(() => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n }, delay);\n signal?.addEventListener('abort', onAbort, { once: true });\n if (isSignalAborted(signal)) {\n onAbort();\n }\n });\n}\n\nexport type SmoothPiece = {\n text: string;\n isFirst: boolean;\n isLast: boolean;\n};\n\n/**\n * One classified unit of provider stream output.\n *\n * - `smooth: true` — visible text, paced at the configured cadence and (unless\n * `atomic`) sliced adaptively at dequeue time.\n * - `atomic: true` — paced as a single piece, never split (text-bearing chunks\n * whose metadata cannot survive slicing, e.g. logprobs / finish_reason).\n * - `smooth: false` — passthrough: tool-call deltas, usage-only, id-only and\n * seal chunks. Zero delay, strict FIFO with the text around them.\n *\n * `emit` builds the provider-specific output for one piece; `isFirst` lets\n * providers keep usage_metadata on only the first piece of a split.\n */\nexport type SmoothItem<TEmit> = {\n text: string;\n smooth: boolean;\n atomic?: boolean;\n emit: (piece: SmoothPiece) => TEmit;\n};\n\ntype ProducerState = {\n done: boolean;\n failed: boolean;\n error?: unknown;\n};\n\ntype QueuedSmoothItem<TEmit> = {\n item: SmoothItem<TEmit>;\n textLength: number;\n};\n\n/**\n * Bounded producer/consumer smoothing engine.\n *\n * The producer drains `source` eagerly into a bounded queue (the buffer is the\n * backlog measurement adaptive sizing needs); at capacity it parks, applying\n * backpressure to the underlying stream. The consumer emits paced pieces,\n * decrementing the text budget and waking the producer *before* each cadenced\n * sleep so the provider stream keeps being read during pacing.\n *\n * `delayMs <= 0` disables smoothing entirely: every item passes through FIFO,\n * unsplit and undelayed.\n */\nexport async function* smoothStream<TEmit>({\n source,\n delayMs,\n signal,\n abortUpstream,\n}: {\n source: AsyncIterable<SmoothItem<TEmit>>;\n delayMs: number;\n signal?: AbortSignal;\n abortUpstream?: () => void;\n}): AsyncGenerator<TEmit> {\n if (!(delayMs > 0)) {\n /** Disabled smoothing preserves fully lazy streaming: no background\n * producer, no read-ahead — each provider chunk is pulled only when the\n * consumer asks, exactly like the pre-engine pass-through paths. */\n for await (const item of source) {\n if (isSignalAborted(signal)) {\n abortUpstream?.();\n throw new Error(STREAM_ABORT_MESSAGE);\n }\n yield item.emit({ text: item.text, isFirst: true, isLast: true });\n }\n return;\n }\n\n const queuedItems: QueuedSmoothItem<TEmit>[] = [];\n const producerState: ProducerState = { done: false, failed: false };\n let queuedItemIndex = 0;\n let bufferedTextLength = 0;\n let consumerClosed = false;\n let notifyConsumer: (() => void) | undefined;\n let notifyProducer: (() => void) | undefined;\n\n const notifyConsumerForItem = (): void => {\n notifyConsumer?.();\n notifyConsumer = undefined;\n };\n\n const notifyProducerForSpace = (): void => {\n notifyProducer?.();\n notifyProducer = undefined;\n };\n\n const hasQueuedItems = (): boolean => queuedItemIndex < queuedItems.length;\n\n const getQueuedItemCount = (): number =>\n queuedItems.length - queuedItemIndex;\n\n const isQueueAtCapacity = (): boolean =>\n getQueuedItemCount() >= MAX_STREAM_QUEUE_CHUNKS ||\n bufferedTextLength >= MAX_STREAM_QUEUE_TEXT_CHARS;\n\n /** Abort-aware: a consumer parked on an empty queue must wake when the\n * signal fires even if the provider stream never honors the abort — the\n * loop's top-of-iteration check then throws the canonical error. */\n const waitForNextItem = async (): Promise<void> => {\n if (\n hasQueuedItems() ||\n producerState.done ||\n producerState.failed ||\n isSignalAborted(signal)\n ) {\n return;\n }\n await new Promise<void>((resolve) => {\n const onAbort = (): void => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n };\n notifyConsumer = (): void => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n };\n signal?.addEventListener('abort', onAbort, { once: true });\n if (isSignalAborted(signal)) {\n onAbort();\n }\n });\n };\n\n const waitForQueueSpace = async (): Promise<void> => {\n while (\n isQueueAtCapacity() &&\n !consumerClosed &&\n !isSignalAborted(signal)\n ) {\n await new Promise<void>((resolve) => {\n const onAbort = (): void => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n };\n const onSpace = (): void => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n };\n notifyProducer = onSpace;\n signal?.addEventListener('abort', onAbort, { once: true });\n if (isSignalAborted(signal)) {\n onAbort();\n }\n });\n }\n };\n\n const dequeue = (): QueuedSmoothItem<TEmit> | undefined => {\n if (!hasQueuedItems()) {\n return undefined;\n }\n const queuedItem = queuedItems[queuedItemIndex];\n queuedItemIndex++;\n if (queuedItemIndex > 128 && queuedItemIndex * 2 >= queuedItems.length) {\n queuedItems.splice(0, queuedItemIndex);\n queuedItemIndex = 0;\n }\n return queuedItem;\n };\n\n const throwAborted = (): never => {\n abortUpstream?.();\n throw new Error(STREAM_ABORT_MESSAGE);\n };\n\n const enqueue = async (item: SmoothItem<TEmit>): Promise<void> => {\n await waitForQueueSpace();\n if (consumerClosed || isSignalAborted(signal)) {\n throwAborted();\n }\n const textLength = item.smooth ? item.text.length : 0;\n queuedItems.push({ item, textLength });\n bufferedTextLength += textLength;\n notifyConsumerForItem();\n };\n\n /**\n * Oversized splittable items are segmented at admission so a single giant\n * provider chunk cannot blow past the text budget: each segment re-checks\n * capacity, so the producer parks mid-chunk once the buffer fills — the\n * same bound the legacy split-before-enqueue queues enforced. The wrapped\n * emit maps segment-local pieces back to chunk-global isFirst/isLast so\n * provider clone contracts are unaffected.\n */\n const enqueueSegmented = async (item: SmoothItem<TEmit>): Promise<void> => {\n if (\n !item.smooth ||\n item.atomic === true ||\n item.text.length <= MAX_SMOOTH_ITEM_SEGMENT_CHARS\n ) {\n await enqueue(item);\n return;\n }\n\n const segments: { start: number; end: number }[] = [];\n let offset = 0;\n while (offset < item.text.length) {\n const end =\n offset +\n findStreamChunkBoundary(\n item.text.slice(offset),\n MAX_SMOOTH_ITEM_SEGMENT_CHARS\n );\n segments.push({ start: offset, end });\n offset = end;\n }\n\n for (let i = 0; i < segments.length; i++) {\n const isFirstSegment = i === 0;\n const isLastSegment = i === segments.length - 1;\n await enqueue({\n text: item.text.slice(segments[i].start, segments[i].end),\n smooth: true,\n emit: (piece) =>\n item.emit({\n text: piece.text,\n isFirst: isFirstSegment && piece.isFirst,\n isLast: isLastSegment && piece.isLast,\n }),\n });\n }\n };\n\n const producer = (async (): Promise<void> => {\n try {\n for await (const item of source) {\n if (isSignalAborted(signal)) {\n throwAborted();\n }\n await enqueueSegmented(item);\n }\n } catch (error) {\n producerState.failed = true;\n producerState.error = error;\n } finally {\n producerState.done = true;\n notifyConsumerForItem();\n }\n })();\n\n let hasEmittedText = false;\n let lastVisibleTextAt: number | undefined;\n let drainTicksRemaining: number | undefined;\n let current: QueuedSmoothItem<TEmit> | undefined;\n let headOffset = 0;\n let keepStreaming = true;\n try {\n while (keepStreaming) {\n if (isSignalAborted(signal)) {\n throwAborted();\n }\n\n if (current == null) {\n await waitForNextItem();\n current = dequeue();\n headOffset = 0;\n }\n\n if (current == null) {\n if (producerState.failed) {\n throw producerState.error ?? new Error(STREAM_PRODUCER_FAILURE);\n }\n if (producerState.done) {\n keepStreaming = false;\n }\n continue;\n }\n\n const { item } = current;\n\n if (!item.smooth) {\n notifyProducerForSpace();\n current = undefined;\n yield item.emit({ text: item.text, isFirst: true, isLast: true });\n continue;\n }\n\n if (item.text === '') {\n bufferedTextLength = Math.max(\n 0,\n bufferedTextLength - current.textLength\n );\n notifyProducerForSpace();\n current = undefined;\n continue;\n }\n\n /** Once the producer is done the backlog is final: drain it linearly\n * across the remaining target window instead of letting the\n * proportional formula decay geometrically and stretch the tail. */\n if (producerState.done && drainTicksRemaining == null) {\n drainTicksRemaining = Math.max(\n 1,\n Math.floor(SMOOTH_TARGET_LATENCY_MS / delayMs)\n );\n }\n const tickBudget =\n drainTicksRemaining != null\n ? Math.max(\n STREAM_CHUNK_MIN_SIZE,\n Math.ceil(bufferedTextLength / drainTicksRemaining)\n )\n : computeAdaptivePieceSize(bufferedTextLength, delayMs);\n if (drainTicksRemaining != null && drainTicksRemaining > 1) {\n drainTicksRemaining -= 1;\n }\n\n await waitForStreamDelay(\n getCadencedStreamDelay({\n targetDelay: hasEmittedText ? delayMs : 0,\n lastVisibleTextAt,\n now: Date.now(),\n }),\n signal\n );\n if (isSignalAborted(signal)) {\n throwAborted();\n }\n hasEmittedText = true;\n lastVisibleTextAt = Date.now();\n\n if (item.atomic === true) {\n bufferedTextLength = Math.max(\n 0,\n bufferedTextLength - current.textLength\n );\n notifyProducerForSpace();\n current = undefined;\n yield item.emit({ text: item.text, isFirst: true, isLast: true });\n continue;\n }\n\n /** One cadence tick drains up to the adaptive budget ACROSS queued\n * items, so token-sized provider deltas coalesce instead of costing a\n * full tick each; passthrough items flush free mid-batch (FIFO), and\n * atomic items end the batch to take their own tick. */\n let consumed = 0;\n while (consumed < tickBudget) {\n if (isSignalAborted(signal)) {\n throwAborted();\n }\n if (current == null) {\n if (!hasQueuedItems()) {\n break;\n }\n current = dequeue();\n headOffset = 0;\n if (current == null) {\n break;\n }\n }\n\n const batchItem = current.item;\n if (!batchItem.smooth) {\n notifyProducerForSpace();\n current = undefined;\n yield batchItem.emit({\n text: batchItem.text,\n isFirst: true,\n isLast: true,\n });\n continue;\n }\n if (batchItem.text === '') {\n bufferedTextLength = Math.max(\n 0,\n bufferedTextLength - current.textLength\n );\n notifyProducerForSpace();\n current = undefined;\n continue;\n }\n if (batchItem.atomic === true) {\n break;\n }\n\n const remainingText = batchItem.text.slice(headOffset);\n const pieceLength = findStreamChunkBoundary(\n remainingText,\n tickBudget - consumed\n );\n const pieceEnd = headOffset + pieceLength;\n const piece = batchItem.text.slice(headOffset, pieceEnd);\n const isFirst = headOffset === 0;\n const isLast = pieceEnd === batchItem.text.length;\n\n bufferedTextLength = Math.max(0, bufferedTextLength - piece.length);\n notifyProducerForSpace();\n if (isLast) {\n current = undefined;\n } else {\n headOffset = pieceEnd;\n }\n consumed += piece.length;\n yield batchItem.emit({ text: piece, isFirst, isLast });\n }\n }\n } finally {\n consumerClosed = true;\n if (producerState.done) {\n await producer;\n } else {\n abortUpstream?.();\n notifyProducerForSpace();\n const iterator = source as Partial<AsyncGenerator<SmoothItem<TEmit>>>;\n const closing = iterator.return?.call(source, undefined as never);\n if (closing != null) {\n void closing.then(\n () => undefined,\n () => undefined\n );\n }\n await Promise.race([\n producer,\n new Promise<void>((resolve) => {\n const timeout = setTimeout(resolve, PRODUCER_CLOSE_GRACE_MS);\n timeout.unref();\n }),\n ]);\n }\n }\n}\n"],"mappings":";AAAA,MAAa,uBAAuB;AAIpC,MAAa,gCAAgC;AAE7C,MAAa,oBAAyC,IAAI,IAAI;CAC5D;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAa,uBAAuB;;;;;;;;AAUpC,MAAa,0BAA0B;;;;;;;AAQvC,SAAgB,mBAAmB,OAAwB;CACzD,IAAI,SAAS,QAAQ,CAAC,OAAO,SAAS,KAAK,GACzC,OAAA;CAEF,OAAO,KAAK,IAAI,GAAG,KAAK;AAC1B;AAEA,SAAgB,gBAAgB,QAA+B;CAC7D,OAAO,QAAQ,YAAY;AAC7B;AAWA,SAAgB,wBACd,MACA,SACQ;CACR,IAAI,WAAW,KAAK,QAClB,OAAO,KAAK;CAGd,MAAM,UAAU,KAAK,IACnB,KAAK,QACL,UAAA,EACF;CACA,KAAK,IAAI,WAAW,SAAS,WAAW,SAAS,YAC/C,IAAI,kBAAkB,IAAI,KAAK,SAAS,GACtC,OAAO,WAAW;CAItB,OAAO;AACT;;;;;;;AAQA,SAAgB,yBACd,oBACA,QACA,kBAAA,KACQ;CACR,IAAI,sBAAsB,GACxB,OAAA;CAEF,IAAI,UAAU,KAAK,mBAAmB,GACpC,OAAO;CAET,OAAO,KAAK,IAAA,GAEV,KAAK,KAAM,qBAAqB,SAAU,eAAe,CAC3D;AACF;;;;;;AAOA,SAAgB,uBAAuB,EACrC,aACA,mBACA,OAKS;CACT,IAAI,eAAe,KAAK,qBAAqB,MAC3C,OAAO;CAET,OAAO,KAAK,IAAI,GAAG,eAAe,MAAM,kBAAkB;AAC5D;;AAGA,eAAsB,mBACpB,OACA,QACe;CACf,IAAI,SAAS,KAAK,gBAAgB,MAAM,GACtC;CAEF,MAAM,IAAI,SAAe,YAAY;EACnC,MAAM,aAA0D,CAAC;EACjE,MAAM,gBAAsB;GAC1B,IAAI,WAAW,SACb,aAAa,WAAW,OAAO;GAEjC,QAAQ,oBAAoB,SAAS,OAAO;GAC5C,QAAQ;EACV;EACA,WAAW,UAAU,iBAAiB;GACpC,QAAQ,oBAAoB,SAAS,OAAO;GAC5C,QAAQ;EACV,GAAG,KAAK;EACR,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;EACzD,IAAI,gBAAgB,MAAM,GACxB,QAAQ;CAEZ,CAAC;AACH;;;;;;;;;;;;;AAmDA,gBAAuB,aAAoB,EACzC,QACA,SACA,QACA,iBAMwB;CACxB,IAAI,EAAE,UAAU,IAAI;;;;EAIlB,WAAW,MAAM,QAAQ,QAAQ;GAC/B,IAAI,gBAAgB,MAAM,GAAG;IAC3B,gBAAgB;IAChB,MAAM,IAAI,MAAM,oBAAoB;GACtC;GACA,MAAM,KAAK,KAAK;IAAE,MAAM,KAAK;IAAM,SAAS;IAAM,QAAQ;GAAK,CAAC;EAClE;EACA;CACF;CAEA,MAAM,cAAyC,CAAC;CAChD,MAAM,gBAA+B;EAAE,MAAM;EAAO,QAAQ;CAAM;CAClE,IAAI,kBAAkB;CACtB,IAAI,qBAAqB;CACzB,IAAI,iBAAiB;CACrB,IAAI;CACJ,IAAI;CAEJ,MAAM,8BAAoC;EACxC,iBAAiB;EACjB,iBAAiB,KAAA;CACnB;CAEA,MAAM,+BAAqC;EACzC,iBAAiB;EACjB,iBAAiB,KAAA;CACnB;CAEA,MAAM,uBAAgC,kBAAkB,YAAY;CAEpE,MAAM,2BACJ,YAAY,SAAS;CAEvB,MAAM,0BACJ,mBAAmB,KAAA,OACnB,sBAAA;;;;CAKF,MAAM,kBAAkB,YAA2B;EACjD,IACE,eAAe,KACf,cAAc,QACd,cAAc,UACd,gBAAgB,MAAM,GAEtB;EAEF,MAAM,IAAI,SAAe,YAAY;GACnC,MAAM,gBAAsB;IAC1B,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ;GACV;GACA,uBAA6B;IAC3B,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ;GACV;GACA,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;GACzD,IAAI,gBAAgB,MAAM,GACxB,QAAQ;EAEZ,CAAC;CACH;CAEA,MAAM,oBAAoB,YAA2B;EACnD,OACE,kBAAkB,KAClB,CAAC,kBACD,CAAC,gBAAgB,MAAM,GAEvB,MAAM,IAAI,SAAe,YAAY;GACnC,MAAM,gBAAsB;IAC1B,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ;GACV;GACA,MAAM,gBAAsB;IAC1B,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ;GACV;GACA,iBAAiB;GACjB,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;GACzD,IAAI,gBAAgB,MAAM,GACxB,QAAQ;EAEZ,CAAC;CAEL;CAEA,MAAM,gBAAqD;EACzD,IAAI,CAAC,eAAe,GAClB;EAEF,MAAM,aAAa,YAAY;EAC/B;EACA,IAAI,kBAAkB,OAAO,kBAAkB,KAAK,YAAY,QAAQ;GACtE,YAAY,OAAO,GAAG,eAAe;GACrC,kBAAkB;EACpB;EACA,OAAO;CACT;CAEA,MAAM,qBAA4B;EAChC,gBAAgB;EAChB,MAAM,IAAI,MAAM,oBAAoB;CACtC;CAEA,MAAM,UAAU,OAAO,SAA2C;EAChE,MAAM,kBAAkB;EACxB,IAAI,kBAAkB,gBAAgB,MAAM,GAC1C,aAAa;EAEf,MAAM,aAAa,KAAK,SAAS,KAAK,KAAK,SAAS;EACpD,YAAY,KAAK;GAAE;GAAM;EAAW,CAAC;EACrC,sBAAsB;EACtB,sBAAsB;CACxB;;;;;;;;;CAUA,MAAM,mBAAmB,OAAO,SAA2C;EACzE,IACE,CAAC,KAAK,UACN,KAAK,WAAW,QAChB,KAAK,KAAK,UAAA,MACV;GACA,MAAM,QAAQ,IAAI;GAClB;EACF;EAEA,MAAM,WAA6C,CAAC;EACpD,IAAI,SAAS;EACb,OAAO,SAAS,KAAK,KAAK,QAAQ;GAChC,MAAM,MACJ,SACA,wBACE,KAAK,KAAK,MAAM,MAAM,GACtB,6BACF;GACF,SAAS,KAAK;IAAE,OAAO;IAAQ;GAAI,CAAC;GACpC,SAAS;EACX;EAEA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;GACxC,MAAM,iBAAiB,MAAM;GAC7B,MAAM,gBAAgB,MAAM,SAAS,SAAS;GAC9C,MAAM,QAAQ;IACZ,MAAM,KAAK,KAAK,MAAM,SAAS,EAAE,CAAC,OAAO,SAAS,EAAE,CAAC,GAAG;IACxD,QAAQ;IACR,OAAO,UACL,KAAK,KAAK;KACR,MAAM,MAAM;KACZ,SAAS,kBAAkB,MAAM;KACjC,QAAQ,iBAAiB,MAAM;IACjC,CAAC;GACL,CAAC;EACH;CACF;CAEA,MAAM,YAAY,YAA2B;EAC3C,IAAI;GACF,WAAW,MAAM,QAAQ,QAAQ;IAC/B,IAAI,gBAAgB,MAAM,GACxB,aAAa;IAEf,MAAM,iBAAiB,IAAI;GAC7B;EACF,SAAS,OAAO;GACd,cAAc,SAAS;GACvB,cAAc,QAAQ;EACxB,UAAU;GACR,cAAc,OAAO;GACrB,sBAAsB;EACxB;CACF,EAAA,CAAG;CAEH,IAAI,iBAAiB;CACrB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI,aAAa;CACjB,IAAI,gBAAgB;CACpB,IAAI;EACF,OAAO,eAAe;GACpB,IAAI,gBAAgB,MAAM,GACxB,aAAa;GAGf,IAAI,WAAW,MAAM;IACnB,MAAM,gBAAgB;IACtB,UAAU,QAAQ;IAClB,aAAa;GACf;GAEA,IAAI,WAAW,MAAM;IACnB,IAAI,cAAc,QAChB,MAAM,cAAc,yBAAS,IAAI,MAAA,yBAA6B;IAEhE,IAAI,cAAc,MAChB,gBAAgB;IAElB;GACF;GAEA,MAAM,EAAE,SAAS;GAEjB,IAAI,CAAC,KAAK,QAAQ;IAChB,uBAAuB;IACvB,UAAU,KAAA;IACV,MAAM,KAAK,KAAK;KAAE,MAAM,KAAK;KAAM,SAAS;KAAM,QAAQ;IAAK,CAAC;IAChE;GACF;GAEA,IAAI,KAAK,SAAS,IAAI;IACpB,qBAAqB,KAAK,IACxB,GACA,qBAAqB,QAAQ,UAC/B;IACA,uBAAuB;IACvB,UAAU,KAAA;IACV;GACF;;;;GAKA,IAAI,cAAc,QAAQ,uBAAuB,MAC/C,sBAAsB,KAAK,IACzB,GACA,KAAK,MAAA,MAAiC,OAAO,CAC/C;GAEF,MAAM,aACJ,uBAAuB,OACnB,KAAK,IAAA,GAEL,KAAK,KAAK,qBAAqB,mBAAmB,CACpD,IACE,yBAAyB,oBAAoB,OAAO;GAC1D,IAAI,uBAAuB,QAAQ,sBAAsB,GACvD,uBAAuB;GAGzB,MAAM,mBACJ,uBAAuB;IACrB,aAAa,iBAAiB,UAAU;IACxC;IACA,KAAK,KAAK,IAAI;GAChB,CAAC,GACD,MACF;GACA,IAAI,gBAAgB,MAAM,GACxB,aAAa;GAEf,iBAAiB;GACjB,oBAAoB,KAAK,IAAI;GAE7B,IAAI,KAAK,WAAW,MAAM;IACxB,qBAAqB,KAAK,IACxB,GACA,qBAAqB,QAAQ,UAC/B;IACA,uBAAuB;IACvB,UAAU,KAAA;IACV,MAAM,KAAK,KAAK;KAAE,MAAM,KAAK;KAAM,SAAS;KAAM,QAAQ;IAAK,CAAC;IAChE;GACF;;;;;GAMA,IAAI,WAAW;GACf,OAAO,WAAW,YAAY;IAC5B,IAAI,gBAAgB,MAAM,GACxB,aAAa;IAEf,IAAI,WAAW,MAAM;KACnB,IAAI,CAAC,eAAe,GAClB;KAEF,UAAU,QAAQ;KAClB,aAAa;KACb,IAAI,WAAW,MACb;IAEJ;IAEA,MAAM,YAAY,QAAQ;IAC1B,IAAI,CAAC,UAAU,QAAQ;KACrB,uBAAuB;KACvB,UAAU,KAAA;KACV,MAAM,UAAU,KAAK;MACnB,MAAM,UAAU;MAChB,SAAS;MACT,QAAQ;KACV,CAAC;KACD;IACF;IACA,IAAI,UAAU,SAAS,IAAI;KACzB,qBAAqB,KAAK,IACxB,GACA,qBAAqB,QAAQ,UAC/B;KACA,uBAAuB;KACvB,UAAU,KAAA;KACV;IACF;IACA,IAAI,UAAU,WAAW,MACvB;IAIF,MAAM,cAAc,wBADE,UAAU,KAAK,MAAM,UAE7B,GACZ,aAAa,QACf;IACA,MAAM,WAAW,aAAa;IAC9B,MAAM,QAAQ,UAAU,KAAK,MAAM,YAAY,QAAQ;IACvD,MAAM,UAAU,eAAe;IAC/B,MAAM,SAAS,aAAa,UAAU,KAAK;IAE3C,qBAAqB,KAAK,IAAI,GAAG,qBAAqB,MAAM,MAAM;IAClE,uBAAuB;IACvB,IAAI,QACF,UAAU,KAAA;SAEV,aAAa;IAEf,YAAY,MAAM;IAClB,MAAM,UAAU,KAAK;KAAE,MAAM;KAAO;KAAS;IAAO,CAAC;GACvD;EACF;CACF,UAAU;EACR,iBAAiB;EACjB,IAAI,cAAc,MAChB,MAAM;OACD;GACL,gBAAgB;GAChB,uBAAuB;GAEvB,MAAM,UAAUA,OAAS,QAAQ,KAAK,QAAQ,KAAA,CAAkB;GAChE,IAAI,WAAW,MACb,QAAa,WACL,KAAA,SACA,KAAA,CACR;GAEF,MAAM,QAAQ,KAAK,CACjB,UACA,IAAI,SAAe,YAAY;IAE7B,WAD2B,SAAS,uBAC9B,CAAC,CAAC,MAAM;GAChB,CAAC,CACH,CAAC;EACH;CACF;AACF"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
const require_streamedToolCallSeals = require("../../tools/streamedToolCallSeals.cjs");
|
|
2
|
+
const require_smoother = require("../stream/smoother.cjs");
|
|
3
|
+
const require_chunkAdapters = require("../stream/chunkAdapters.cjs");
|
|
2
4
|
let _langchain_core_messages = require("@langchain/core/messages");
|
|
3
5
|
let _langchain_google_gauth = require("@langchain/google-gauth");
|
|
4
6
|
let _langchain_google_common = require("@langchain/google-common");
|
|
@@ -396,6 +398,7 @@ var ChatVertexAI = class extends _langchain_google_gauth.ChatGoogle {
|
|
|
396
398
|
"chat_models",
|
|
397
399
|
"vertexai"
|
|
398
400
|
];
|
|
401
|
+
_lc_stream_delay;
|
|
399
402
|
dynamicThinkingBudget = false;
|
|
400
403
|
thinkingConfig;
|
|
401
404
|
static lc_name() {
|
|
@@ -413,6 +416,7 @@ var ChatVertexAI = class extends _langchain_google_gauth.ChatGoogle {
|
|
|
413
416
|
});
|
|
414
417
|
this.dynamicThinkingBudget = dynamicThinkingBudget;
|
|
415
418
|
this.thinkingConfig = fields?.thinkingConfig;
|
|
419
|
+
this._lc_stream_delay = require_smoother.resolveStreamDelay(fields?._lc_stream_delay);
|
|
416
420
|
}
|
|
417
421
|
invocationParams(options) {
|
|
418
422
|
const params = super.invocationParams(options);
|
|
@@ -420,8 +424,16 @@ var ChatVertexAI = class extends _langchain_google_gauth.ChatGoogle {
|
|
|
420
424
|
return params;
|
|
421
425
|
}
|
|
422
426
|
async *_streamResponseChunks(messages, options, runManager) {
|
|
427
|
+
yield* require_chunkAdapters.smoothGenerationChunks({
|
|
428
|
+
chunks: this._streamRepairedChunks(messages, options),
|
|
429
|
+
delayMs: this._lc_stream_delay,
|
|
430
|
+
signal: options.signal,
|
|
431
|
+
runManager
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
async *_streamRepairedChunks(messages, options) {
|
|
423
435
|
let lastGoodUsage;
|
|
424
|
-
for await (const chunk of super._streamResponseChunks(messages, options,
|
|
436
|
+
for await (const chunk of super._streamResponseChunks(messages, options, void 0)) {
|
|
425
437
|
const genUsage = chunk.generationInfo?.usage_metadata;
|
|
426
438
|
if (genUsage) lastGoodUsage = genUsage;
|
|
427
439
|
if (chunk.message instanceof _langchain_core_messages.AIMessageChunk) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.cjs","names":["STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY","GOOGLE_STREAMED_TOOL_CALL_ADAPTER","STREAMED_TOOL_CALL_SEAL_METADATA_KEY","isAIMessage","ChatConnection","ChatGoogle","AIMessageChunk"],"sources":["../../../../src/llm/vertexai/index.ts"],"sourcesContent":["import { ChatGoogle } from '@langchain/google-gauth';\nimport { ChatConnection } from '@langchain/google-common';\nimport { AIMessageChunk, isAIMessage } from '@langchain/core/messages';\nimport type {\n GeminiContent,\n GeminiRequest,\n GoogleAIModelRequestParams,\n GoogleAbstractedClient,\n} from '@langchain/google-common';\nimport type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';\nimport type { BaseMessage, UsageMetadata } from '@langchain/core/messages';\nimport type { ChatGenerationChunk } from '@langchain/core/outputs';\nimport type { GoogleThinkingConfig, VertexAIClientOptions } from '@/types';\nimport {\n STREAMED_TOOL_CALL_SEAL_METADATA_KEY,\n STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,\n GOOGLE_STREAMED_TOOL_CALL_ADAPTER,\n} from '@/tools/streamedToolCallSeals';\n\n/**\n * `@langchain/google-common`'s `_streamResponseChunks` emits usage on TWO\n * different paths within the same stream:\n *\n * - Streaming chunks set `chunk.generationInfo.usage_metadata` via\n * `responseToUsageMetadata`, which correctly sums\n * `candidatesTokenCount + thoughtsTokenCount` and includes\n * `output_token_details.reasoning`.\n * - The trailing fallback chunk (emitted after the API stream exhausts)\n * attaches its own `chunk.message.usage_metadata` built inline as\n * `output_tokens = candidatesTokenCount` only — dropping\n * `thoughtsTokenCount` and `output_token_details` entirely.\n *\n * After `AIMessageChunk.concat`, only `message.usage_metadata` survives —\n * which is the buggy fallback value. This breaks the documented\n * `total_tokens === input_tokens + output_tokens` invariant and silently\n * undercharges thinking models for reasoning tokens.\n *\n * The repair: track the last `generationInfo.usage_metadata` we see, and\n * when the fallback chunk arrives with its buggy `message.usage_metadata`,\n * replace it with the tracked good value. `CustomChatGoogleGenerativeAI`\n * solves the same problem for the Google API path differently — by\n * overriding `_convertToUsageMetadata`.\n */\nexport function repairStreamUsageMetadata(\n current: UsageMetadata | undefined,\n generationInfoUsage: UsageMetadata | undefined\n): UsageMetadata | undefined {\n if (!current) return current;\n if (!generationInfoUsage) return current;\n if (generationInfoUsage.total_tokens !== current.total_tokens) return current;\n if (generationInfoUsage.output_tokens <= current.output_tokens)\n return current;\n return generationInfoUsage;\n}\n\n/**\n * The Gemini API delivers function calls as complete objects — never as\n * partial arg deltas. `@langchain/google-common` pre-parses each streamed\n * functionCall part into `tool_calls` (invalid args land in\n * `invalid_tool_calls` instead), so a chunk whose tool-call chunks all parsed\n * cleanly is sealed on arrival for eager tool execution. Anything that fails\n * the parse check is left unstamped and falls back to the lazy path.\n */\nexport function sealCompleteStreamedToolCalls(message: AIMessageChunk): void {\n const chunkCount = message.tool_call_chunks?.length ?? 0;\n if (\n chunkCount === 0 ||\n (message.invalid_tool_calls?.length ?? 0) > 0 ||\n (message.tool_calls?.length ?? 0) !== chunkCount\n ) {\n return;\n }\n message.response_metadata = {\n ...message.response_metadata,\n [STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:\n GOOGLE_STREAMED_TOOL_CALL_ADAPTER,\n [STREAMED_TOOL_CALL_SEAL_METADATA_KEY]: { kind: 'all' },\n };\n}\n\ntype AdditionalKwargs =\n | undefined\n | (BaseMessage['additional_kwargs'] & {\n signatures?: Array<string | undefined>;\n });\n\n/**\n * Fixes thought signatures on functionCall parts in the formatted Gemini request.\n *\n * `@langchain/google-common` stores signatures as a flat array in\n * `additional_kwargs.signatures` (one per response part) and re-attaches them\n * by index only when `signatures.length === parts.length`. This fails when:\n * - The API omits a signature (length mismatch)\n * - Streaming chunks merge with different part counts\n * - The signature for a functionCall part is an empty string\n *\n * This function correlates each \"model\" content block in the formatted request\n * back to its originating AI message by *position*, then re-attaches non-empty\n * signatures that the library failed to apply. AI messages without signatures\n * still consume their slot — filtering them out shifted later messages onto\n * the wrong content block and dropped real signatures on the floor.\n */\nexport function fixThoughtSignatures(\n contents: GeminiContent[],\n input: BaseMessage[]\n): void {\n // All AI messages, in order — non-signature ones still consume positional\n // slots so later messages line up with their model content blocks.\n const aiMessages = input.filter(isAIMessage);\n const modelContents = contents.filter((c) => c.role === 'model');\n\n const count = Math.min(aiMessages.length, modelContents.length);\n for (let i = 0; i < count; i++) {\n const signatures = (aiMessages[i].additional_kwargs as AdditionalKwargs)\n ?.signatures;\n if (!Array.isArray(signatures) || signatures.length === 0) continue;\n\n const content = modelContents[i];\n const attachedSignatures = new Set(\n content.parts\n .map((p) => p.thoughtSignature)\n .filter((s): s is string => s != null && s !== '')\n );\n const availableSignatures = signatures.filter(\n (s): s is string => s != null && s !== '' && !attachedSignatures.has(s)\n );\n\n let sigIdx = 0;\n for (const part of content.parts) {\n if (\n 'functionCall' in part &&\n (part.thoughtSignature == null || part.thoughtSignature === '') &&\n sigIdx < availableSignatures.length\n ) {\n part.thoughtSignature = availableSignatures[sigIdx];\n sigIdx++;\n }\n }\n }\n}\n\nclass CustomChatConnection extends ChatConnection<VertexAIClientOptions> {\n thinkingConfig?: GoogleThinkingConfig;\n\n async formatData(\n input: BaseMessage[],\n parameters: GoogleAIModelRequestParams\n ): Promise<unknown> {\n const formattedData = (await super.formatData(\n input,\n parameters\n )) as GeminiRequest;\n if (formattedData.generationConfig?.thinkingConfig?.thinkingBudget === -1) {\n // -1 means \"let the model decide\" - delete the property so the API doesn't receive an invalid value\n if (\n formattedData.generationConfig.thinkingConfig.includeThoughts === false\n ) {\n formattedData.generationConfig.thinkingConfig.includeThoughts = true;\n }\n delete formattedData.generationConfig.thinkingConfig.thinkingBudget;\n }\n if (this.thinkingConfig?.thinkingLevel != null) {\n formattedData.generationConfig ??= {};\n // thinkingLevel and thinkingBudget cannot coexist — the API rejects the request.\n // Remove thinkingBudget when thinkingLevel is set.\n const { thinkingBudget: _, ...existingThinkingConfig } =\n (formattedData.generationConfig.thinkingConfig as\n | Record<string, unknown>\n | undefined) ?? {};\n (\n formattedData.generationConfig as Record<string, unknown>\n ).thinkingConfig = {\n ...existingThinkingConfig,\n thinkingLevel: this.thinkingConfig.thinkingLevel,\n ...(this.thinkingConfig.includeThoughts != null && {\n includeThoughts: this.thinkingConfig.includeThoughts,\n }),\n };\n }\n if (formattedData.contents) {\n fixThoughtSignatures(formattedData.contents, input);\n // gemini-3.1+ models reject role=\"function\"; convert to role=\"user\"\n for (const content of formattedData.contents) {\n if (content.role === 'function') {\n (content as { role: string }).role = 'user';\n }\n }\n }\n return formattedData;\n }\n}\n\n/**\n * Integration with Google Vertex AI chat models.\n *\n * Setup:\n * Install `@langchain/google-vertexai` and set your stringified\n * Vertex AI credentials as an environment variable named `GOOGLE_APPLICATION_CREDENTIALS`.\n *\n * ```bash\n * npm install @langchain/google-vertexai\n * export GOOGLE_APPLICATION_CREDENTIALS=\"path/to/credentials\"\n * ```\n *\n * ## [Constructor args](https://api.js.langchain.com/classes/_langchain_google_vertexai.index.ChatVertexAI.html#constructor.new_ChatVertexAI)\n *\n * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_common_types.GoogleAIBaseLanguageModelCallOptions.html)\n *\n * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.\n * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:\n *\n * ```typescript\n * // When calling `.withConfig`, call options should be passed via the first argument\n * const llmWithArgsBound = llm.withConfig({\n * stop: [\"\\n\"],\n * tools: [...],\n * });\n *\n * // When calling `.bindTools`, call options should be passed via the second argument\n * const llmWithTools = llm.bindTools(\n * [...],\n * {\n * tool_choice: \"auto\",\n * }\n * );\n * ```\n *\n * ## Examples\n *\n * <details open>\n * <summary><strong>Instantiate</strong></summary>\n *\n * ```typescript\n * import { ChatVertexAI } from '@langchain/google-vertexai';\n *\n * const llm = new ChatVertexAI({\n * model: \"gemini-1.5-pro\",\n * temperature: 0,\n * // other params...\n * });\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Invoking</strong></summary>\n *\n * ```typescript\n * const input = `Translate \"I love programming\" into French.`;\n *\n * // Models also accept a list of chat messages or a formatted prompt\n * const result = await llm.invoke(input);\n * console.log(result);\n * ```\n *\n * ```txt\n * AIMessageChunk {\n * \"content\": \"\\\"J'adore programmer\\\" \\n\\nHere's why this is the best translation:\\n\\n* **J'adore** means \\\"I love\\\" and conveys a strong passion.\\n* **Programmer** is the French verb for \\\"to program.\\\"\\n\\nThis translation is natural and idiomatic in French. \\n\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": [],\n * \"usage_metadata\": {\n * \"input_tokens\": 9,\n * \"output_tokens\": 63,\n * \"total_tokens\": 72\n * }\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Streaming Chunks</strong></summary>\n *\n * ```typescript\n * for await (const chunk of await llm.stream(input)) {\n * console.log(chunk);\n * }\n * ```\n *\n * ```txt\n * AIMessageChunk {\n * \"content\": \"\\\"\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": []\n * }\n * AIMessageChunk {\n * \"content\": \"J'adore programmer\\\" \\n\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": []\n * }\n * AIMessageChunk {\n * \"content\": \"\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": []\n * }\n * AIMessageChunk {\n * \"content\": \"\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {\n * \"finishReason\": \"stop\"\n * },\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": [],\n * \"usage_metadata\": {\n * \"input_tokens\": 9,\n * \"output_tokens\": 8,\n * \"total_tokens\": 17\n * }\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Aggregate Streamed Chunks</strong></summary>\n *\n * ```typescript\n * import { AIMessageChunk } from '@langchain/core/messages';\n * import { concat } from '@langchain/core/utils/stream';\n *\n * const stream = await llm.stream(input);\n * let full: AIMessageChunk | undefined;\n * for await (const chunk of stream) {\n * full = !full ? chunk : concat(full, chunk);\n * }\n * console.log(full);\n * ```\n *\n * ```txt\n * AIMessageChunk {\n * \"content\": \"\\\"J'adore programmer\\\" \\n\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {\n * \"finishReason\": \"stop\"\n * },\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": [],\n * \"usage_metadata\": {\n * \"input_tokens\": 9,\n * \"output_tokens\": 8,\n * \"total_tokens\": 17\n * }\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Bind tools</strong></summary>\n *\n * ```typescript\n * import { z } from 'zod';\n *\n * const GetWeather = {\n * name: \"GetWeather\",\n * description: \"Get the current weather in a given location\",\n * schema: z.object({\n * location: z.string().describe(\"The city and state, e.g. San Francisco, CA\")\n * }),\n * }\n *\n * const GetPopulation = {\n * name: \"GetPopulation\",\n * description: \"Get the current population in a given location\",\n * schema: z.object({\n * location: z.string().describe(\"The city and state, e.g. San Francisco, CA\")\n * }),\n * }\n *\n * const llmWithTools = llm.bindTools([GetWeather, GetPopulation]);\n * const aiMsg = await llmWithTools.invoke(\n * \"Which city is hotter today and which is bigger: LA or NY?\"\n * );\n * console.log(aiMsg.tool_calls);\n * ```\n *\n * ```txt\n * [\n * {\n * name: 'GetPopulation',\n * args: { location: 'New York City, NY' },\n * id: '33c1c1f47e2f492799c77d2800a43912',\n * type: 'tool_call'\n * }\n * ]\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Structured Output</strong></summary>\n *\n * ```typescript\n * import { z } from 'zod';\n *\n * const Joke = z.object({\n * setup: z.string().describe(\"The setup of the joke\"),\n * punchline: z.string().describe(\"The punchline to the joke\"),\n * rating: z.number().optional().describe(\"How funny the joke is, from 1 to 10\")\n * }).describe('Joke to tell user.');\n *\n * const structuredLlm = llm.withStructuredOutput(Joke, { name: \"Joke\" });\n * const jokeResult = await structuredLlm.invoke(\"Tell me a joke about cats\");\n * console.log(jokeResult);\n * ```\n *\n * ```txt\n * {\n * setup: 'What do you call a cat that loves to bowl?',\n * punchline: 'An alley cat!'\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Usage Metadata</strong></summary>\n *\n * ```typescript\n * const aiMsgForMetadata = await llm.invoke(input);\n * console.log(aiMsgForMetadata.usage_metadata);\n * ```\n *\n * ```txt\n * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Stream Usage Metadata</strong></summary>\n *\n * ```typescript\n * const streamForMetadata = await llm.stream(\n * input,\n * {\n * streamUsage: true\n * }\n * );\n * let fullForMetadata: AIMessageChunk | undefined;\n * for await (const chunk of streamForMetadata) {\n * fullForMetadata = !fullForMetadata ? chunk : concat(fullForMetadata, chunk);\n * }\n * console.log(fullForMetadata?.usage_metadata);\n * ```\n *\n * ```txt\n * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }\n * ```\n * </details>\n *\n * <br />\n */\nexport class ChatVertexAI extends ChatGoogle {\n lc_namespace = ['langchain', 'chat_models', 'vertexai'];\n dynamicThinkingBudget = false;\n thinkingConfig?: GoogleThinkingConfig;\n\n static lc_name(): 'LibreChatVertexAI' {\n return 'LibreChatVertexAI';\n }\n\n constructor(model: string, fields?: Omit<VertexAIClientOptions, 'model'>);\n constructor(fields?: VertexAIClientOptions);\n constructor(\n modelOrFields?: string | VertexAIClientOptions,\n params?: Omit<VertexAIClientOptions, 'model'>\n ) {\n const fields =\n typeof modelOrFields === 'string'\n ? { ...(params ?? {}), model: modelOrFields }\n : modelOrFields;\n const dynamicThinkingBudget = fields?.thinkingBudget === -1;\n super({\n ...fields,\n platformType: 'gcp',\n });\n this.dynamicThinkingBudget = dynamicThinkingBudget;\n this.thinkingConfig = fields?.thinkingConfig;\n }\n invocationParams(\n options?: this['ParsedCallOptions'] | undefined\n ): GoogleAIModelRequestParams {\n const params = super.invocationParams(options);\n if (this.dynamicThinkingBudget) {\n params.maxReasoningTokens = -1;\n }\n return params;\n }\n async *_streamResponseChunks(\n messages: BaseMessage[],\n options: this['ParsedCallOptions'],\n runManager?: CallbackManagerForLLMRun\n ): AsyncGenerator<ChatGenerationChunk> {\n let lastGoodUsage: UsageMetadata | undefined;\n for await (const chunk of super._streamResponseChunks(\n messages,\n options,\n runManager\n )) {\n const genUsage = (\n chunk.generationInfo as { usage_metadata?: UsageMetadata } | undefined\n )?.usage_metadata;\n if (genUsage) {\n lastGoodUsage = genUsage;\n }\n if (chunk.message instanceof AIMessageChunk) {\n const repaired = repairStreamUsageMetadata(\n chunk.message.usage_metadata,\n lastGoodUsage\n );\n if (repaired !== chunk.message.usage_metadata) {\n chunk.message.usage_metadata = repaired;\n }\n sealCompleteStreamedToolCalls(chunk.message);\n }\n yield chunk;\n }\n }\n buildConnection(\n fields: VertexAIClientOptions | undefined,\n client: GoogleAbstractedClient\n ): void {\n // Note: buildConnection is called from super() BEFORE this.thinkingConfig is set,\n // so we must read thinkingConfig from `fields` directly.\n const thinkingConfig = fields?.thinkingConfig ?? this.thinkingConfig;\n\n const connection = new CustomChatConnection(\n { ...fields, ...this },\n this.caller,\n client,\n false\n );\n connection.thinkingConfig = thinkingConfig;\n this.connection = connection;\n\n const streamedConnection = new CustomChatConnection(\n { ...fields, ...this },\n this.caller,\n client,\n true\n );\n streamedConnection.thinkingConfig = thinkingConfig;\n this.streamedConnection = streamedConnection;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2CA,SAAgB,0BACd,SACA,qBAC2B;CAC3B,IAAI,CAAC,SAAS,OAAO;CACrB,IAAI,CAAC,qBAAqB,OAAO;CACjC,IAAI,oBAAoB,iBAAiB,QAAQ,cAAc,OAAO;CACtE,IAAI,oBAAoB,iBAAiB,QAAQ,eAC/C,OAAO;CACT,OAAO;AACT;;;;;;;;;AAUA,SAAgB,8BAA8B,SAA+B;CAC3E,MAAM,aAAa,QAAQ,kBAAkB,UAAU;CACvD,IACE,eAAe,MACd,QAAQ,oBAAoB,UAAU,KAAK,MAC3C,QAAQ,YAAY,UAAU,OAAO,YAEtC;CAEF,QAAQ,oBAAoB;EAC1B,GAAG,QAAQ;GACVA,8BAAAA,0CACCC,8BAAAA;GACDC,8BAAAA,uCAAuC,EAAE,MAAM,MAAM;CACxD;AACF;;;;;;;;;;;;;;;;;AAwBA,SAAgB,qBACd,UACA,OACM;CAGN,MAAM,aAAa,MAAM,OAAOC,yBAAAA,WAAW;CAC3C,MAAM,gBAAgB,SAAS,QAAQ,MAAM,EAAE,SAAS,OAAO;CAE/D,MAAM,QAAQ,KAAK,IAAI,WAAW,QAAQ,cAAc,MAAM;CAC9D,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;EAC9B,MAAM,aAAc,WAAW,EAAE,CAAC,mBAC9B;EACJ,IAAI,CAAC,MAAM,QAAQ,UAAU,KAAK,WAAW,WAAW,GAAG;EAE3D,MAAM,UAAU,cAAc;EAC9B,MAAM,qBAAqB,IAAI,IAC7B,QAAQ,MACL,KAAK,MAAM,EAAE,gBAAgB,CAAC,CAC9B,QAAQ,MAAmB,KAAK,QAAQ,MAAM,EAAE,CACrD;EACA,MAAM,sBAAsB,WAAW,QACpC,MAAmB,KAAK,QAAQ,MAAM,MAAM,CAAC,mBAAmB,IAAI,CAAC,CACxE;EAEA,IAAI,SAAS;EACb,KAAK,MAAM,QAAQ,QAAQ,OACzB,IACE,kBAAkB,SACjB,KAAK,oBAAoB,QAAQ,KAAK,qBAAqB,OAC5D,SAAS,oBAAoB,QAC7B;GACA,KAAK,mBAAmB,oBAAoB;GAC5C;EACF;CAEJ;AACF;AAEA,IAAM,uBAAN,cAAmCC,yBAAAA,eAAsC;CACvE;CAEA,MAAM,WACJ,OACA,YACkB;EAClB,MAAM,gBAAiB,MAAM,MAAM,WACjC,OACA,UACF;EACA,IAAI,cAAc,kBAAkB,gBAAgB,mBAAmB,IAAI;GAEzE,IACE,cAAc,iBAAiB,eAAe,oBAAoB,OAElE,cAAc,iBAAiB,eAAe,kBAAkB;GAElE,OAAO,cAAc,iBAAiB,eAAe;EACvD;EACA,IAAI,KAAK,gBAAgB,iBAAiB,MAAM;GAC9C,cAAc,qBAAqB,CAAC;GAGpC,MAAM,EAAE,gBAAgB,GAAG,GAAG,2BAC3B,cAAc,iBAAiB,kBAEd,CAAC;GACrB,cACgB,iBACd,iBAAiB;IACjB,GAAG;IACH,eAAe,KAAK,eAAe;IACnC,GAAI,KAAK,eAAe,mBAAmB,QAAQ,EACjD,iBAAiB,KAAK,eAAe,gBACvC;GACF;EACF;EACA,IAAI,cAAc,UAAU;GAC1B,qBAAqB,cAAc,UAAU,KAAK;GAElD,KAAK,MAAM,WAAW,cAAc,UAClC,IAAI,QAAQ,SAAS,YACnB,QAA8B,OAAO;EAG3C;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4RA,IAAa,eAAb,cAAkCC,wBAAAA,WAAW;CAC3C,eAAe;EAAC;EAAa;EAAe;CAAU;CACtD,wBAAwB;CACxB;CAEA,OAAO,UAA+B;EACpC,OAAO;CACT;CAIA,YACE,eACA,QACA;EACA,MAAM,SACJ,OAAO,kBAAkB,WACrB;GAAE,GAAI,UAAU,CAAC;GAAI,OAAO;EAAc,IAC1C;EACN,MAAM,wBAAwB,QAAQ,mBAAmB;EACzD,MAAM;GACJ,GAAG;GACH,cAAc;EAChB,CAAC;EACD,KAAK,wBAAwB;EAC7B,KAAK,iBAAiB,QAAQ;CAChC;CACA,iBACE,SAC4B;EAC5B,MAAM,SAAS,MAAM,iBAAiB,OAAO;EAC7C,IAAI,KAAK,uBACP,OAAO,qBAAqB;EAE9B,OAAO;CACT;CACA,OAAO,sBACL,UACA,SACA,YACqC;EACrC,IAAI;EACJ,WAAW,MAAM,SAAS,MAAM,sBAC9B,UACA,SACA,UACF,GAAG;GACD,MAAM,WACJ,MAAM,gBACL;GACH,IAAI,UACF,gBAAgB;GAElB,IAAI,MAAM,mBAAmBC,yBAAAA,gBAAgB;IAC3C,MAAM,WAAW,0BACf,MAAM,QAAQ,gBACd,aACF;IACA,IAAI,aAAa,MAAM,QAAQ,gBAC7B,MAAM,QAAQ,iBAAiB;IAEjC,8BAA8B,MAAM,OAAO;GAC7C;GACA,MAAM;EACR;CACF;CACA,gBACE,QACA,QACM;EAGN,MAAM,iBAAiB,QAAQ,kBAAkB,KAAK;EAEtD,MAAM,aAAa,IAAI,qBACrB;GAAE,GAAG;GAAQ,GAAG;EAAK,GACrB,KAAK,QACL,QACA,KACF;EACA,WAAW,iBAAiB;EAC5B,KAAK,aAAa;EAElB,MAAM,qBAAqB,IAAI,qBAC7B;GAAE,GAAG;GAAQ,GAAG;EAAK,GACrB,KAAK,QACL,QACA,IACF;EACA,mBAAmB,iBAAiB;EACpC,KAAK,qBAAqB;CAC5B;AACF"}
|
|
1
|
+
{"version":3,"file":"index.cjs","names":["STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY","GOOGLE_STREAMED_TOOL_CALL_ADAPTER","STREAMED_TOOL_CALL_SEAL_METADATA_KEY","isAIMessage","ChatConnection","ChatGoogle","resolveStreamDelay","smoothGenerationChunks","AIMessageChunk"],"sources":["../../../../src/llm/vertexai/index.ts"],"sourcesContent":["import { ChatGoogle } from '@langchain/google-gauth';\nimport { ChatConnection } from '@langchain/google-common';\nimport { AIMessageChunk, isAIMessage } from '@langchain/core/messages';\nimport type {\n GeminiContent,\n GeminiRequest,\n GoogleAIModelRequestParams,\n GoogleAbstractedClient,\n} from '@langchain/google-common';\nimport type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';\nimport type { BaseMessage, UsageMetadata } from '@langchain/core/messages';\nimport type { ChatGenerationChunk } from '@langchain/core/outputs';\nimport type { GoogleThinkingConfig, VertexAIClientOptions } from '@/types';\nimport {\n STREAMED_TOOL_CALL_SEAL_METADATA_KEY,\n STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,\n GOOGLE_STREAMED_TOOL_CALL_ADAPTER,\n} from '@/tools/streamedToolCallSeals';\nimport { smoothGenerationChunks } from '@/llm/stream/chunkAdapters';\nimport { resolveStreamDelay } from '@/llm/stream/smoother';\n\n/**\n * `@langchain/google-common`'s `_streamResponseChunks` emits usage on TWO\n * different paths within the same stream:\n *\n * - Streaming chunks set `chunk.generationInfo.usage_metadata` via\n * `responseToUsageMetadata`, which correctly sums\n * `candidatesTokenCount + thoughtsTokenCount` and includes\n * `output_token_details.reasoning`.\n * - The trailing fallback chunk (emitted after the API stream exhausts)\n * attaches its own `chunk.message.usage_metadata` built inline as\n * `output_tokens = candidatesTokenCount` only — dropping\n * `thoughtsTokenCount` and `output_token_details` entirely.\n *\n * After `AIMessageChunk.concat`, only `message.usage_metadata` survives —\n * which is the buggy fallback value. This breaks the documented\n * `total_tokens === input_tokens + output_tokens` invariant and silently\n * undercharges thinking models for reasoning tokens.\n *\n * The repair: track the last `generationInfo.usage_metadata` we see, and\n * when the fallback chunk arrives with its buggy `message.usage_metadata`,\n * replace it with the tracked good value. `CustomChatGoogleGenerativeAI`\n * solves the same problem for the Google API path differently — by\n * overriding `_convertToUsageMetadata`.\n */\nexport function repairStreamUsageMetadata(\n current: UsageMetadata | undefined,\n generationInfoUsage: UsageMetadata | undefined\n): UsageMetadata | undefined {\n if (!current) return current;\n if (!generationInfoUsage) return current;\n if (generationInfoUsage.total_tokens !== current.total_tokens) return current;\n if (generationInfoUsage.output_tokens <= current.output_tokens)\n return current;\n return generationInfoUsage;\n}\n\n/**\n * The Gemini API delivers function calls as complete objects — never as\n * partial arg deltas. `@langchain/google-common` pre-parses each streamed\n * functionCall part into `tool_calls` (invalid args land in\n * `invalid_tool_calls` instead), so a chunk whose tool-call chunks all parsed\n * cleanly is sealed on arrival for eager tool execution. Anything that fails\n * the parse check is left unstamped and falls back to the lazy path.\n */\nexport function sealCompleteStreamedToolCalls(message: AIMessageChunk): void {\n const chunkCount = message.tool_call_chunks?.length ?? 0;\n if (\n chunkCount === 0 ||\n (message.invalid_tool_calls?.length ?? 0) > 0 ||\n (message.tool_calls?.length ?? 0) !== chunkCount\n ) {\n return;\n }\n message.response_metadata = {\n ...message.response_metadata,\n [STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY]:\n GOOGLE_STREAMED_TOOL_CALL_ADAPTER,\n [STREAMED_TOOL_CALL_SEAL_METADATA_KEY]: { kind: 'all' },\n };\n}\n\ntype AdditionalKwargs =\n | undefined\n | (BaseMessage['additional_kwargs'] & {\n signatures?: Array<string | undefined>;\n });\n\n/**\n * Fixes thought signatures on functionCall parts in the formatted Gemini request.\n *\n * `@langchain/google-common` stores signatures as a flat array in\n * `additional_kwargs.signatures` (one per response part) and re-attaches them\n * by index only when `signatures.length === parts.length`. This fails when:\n * - The API omits a signature (length mismatch)\n * - Streaming chunks merge with different part counts\n * - The signature for a functionCall part is an empty string\n *\n * This function correlates each \"model\" content block in the formatted request\n * back to its originating AI message by *position*, then re-attaches non-empty\n * signatures that the library failed to apply. AI messages without signatures\n * still consume their slot — filtering them out shifted later messages onto\n * the wrong content block and dropped real signatures on the floor.\n */\nexport function fixThoughtSignatures(\n contents: GeminiContent[],\n input: BaseMessage[]\n): void {\n // All AI messages, in order — non-signature ones still consume positional\n // slots so later messages line up with their model content blocks.\n const aiMessages = input.filter(isAIMessage);\n const modelContents = contents.filter((c) => c.role === 'model');\n\n const count = Math.min(aiMessages.length, modelContents.length);\n for (let i = 0; i < count; i++) {\n const signatures = (aiMessages[i].additional_kwargs as AdditionalKwargs)\n ?.signatures;\n if (!Array.isArray(signatures) || signatures.length === 0) continue;\n\n const content = modelContents[i];\n const attachedSignatures = new Set(\n content.parts\n .map((p) => p.thoughtSignature)\n .filter((s): s is string => s != null && s !== '')\n );\n const availableSignatures = signatures.filter(\n (s): s is string => s != null && s !== '' && !attachedSignatures.has(s)\n );\n\n let sigIdx = 0;\n for (const part of content.parts) {\n if (\n 'functionCall' in part &&\n (part.thoughtSignature == null || part.thoughtSignature === '') &&\n sigIdx < availableSignatures.length\n ) {\n part.thoughtSignature = availableSignatures[sigIdx];\n sigIdx++;\n }\n }\n }\n}\n\nclass CustomChatConnection extends ChatConnection<VertexAIClientOptions> {\n thinkingConfig?: GoogleThinkingConfig;\n\n async formatData(\n input: BaseMessage[],\n parameters: GoogleAIModelRequestParams\n ): Promise<unknown> {\n const formattedData = (await super.formatData(\n input,\n parameters\n )) as GeminiRequest;\n if (formattedData.generationConfig?.thinkingConfig?.thinkingBudget === -1) {\n // -1 means \"let the model decide\" - delete the property so the API doesn't receive an invalid value\n if (\n formattedData.generationConfig.thinkingConfig.includeThoughts === false\n ) {\n formattedData.generationConfig.thinkingConfig.includeThoughts = true;\n }\n delete formattedData.generationConfig.thinkingConfig.thinkingBudget;\n }\n if (this.thinkingConfig?.thinkingLevel != null) {\n formattedData.generationConfig ??= {};\n // thinkingLevel and thinkingBudget cannot coexist — the API rejects the request.\n // Remove thinkingBudget when thinkingLevel is set.\n const { thinkingBudget: _, ...existingThinkingConfig } =\n (formattedData.generationConfig.thinkingConfig as\n | Record<string, unknown>\n | undefined) ?? {};\n (\n formattedData.generationConfig as Record<string, unknown>\n ).thinkingConfig = {\n ...existingThinkingConfig,\n thinkingLevel: this.thinkingConfig.thinkingLevel,\n ...(this.thinkingConfig.includeThoughts != null && {\n includeThoughts: this.thinkingConfig.includeThoughts,\n }),\n };\n }\n if (formattedData.contents) {\n fixThoughtSignatures(formattedData.contents, input);\n // gemini-3.1+ models reject role=\"function\"; convert to role=\"user\"\n for (const content of formattedData.contents) {\n if (content.role === 'function') {\n (content as { role: string }).role = 'user';\n }\n }\n }\n return formattedData;\n }\n}\n\n/**\n * Integration with Google Vertex AI chat models.\n *\n * Setup:\n * Install `@langchain/google-vertexai` and set your stringified\n * Vertex AI credentials as an environment variable named `GOOGLE_APPLICATION_CREDENTIALS`.\n *\n * ```bash\n * npm install @langchain/google-vertexai\n * export GOOGLE_APPLICATION_CREDENTIALS=\"path/to/credentials\"\n * ```\n *\n * ## [Constructor args](https://api.js.langchain.com/classes/_langchain_google_vertexai.index.ChatVertexAI.html#constructor.new_ChatVertexAI)\n *\n * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_common_types.GoogleAIBaseLanguageModelCallOptions.html)\n *\n * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.\n * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:\n *\n * ```typescript\n * // When calling `.withConfig`, call options should be passed via the first argument\n * const llmWithArgsBound = llm.withConfig({\n * stop: [\"\\n\"],\n * tools: [...],\n * });\n *\n * // When calling `.bindTools`, call options should be passed via the second argument\n * const llmWithTools = llm.bindTools(\n * [...],\n * {\n * tool_choice: \"auto\",\n * }\n * );\n * ```\n *\n * ## Examples\n *\n * <details open>\n * <summary><strong>Instantiate</strong></summary>\n *\n * ```typescript\n * import { ChatVertexAI } from '@langchain/google-vertexai';\n *\n * const llm = new ChatVertexAI({\n * model: \"gemini-1.5-pro\",\n * temperature: 0,\n * // other params...\n * });\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Invoking</strong></summary>\n *\n * ```typescript\n * const input = `Translate \"I love programming\" into French.`;\n *\n * // Models also accept a list of chat messages or a formatted prompt\n * const result = await llm.invoke(input);\n * console.log(result);\n * ```\n *\n * ```txt\n * AIMessageChunk {\n * \"content\": \"\\\"J'adore programmer\\\" \\n\\nHere's why this is the best translation:\\n\\n* **J'adore** means \\\"I love\\\" and conveys a strong passion.\\n* **Programmer** is the French verb for \\\"to program.\\\"\\n\\nThis translation is natural and idiomatic in French. \\n\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": [],\n * \"usage_metadata\": {\n * \"input_tokens\": 9,\n * \"output_tokens\": 63,\n * \"total_tokens\": 72\n * }\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Streaming Chunks</strong></summary>\n *\n * ```typescript\n * for await (const chunk of await llm.stream(input)) {\n * console.log(chunk);\n * }\n * ```\n *\n * ```txt\n * AIMessageChunk {\n * \"content\": \"\\\"\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": []\n * }\n * AIMessageChunk {\n * \"content\": \"J'adore programmer\\\" \\n\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": []\n * }\n * AIMessageChunk {\n * \"content\": \"\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {},\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": []\n * }\n * AIMessageChunk {\n * \"content\": \"\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {\n * \"finishReason\": \"stop\"\n * },\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": [],\n * \"usage_metadata\": {\n * \"input_tokens\": 9,\n * \"output_tokens\": 8,\n * \"total_tokens\": 17\n * }\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Aggregate Streamed Chunks</strong></summary>\n *\n * ```typescript\n * import { AIMessageChunk } from '@langchain/core/messages';\n * import { concat } from '@langchain/core/utils/stream';\n *\n * const stream = await llm.stream(input);\n * let full: AIMessageChunk | undefined;\n * for await (const chunk of stream) {\n * full = !full ? chunk : concat(full, chunk);\n * }\n * console.log(full);\n * ```\n *\n * ```txt\n * AIMessageChunk {\n * \"content\": \"\\\"J'adore programmer\\\" \\n\",\n * \"additional_kwargs\": {},\n * \"response_metadata\": {\n * \"finishReason\": \"stop\"\n * },\n * \"tool_calls\": [],\n * \"tool_call_chunks\": [],\n * \"invalid_tool_calls\": [],\n * \"usage_metadata\": {\n * \"input_tokens\": 9,\n * \"output_tokens\": 8,\n * \"total_tokens\": 17\n * }\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Bind tools</strong></summary>\n *\n * ```typescript\n * import { z } from 'zod';\n *\n * const GetWeather = {\n * name: \"GetWeather\",\n * description: \"Get the current weather in a given location\",\n * schema: z.object({\n * location: z.string().describe(\"The city and state, e.g. San Francisco, CA\")\n * }),\n * }\n *\n * const GetPopulation = {\n * name: \"GetPopulation\",\n * description: \"Get the current population in a given location\",\n * schema: z.object({\n * location: z.string().describe(\"The city and state, e.g. San Francisco, CA\")\n * }),\n * }\n *\n * const llmWithTools = llm.bindTools([GetWeather, GetPopulation]);\n * const aiMsg = await llmWithTools.invoke(\n * \"Which city is hotter today and which is bigger: LA or NY?\"\n * );\n * console.log(aiMsg.tool_calls);\n * ```\n *\n * ```txt\n * [\n * {\n * name: 'GetPopulation',\n * args: { location: 'New York City, NY' },\n * id: '33c1c1f47e2f492799c77d2800a43912',\n * type: 'tool_call'\n * }\n * ]\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Structured Output</strong></summary>\n *\n * ```typescript\n * import { z } from 'zod';\n *\n * const Joke = z.object({\n * setup: z.string().describe(\"The setup of the joke\"),\n * punchline: z.string().describe(\"The punchline to the joke\"),\n * rating: z.number().optional().describe(\"How funny the joke is, from 1 to 10\")\n * }).describe('Joke to tell user.');\n *\n * const structuredLlm = llm.withStructuredOutput(Joke, { name: \"Joke\" });\n * const jokeResult = await structuredLlm.invoke(\"Tell me a joke about cats\");\n * console.log(jokeResult);\n * ```\n *\n * ```txt\n * {\n * setup: 'What do you call a cat that loves to bowl?',\n * punchline: 'An alley cat!'\n * }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Usage Metadata</strong></summary>\n *\n * ```typescript\n * const aiMsgForMetadata = await llm.invoke(input);\n * console.log(aiMsgForMetadata.usage_metadata);\n * ```\n *\n * ```txt\n * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }\n * ```\n * </details>\n *\n * <br />\n *\n * <details>\n * <summary><strong>Stream Usage Metadata</strong></summary>\n *\n * ```typescript\n * const streamForMetadata = await llm.stream(\n * input,\n * {\n * streamUsage: true\n * }\n * );\n * let fullForMetadata: AIMessageChunk | undefined;\n * for await (const chunk of streamForMetadata) {\n * fullForMetadata = !fullForMetadata ? chunk : concat(fullForMetadata, chunk);\n * }\n * console.log(fullForMetadata?.usage_metadata);\n * ```\n *\n * ```txt\n * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }\n * ```\n * </details>\n *\n * <br />\n */\nexport class ChatVertexAI extends ChatGoogle {\n lc_namespace = ['langchain', 'chat_models', 'vertexai'];\n _lc_stream_delay: number;\n dynamicThinkingBudget = false;\n thinkingConfig?: GoogleThinkingConfig;\n\n static lc_name(): 'LibreChatVertexAI' {\n return 'LibreChatVertexAI';\n }\n\n constructor(model: string, fields?: Omit<VertexAIClientOptions, 'model'>);\n constructor(fields?: VertexAIClientOptions);\n constructor(\n modelOrFields?: string | VertexAIClientOptions,\n params?: Omit<VertexAIClientOptions, 'model'>\n ) {\n const fields =\n typeof modelOrFields === 'string'\n ? { ...(params ?? {}), model: modelOrFields }\n : modelOrFields;\n const dynamicThinkingBudget = fields?.thinkingBudget === -1;\n super({\n ...fields,\n platformType: 'gcp',\n });\n this.dynamicThinkingBudget = dynamicThinkingBudget;\n this.thinkingConfig = fields?.thinkingConfig;\n this._lc_stream_delay = resolveStreamDelay(fields?._lc_stream_delay);\n }\n invocationParams(\n options?: this['ParsedCallOptions'] | undefined\n ): GoogleAIModelRequestParams {\n const params = super.invocationParams(options);\n if (this.dynamicThinkingBudget) {\n params.maxReasoningTokens = -1;\n }\n return params;\n }\n async *_streamResponseChunks(\n messages: BaseMessage[],\n options: this['ParsedCallOptions'],\n runManager?: CallbackManagerForLLMRun\n ): AsyncGenerator<ChatGenerationChunk> {\n yield* smoothGenerationChunks({\n chunks: this._streamRepairedChunks(messages, options),\n delayMs: this._lc_stream_delay,\n signal: options.signal,\n runManager,\n });\n }\n\n private async *_streamRepairedChunks(\n messages: BaseMessage[],\n options: this['ParsedCallOptions']\n ): AsyncGenerator<ChatGenerationChunk> {\n let lastGoodUsage: UsageMetadata | undefined;\n for await (const chunk of super._streamResponseChunks(\n messages,\n options,\n undefined\n )) {\n const genUsage = (\n chunk.generationInfo as { usage_metadata?: UsageMetadata } | undefined\n )?.usage_metadata;\n if (genUsage) {\n lastGoodUsage = genUsage;\n }\n if (chunk.message instanceof AIMessageChunk) {\n const repaired = repairStreamUsageMetadata(\n chunk.message.usage_metadata,\n lastGoodUsage\n );\n if (repaired !== chunk.message.usage_metadata) {\n chunk.message.usage_metadata = repaired;\n }\n sealCompleteStreamedToolCalls(chunk.message);\n }\n yield chunk;\n }\n }\n buildConnection(\n fields: VertexAIClientOptions | undefined,\n client: GoogleAbstractedClient\n ): void {\n // Note: buildConnection is called from super() BEFORE this.thinkingConfig is set,\n // so we must read thinkingConfig from `fields` directly.\n const thinkingConfig = fields?.thinkingConfig ?? this.thinkingConfig;\n\n const connection = new CustomChatConnection(\n { ...fields, ...this },\n this.caller,\n client,\n false\n );\n connection.thinkingConfig = thinkingConfig;\n this.connection = connection;\n\n const streamedConnection = new CustomChatConnection(\n { ...fields, ...this },\n this.caller,\n client,\n true\n );\n streamedConnection.thinkingConfig = thinkingConfig;\n this.streamedConnection = streamedConnection;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CA,SAAgB,0BACd,SACA,qBAC2B;CAC3B,IAAI,CAAC,SAAS,OAAO;CACrB,IAAI,CAAC,qBAAqB,OAAO;CACjC,IAAI,oBAAoB,iBAAiB,QAAQ,cAAc,OAAO;CACtE,IAAI,oBAAoB,iBAAiB,QAAQ,eAC/C,OAAO;CACT,OAAO;AACT;;;;;;;;;AAUA,SAAgB,8BAA8B,SAA+B;CAC3E,MAAM,aAAa,QAAQ,kBAAkB,UAAU;CACvD,IACE,eAAe,MACd,QAAQ,oBAAoB,UAAU,KAAK,MAC3C,QAAQ,YAAY,UAAU,OAAO,YAEtC;CAEF,QAAQ,oBAAoB;EAC1B,GAAG,QAAQ;GACVA,8BAAAA,0CACCC,8BAAAA;GACDC,8BAAAA,uCAAuC,EAAE,MAAM,MAAM;CACxD;AACF;;;;;;;;;;;;;;;;;AAwBA,SAAgB,qBACd,UACA,OACM;CAGN,MAAM,aAAa,MAAM,OAAOC,yBAAAA,WAAW;CAC3C,MAAM,gBAAgB,SAAS,QAAQ,MAAM,EAAE,SAAS,OAAO;CAE/D,MAAM,QAAQ,KAAK,IAAI,WAAW,QAAQ,cAAc,MAAM;CAC9D,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;EAC9B,MAAM,aAAc,WAAW,EAAE,CAAC,mBAC9B;EACJ,IAAI,CAAC,MAAM,QAAQ,UAAU,KAAK,WAAW,WAAW,GAAG;EAE3D,MAAM,UAAU,cAAc;EAC9B,MAAM,qBAAqB,IAAI,IAC7B,QAAQ,MACL,KAAK,MAAM,EAAE,gBAAgB,CAAC,CAC9B,QAAQ,MAAmB,KAAK,QAAQ,MAAM,EAAE,CACrD;EACA,MAAM,sBAAsB,WAAW,QACpC,MAAmB,KAAK,QAAQ,MAAM,MAAM,CAAC,mBAAmB,IAAI,CAAC,CACxE;EAEA,IAAI,SAAS;EACb,KAAK,MAAM,QAAQ,QAAQ,OACzB,IACE,kBAAkB,SACjB,KAAK,oBAAoB,QAAQ,KAAK,qBAAqB,OAC5D,SAAS,oBAAoB,QAC7B;GACA,KAAK,mBAAmB,oBAAoB;GAC5C;EACF;CAEJ;AACF;AAEA,IAAM,uBAAN,cAAmCC,yBAAAA,eAAsC;CACvE;CAEA,MAAM,WACJ,OACA,YACkB;EAClB,MAAM,gBAAiB,MAAM,MAAM,WACjC,OACA,UACF;EACA,IAAI,cAAc,kBAAkB,gBAAgB,mBAAmB,IAAI;GAEzE,IACE,cAAc,iBAAiB,eAAe,oBAAoB,OAElE,cAAc,iBAAiB,eAAe,kBAAkB;GAElE,OAAO,cAAc,iBAAiB,eAAe;EACvD;EACA,IAAI,KAAK,gBAAgB,iBAAiB,MAAM;GAC9C,cAAc,qBAAqB,CAAC;GAGpC,MAAM,EAAE,gBAAgB,GAAG,GAAG,2BAC3B,cAAc,iBAAiB,kBAEd,CAAC;GACrB,cACgB,iBACd,iBAAiB;IACjB,GAAG;IACH,eAAe,KAAK,eAAe;IACnC,GAAI,KAAK,eAAe,mBAAmB,QAAQ,EACjD,iBAAiB,KAAK,eAAe,gBACvC;GACF;EACF;EACA,IAAI,cAAc,UAAU;GAC1B,qBAAqB,cAAc,UAAU,KAAK;GAElD,KAAK,MAAM,WAAW,cAAc,UAClC,IAAI,QAAQ,SAAS,YACnB,QAA8B,OAAO;EAG3C;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4RA,IAAa,eAAb,cAAkCC,wBAAAA,WAAW;CAC3C,eAAe;EAAC;EAAa;EAAe;CAAU;CACtD;CACA,wBAAwB;CACxB;CAEA,OAAO,UAA+B;EACpC,OAAO;CACT;CAIA,YACE,eACA,QACA;EACA,MAAM,SACJ,OAAO,kBAAkB,WACrB;GAAE,GAAI,UAAU,CAAC;GAAI,OAAO;EAAc,IAC1C;EACN,MAAM,wBAAwB,QAAQ,mBAAmB;EACzD,MAAM;GACJ,GAAG;GACH,cAAc;EAChB,CAAC;EACD,KAAK,wBAAwB;EAC7B,KAAK,iBAAiB,QAAQ;EAC9B,KAAK,mBAAmBC,iBAAAA,mBAAmB,QAAQ,gBAAgB;CACrE;CACA,iBACE,SAC4B;EAC5B,MAAM,SAAS,MAAM,iBAAiB,OAAO;EAC7C,IAAI,KAAK,uBACP,OAAO,qBAAqB;EAE9B,OAAO;CACT;CACA,OAAO,sBACL,UACA,SACA,YACqC;EACrC,OAAOC,sBAAAA,uBAAuB;GAC5B,QAAQ,KAAK,sBAAsB,UAAU,OAAO;GACpD,SAAS,KAAK;GACd,QAAQ,QAAQ;GAChB;EACF,CAAC;CACH;CAEA,OAAe,sBACb,UACA,SACqC;EACrC,IAAI;EACJ,WAAW,MAAM,SAAS,MAAM,sBAC9B,UACA,SACA,KAAA,CACF,GAAG;GACD,MAAM,WACJ,MAAM,gBACL;GACH,IAAI,UACF,gBAAgB;GAElB,IAAI,MAAM,mBAAmBC,yBAAAA,gBAAgB;IAC3C,MAAM,WAAW,0BACf,MAAM,QAAQ,gBACd,aACF;IACA,IAAI,aAAa,MAAM,QAAQ,gBAC7B,MAAM,QAAQ,iBAAiB;IAEjC,8BAA8B,MAAM,OAAO;GAC7C;GACA,MAAM;EACR;CACF;CACA,gBACE,QACA,QACM;EAGN,MAAM,iBAAiB,QAAQ,kBAAkB,KAAK;EAEtD,MAAM,aAAa,IAAI,qBACrB;GAAE,GAAG;GAAQ,GAAG;EAAK,GACrB,KAAK,QACL,QACA,KACF;EACA,WAAW,iBAAiB;EAC5B,KAAK,aAAa;EAElB,MAAM,qBAAqB,IAAI,qBAC7B;GAAE,GAAG;GAAQ,GAAG;EAAK,GACrB,KAAK,QACL,QACA,IACF;EACA,mBAAmB,iBAAiB;EACpC,KAAK,qBAAqB;CAC5B;AACF"}
|