@juspay/neurolink 10.10.6 → 10.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/browser/neurolink.min.js +399 -399
- package/dist/constants/contextWindows.js +10 -1
- package/dist/context/anthropicLoopGuard.d.ts +1 -0
- package/dist/context/anthropicLoopGuard.js +30 -12
- package/dist/context/contextCompactor.js +19 -0
- package/dist/context/geminiLoopGuard.d.ts +54 -0
- package/dist/context/geminiLoopGuard.js +140 -0
- package/dist/core/redisConversationMemoryManager.d.ts +27 -0
- package/dist/core/redisConversationMemoryManager.js +146 -25
- package/dist/lib/constants/contextWindows.js +10 -1
- package/dist/lib/context/anthropicLoopGuard.d.ts +1 -0
- package/dist/lib/context/anthropicLoopGuard.js +30 -12
- package/dist/lib/context/contextCompactor.js +19 -0
- package/dist/lib/context/geminiLoopGuard.d.ts +54 -0
- package/dist/lib/context/geminiLoopGuard.js +141 -0
- package/dist/lib/core/redisConversationMemoryManager.d.ts +27 -0
- package/dist/lib/core/redisConversationMemoryManager.js +146 -25
- package/dist/lib/providers/googleAiStudio/client.d.ts +0 -31
- package/dist/lib/providers/googleAiStudio/client.js +118 -1
- package/dist/lib/providers/googleNativeGemini3/utils.d.ts +9 -0
- package/dist/lib/providers/googleNativeGemini3/utils.js +12 -0
- package/dist/lib/providers/googleVertex/client.d.ts +0 -45
- package/dist/lib/providers/googleVertex/client.js +201 -21
- package/dist/lib/types/context.d.ts +9 -0
- package/dist/lib/utils/redis.d.ts +60 -1
- package/dist/lib/utils/redis.js +143 -12
- package/dist/providers/googleAiStudio/client.d.ts +0 -31
- package/dist/providers/googleAiStudio/client.js +118 -1
- package/dist/providers/googleNativeGemini3/utils.d.ts +9 -0
- package/dist/providers/googleNativeGemini3/utils.js +12 -0
- package/dist/providers/googleVertex/client.d.ts +0 -45
- package/dist/providers/googleVertex/client.js +201 -21
- package/dist/types/context.d.ts +9 -0
- package/dist/utils/redis.d.ts +60 -1
- package/dist/utils/redis.js +143 -12
- package/package.json +3 -1
|
@@ -2,37 +2,6 @@ import { type AIProviderName } from "../../constants/enums.js";
|
|
|
2
2
|
import { BaseProvider } from "../../core/baseProvider.js";
|
|
3
3
|
import type { ZodUnknownSchema, EnhancedGenerateResult, TextGenerationOptions, StreamOptions, StreamResult } from "../../types/index.js";
|
|
4
4
|
import type { LanguageModel, Schema } from "../../types/index.js";
|
|
5
|
-
/**
|
|
6
|
-
* Google AI Studio provider implementation using BaseProvider
|
|
7
|
-
* Migrated from original GoogleAIStudio class to new factory pattern
|
|
8
|
-
*
|
|
9
|
-
* @important Structured Output Limitation
|
|
10
|
-
* Google Gemini models cannot combine function calling (tools) with structured
|
|
11
|
-
* output (JSON schema). When using schemas with output.format: "json", you MUST
|
|
12
|
-
* set disableTools: true.
|
|
13
|
-
*
|
|
14
|
-
* Error without disableTools:
|
|
15
|
-
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
16
|
-
*
|
|
17
|
-
* This is a Google API limitation documented at:
|
|
18
|
-
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
19
|
-
*
|
|
20
|
-
* @example
|
|
21
|
-
* ```typescript
|
|
22
|
-
* // ✅ Correct usage with schemas
|
|
23
|
-
* const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
|
|
24
|
-
* const result = await provider.generate({
|
|
25
|
-
* input: { text: "Analyze data" },
|
|
26
|
-
* schema: MySchema,
|
|
27
|
-
* output: { format: "json" },
|
|
28
|
-
* disableTools: true // Required
|
|
29
|
-
* });
|
|
30
|
-
* ```
|
|
31
|
-
*
|
|
32
|
-
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
33
|
-
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
34
|
-
* Solution: Simplify schema or use disableTools: true
|
|
35
|
-
*/
|
|
36
5
|
export declare class GoogleAIStudioProvider extends BaseProvider {
|
|
37
6
|
private credentials?;
|
|
38
7
|
constructor(modelName?: string, sdk?: unknown, credentials?: {
|
|
@@ -6,12 +6,14 @@ import { ATTR, tracers, withClientSpan, withClientStreamSpan, withSpan, } from "
|
|
|
6
6
|
import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
7
7
|
import { ERROR_CODES, NeuroLinkError } from "../../utils/errorHandling.js";
|
|
8
8
|
import { logger } from "../../utils/logger.js";
|
|
9
|
+
import { GEMINI_ELISION_NOTE, planGeminiLoopReclaim, previewGeminiToolResponseText, } from "../../context/geminiLoopGuard.js";
|
|
10
|
+
import { getAvailableInputTokens, getContextWindowSize, } from "../../constants/contextWindows.js";
|
|
9
11
|
import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../../utils/timeout.js";
|
|
10
12
|
import { withTimeout } from "../../utils/async/index.js";
|
|
11
13
|
import { estimateTokens } from "../../utils/tokenEstimation.js";
|
|
12
14
|
import { transformToolExecutions } from "../../utils/transformationUtils.js";
|
|
13
15
|
import { resolveToolExecutionRecords } from "../../core/toolExecutionRecorder.js";
|
|
14
|
-
import { buildGeminiResponseSchema, buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, buildUserPartsWithMultimodal, executeNativeToolCalls, extractTextFromParts, extractThoughtSignature, handleMaxStepsTermination, prependConversationMessages, pushModelResponseToHistory, refreshNativeToolDeclarations, DedupExecuteMap, } from "../googleNativeGemini3/index.js";
|
|
16
|
+
import { buildGeminiResponseSchema, buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createContextGuard, createTextChannel, buildUserPartsWithMultimodal, executeNativeToolCalls, extractTextFromParts, extractThoughtSignature, handleMaxStepsTermination, prependConversationMessages, pushModelResponseToHistory, refreshNativeToolDeclarations, DedupExecuteMap, } from "../googleNativeGemini3/index.js";
|
|
15
17
|
import { createProxyFetch } from "../../proxy/proxyFetch.js";
|
|
16
18
|
// Google AI Live API types now imported from ../types/providerSpecific.js
|
|
17
19
|
// Import proper types for multimodal message handling
|
|
@@ -69,6 +71,74 @@ async function createGoogleGenAIClient(apiKey) {
|
|
|
69
71
|
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
70
72
|
* Solution: Simplify schema or use disableTools: true
|
|
71
73
|
*/
|
|
74
|
+
/**
|
|
75
|
+
* Reclaim context from an AI Studio loop history IN PLACE.
|
|
76
|
+
*
|
|
77
|
+
* This loop had NO in-turn guard at all — it appended a model turn plus a tool
|
|
78
|
+
* turn every step with nothing bounding growth, so a long agentic run walked
|
|
79
|
+
* into a provider "context length exceeded" and lost every completed step.
|
|
80
|
+
* Shares its reclaim policy with the other provider loops via loopGuardCore.
|
|
81
|
+
*
|
|
82
|
+
* Returns true when something was reclaimed.
|
|
83
|
+
*/
|
|
84
|
+
function reclaimAiStudioContext(contents, modelName, observedPromptTokens) {
|
|
85
|
+
const plan = planGeminiLoopReclaim({
|
|
86
|
+
contents,
|
|
87
|
+
availableInputTokens: getAvailableInputTokens("googleAiStudio", modelName),
|
|
88
|
+
provider: "googleAiStudio",
|
|
89
|
+
...(observedPromptTokens ? { observedPromptTokens } : {}),
|
|
90
|
+
});
|
|
91
|
+
if (!plan) {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
const dropSet = new Set(plan.drop);
|
|
95
|
+
const truncateSet = new Set(plan.truncate);
|
|
96
|
+
const rebuilt = [];
|
|
97
|
+
for (let i = 0; i < contents.length; i++) {
|
|
98
|
+
if (dropSet.has(i)) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
const content = contents[i];
|
|
102
|
+
if (truncateSet.has(i) && Array.isArray(content.parts)) {
|
|
103
|
+
rebuilt.push({
|
|
104
|
+
...content,
|
|
105
|
+
parts: content.parts.map((part) => {
|
|
106
|
+
const record = part;
|
|
107
|
+
if (!record.functionResponse) {
|
|
108
|
+
return part;
|
|
109
|
+
}
|
|
110
|
+
const text = JSON.stringify(record.functionResponse.response) ?? "";
|
|
111
|
+
if (text.length <= 2048) {
|
|
112
|
+
return part;
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
functionResponse: {
|
|
116
|
+
name: record.functionResponse.name,
|
|
117
|
+
response: { result: previewGeminiToolResponseText(text) },
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}),
|
|
121
|
+
});
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
rebuilt.push(content);
|
|
125
|
+
}
|
|
126
|
+
if (dropSet.size > 0) {
|
|
127
|
+
let noteIndex = rebuilt.findIndex((c) => Array.isArray(c.parts) &&
|
|
128
|
+
c.parts.some((part) => !!part.functionCall ||
|
|
129
|
+
!!part.functionResponse));
|
|
130
|
+
if (noteIndex < 0) {
|
|
131
|
+
noteIndex = Math.min(1, rebuilt.length);
|
|
132
|
+
}
|
|
133
|
+
rebuilt.splice(noteIndex, 0, {
|
|
134
|
+
role: "user",
|
|
135
|
+
parts: [{ text: GEMINI_ELISION_NOTE }],
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
contents.length = 0;
|
|
139
|
+
contents.push(...rebuilt);
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
72
142
|
export class GoogleAIStudioProvider extends BaseProvider {
|
|
73
143
|
credentials;
|
|
74
144
|
constructor(modelName, sdk, credentials) {
|
|
@@ -595,9 +665,27 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
595
665
|
let step = 0;
|
|
596
666
|
let completedWithFinalAnswer = false;
|
|
597
667
|
const failedTools = new Map();
|
|
668
|
+
// Cheap trigger for the in-turn reclaim, mirroring the Vertex twin.
|
|
669
|
+
// Planning serializes the WHOLE accumulated history to estimate it,
|
|
670
|
+
// so running it unconditionally charges that once per step for the
|
|
671
|
+
// life of the turn; the guard tracks real prompt counts plus
|
|
672
|
+
// measured growth instead, and it supplies the observed count that
|
|
673
|
+
// calibrates the planner's char estimate.
|
|
674
|
+
const contextGuard = createContextGuard(getContextWindowSize("googleAiStudio", modelName));
|
|
598
675
|
try {
|
|
599
676
|
// Agentic loop for tool calling
|
|
600
677
|
while (step < maxSteps) {
|
|
678
|
+
// In-turn context guard: this loop appends a model turn plus a
|
|
679
|
+
// tool turn every step with nothing bounding growth. No-op
|
|
680
|
+
// while the request still fits, so a loop that fits never pays
|
|
681
|
+
// a cache invalidation. Step 0 still plans unconditionally —
|
|
682
|
+
// the guard has no usage to go on yet, and the incoming history
|
|
683
|
+
// can already be oversized before the first call.
|
|
684
|
+
if (step === 0 || contextGuard.shouldStop()) {
|
|
685
|
+
if (reclaimAiStudioContext(currentContents, modelName, contextGuard.projectedNextPromptTokens)) {
|
|
686
|
+
contextGuard.resetAfterReclaim();
|
|
687
|
+
}
|
|
688
|
+
}
|
|
601
689
|
if (composedSignal?.aborted) {
|
|
602
690
|
throw composedSignal.reason instanceof Error
|
|
603
691
|
? composedSignal.reason
|
|
@@ -629,6 +717,10 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
629
717
|
totalOutputTokens += chunkResult.outputTokens;
|
|
630
718
|
totalCacheReadTokens += chunkResult.cacheReadTokens ?? 0;
|
|
631
719
|
totalReasoningTokens += chunkResult.reasoningTokens ?? 0;
|
|
720
|
+
// `inputTokens` is this step's promptTokenCount — the FULL
|
|
721
|
+
// prompt size for the request just made, which is what the
|
|
722
|
+
// guard projects the next request from.
|
|
723
|
+
contextGuard.noteUsage(chunkResult.inputTokens, chunkResult.outputTokens);
|
|
632
724
|
const stepText = extractTextFromParts(chunkResult.rawResponseParts);
|
|
633
725
|
// If no function calls, this was the final step — channel
|
|
634
726
|
// already received all text parts incrementally.
|
|
@@ -691,6 +783,14 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
691
783
|
role: "user",
|
|
692
784
|
parts: functionResponses,
|
|
693
785
|
});
|
|
786
|
+
// Project this step's growth: the appended tool results ride
|
|
787
|
+
// the next prompt, which the provider has not reported on yet.
|
|
788
|
+
try {
|
|
789
|
+
contextGuard.noteAppendedChars(JSON.stringify(functionResponses).length);
|
|
790
|
+
}
|
|
791
|
+
catch {
|
|
792
|
+
/* estimation is best-effort — never break the loop */
|
|
793
|
+
}
|
|
694
794
|
}
|
|
695
795
|
catch (error) {
|
|
696
796
|
logger.error("[GoogleAIStudio] Native SDK error", error);
|
|
@@ -877,8 +977,16 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
877
977
|
const toolExecutions = [];
|
|
878
978
|
let step = 0;
|
|
879
979
|
const failedTools = new Map();
|
|
980
|
+
// Cheap reclaim trigger — see the stream twin.
|
|
981
|
+
const contextGuard = createContextGuard(getContextWindowSize("googleAiStudio", modelName));
|
|
880
982
|
// Agentic loop for tool calling
|
|
881
983
|
while (step < maxSteps) {
|
|
984
|
+
// In-turn context guard — see the stream twin.
|
|
985
|
+
if (step === 0 || contextGuard.shouldStop()) {
|
|
986
|
+
if (reclaimAiStudioContext(currentContents, modelName, contextGuard.projectedNextPromptTokens)) {
|
|
987
|
+
contextGuard.resetAfterReclaim();
|
|
988
|
+
}
|
|
989
|
+
}
|
|
882
990
|
if (composedSignal?.aborted) {
|
|
883
991
|
throw composedSignal.reason instanceof Error
|
|
884
992
|
? composedSignal.reason
|
|
@@ -904,6 +1012,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
904
1012
|
totalOutputTokens += chunkResult.outputTokens;
|
|
905
1013
|
totalCacheReadTokens += chunkResult.cacheReadTokens ?? 0;
|
|
906
1014
|
totalReasoningTokens += chunkResult.reasoningTokens ?? 0;
|
|
1015
|
+
contextGuard.noteUsage(chunkResult.inputTokens, chunkResult.outputTokens);
|
|
907
1016
|
const stepText = extractTextFromParts(chunkResult.rawResponseParts);
|
|
908
1017
|
// If no function calls, we're done
|
|
909
1018
|
if (chunkResult.stepFunctionCalls.length === 0) {
|
|
@@ -961,6 +1070,14 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
961
1070
|
role: "user",
|
|
962
1071
|
parts: functionResponses,
|
|
963
1072
|
});
|
|
1073
|
+
// Project this step's growth: the appended tool results ride
|
|
1074
|
+
// the next prompt, which the provider has not reported on yet.
|
|
1075
|
+
try {
|
|
1076
|
+
contextGuard.noteAppendedChars(JSON.stringify(functionResponses).length);
|
|
1077
|
+
}
|
|
1078
|
+
catch {
|
|
1079
|
+
/* estimation is best-effort — never break the loop */
|
|
1080
|
+
}
|
|
964
1081
|
}
|
|
965
1082
|
catch (error) {
|
|
966
1083
|
logger.error("[GoogleAIStudio] Native SDK generate error", error);
|
|
@@ -385,6 +385,15 @@ export declare function createContextGuard(contextWindowTokens: number, threshol
|
|
|
385
385
|
* results, nudge text) using the ~4 chars/token heuristic.
|
|
386
386
|
*/
|
|
387
387
|
noteAppendedChars(chars: number): void;
|
|
388
|
+
/**
|
|
389
|
+
* Clear the projection after the caller has reclaimed context.
|
|
390
|
+
*
|
|
391
|
+
* The observed prompt size reflects the pre-reclaim conversation, so
|
|
392
|
+
* leaving it in place would keep `shouldStop()` true forever and defeat
|
|
393
|
+
* the reclaim. Resetting to the fail-open state means the guard stays
|
|
394
|
+
* quiet until the next real usage report re-establishes the truth.
|
|
395
|
+
*/
|
|
396
|
+
resetAfterReclaim(): void;
|
|
388
397
|
/** True when issuing another model call risks crossing the threshold. */
|
|
389
398
|
shouldStop(): boolean;
|
|
390
399
|
};
|
|
@@ -1229,6 +1229,18 @@ export function createContextGuard(contextWindowTokens, thresholdRatio = DEFAULT
|
|
|
1229
1229
|
projectedGrowthTokens += Math.ceil(chars / 4);
|
|
1230
1230
|
}
|
|
1231
1231
|
},
|
|
1232
|
+
/**
|
|
1233
|
+
* Clear the projection after the caller has reclaimed context.
|
|
1234
|
+
*
|
|
1235
|
+
* The observed prompt size reflects the pre-reclaim conversation, so
|
|
1236
|
+
* leaving it in place would keep `shouldStop()` true forever and defeat
|
|
1237
|
+
* the reclaim. Resetting to the fail-open state means the guard stays
|
|
1238
|
+
* quiet until the next real usage report re-establishes the truth.
|
|
1239
|
+
*/
|
|
1240
|
+
resetAfterReclaim() {
|
|
1241
|
+
observedPromptTokens = 0;
|
|
1242
|
+
projectedGrowthTokens = 0;
|
|
1243
|
+
},
|
|
1232
1244
|
/** True when issuing another model call risks crossing the threshold. */
|
|
1233
1245
|
shouldStop() {
|
|
1234
1246
|
return (observedPromptTokens > 0 &&
|
|
@@ -51,51 +51,6 @@ export declare function stripAdditionalPropertiesDeep(schema: Record<string, unk
|
|
|
51
51
|
* @returns The region string to pass to the @google/genai client.
|
|
52
52
|
*/
|
|
53
53
|
export declare const resolveVertexLocation: (modelName: string | undefined, configuredLocation?: string) => string;
|
|
54
|
-
/**
|
|
55
|
-
* Google Vertex AI Provider v2 - BaseProvider Implementation
|
|
56
|
-
*
|
|
57
|
-
* Features:
|
|
58
|
-
* - Extends BaseProvider for shared functionality
|
|
59
|
-
* - Preserves existing Google Cloud authentication
|
|
60
|
-
* - Maintains Anthropic model support via dynamic imports
|
|
61
|
-
* - Fresh model creation for each request
|
|
62
|
-
* - Enhanced error handling with setup guidance
|
|
63
|
-
* - Tool registration and context management
|
|
64
|
-
*
|
|
65
|
-
* @important Tools + Schema Support (Fixed)
|
|
66
|
-
* Gemini models on Vertex AI now support combining function calling (tools) with
|
|
67
|
-
* structured output (JSON schema) simultaneously. The fix works by NOT setting
|
|
68
|
-
* `responseMimeType: "application/json"` when tools are present, which was
|
|
69
|
-
* causing the Google API error.
|
|
70
|
-
*
|
|
71
|
-
* The `responseSchema` is still set to guide the output structure, allowing
|
|
72
|
-
* tools to execute AND the final output to follow the schema format.
|
|
73
|
-
*
|
|
74
|
-
* @example Gemini models with tools + schemas
|
|
75
|
-
* ```typescript
|
|
76
|
-
* const provider = new GoogleVertexProvider("gemini-2.5-flash");
|
|
77
|
-
* const result = await provider.generate({
|
|
78
|
-
* input: { text: "Analyze data using tools" },
|
|
79
|
-
* schema: MySchema,
|
|
80
|
-
* output: { format: "json" },
|
|
81
|
-
* // No need for disableTools: true anymore!
|
|
82
|
-
* });
|
|
83
|
-
* ```
|
|
84
|
-
*
|
|
85
|
-
* @example Claude models (always supported both)
|
|
86
|
-
* ```typescript
|
|
87
|
-
* const provider = new GoogleVertexProvider("claude-3-5-sonnet-20241022");
|
|
88
|
-
* const result = await provider.generate({
|
|
89
|
-
* input: { text: "Analyze data" },
|
|
90
|
-
* schema: MySchema,
|
|
91
|
-
* output: { format: "json" }
|
|
92
|
-
* });
|
|
93
|
-
* ```
|
|
94
|
-
*
|
|
95
|
-
* @note "Too many states for serving" errors can still occur with very complex schemas + tools.
|
|
96
|
-
* Solution: Simplify schema or reduce number of tools if this occurs.
|
|
97
|
-
* @see https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
|
|
98
|
-
*/
|
|
99
54
|
export declare class GoogleVertexProvider extends BaseProvider {
|
|
100
55
|
private projectId;
|
|
101
56
|
private location;
|
|
@@ -16,6 +16,8 @@ import { applyVertexAnthropicCacheBreakpoints } from "../../utils/anthropicCache
|
|
|
16
16
|
import { FileDetector } from "../../utils/fileDetector.js";
|
|
17
17
|
import { mergeMediaFileAliases, processUnifiedFilesArray, } from "../../utils/messageBuilder.js";
|
|
18
18
|
import { logger } from "../../utils/logger.js";
|
|
19
|
+
import { GEMINI_ELISION_NOTE, planGeminiLoopReclaim, previewGeminiToolResponseText, } from "../../context/geminiLoopGuard.js";
|
|
20
|
+
import { ANTHROPIC_ELISION_NOTE, planAnthropicLoopReclaim, previewAnthropicToolResultText, } from "../../context/anthropicLoopGuard.js";
|
|
19
21
|
import { hasRestrictedOutputLimit, RESTRICTED_OUTPUT_TOKEN_LIMIT, toVertexAnthropicModelId, } from "../../utils/modelDetection.js";
|
|
20
22
|
import { detectImageMimeType } from "../../utils/imageDetection.js";
|
|
21
23
|
import { resolveClaudeMaxTokens } from "../../utils/tokenLimits.js";
|
|
@@ -25,7 +27,7 @@ import { createNativeThinkingConfig } from "../../utils/thinkingConfig.js";
|
|
|
25
27
|
import { TimeoutError, raceWithAbort, withTimeout, } from "../../utils/async/index.js";
|
|
26
28
|
import { parseTimeout } from "../../utils/timeout.js";
|
|
27
29
|
import { appendStepText, buildAbortedTurnMessage, buildContextCapMessage, buildToolLoopCapMessage, buildTurnStalledMessage, buildTurnTimeoutMessage, buildWrapupNudgeText, createContextGuard, createTextChannel, createTurnClock, extractThoughtSignature, isAbortError, mapGeminiFinishReason, prependConversationMessages, resolveTurnStopReason, DedupExecuteMap, } from "../googleNativeGemini3/index.js";
|
|
28
|
-
import { getContextWindowSize } from "../../constants/contextWindows.js";
|
|
30
|
+
import { getAvailableInputTokens, getContextWindowSize, } from "../../constants/contextWindows.js";
|
|
29
31
|
import { resolveLiveTool } from "../../tools/toolDiscovery.js";
|
|
30
32
|
import { ATTR, LANGFUSE_ATTR, spanJsonAttribute, tracers, withClientSpan, withClientStreamSpan, withSpan, } from "../../telemetry/index.js";
|
|
31
33
|
import { SpanKind, SpanStatusCode, context as otelContext, trace as otelTrace, } from "@opentelemetry/api";
|
|
@@ -587,6 +589,148 @@ const isAnthropicModel = (modelName) => {
|
|
|
587
589
|
* Solution: Simplify schema or reduce number of tools if this occurs.
|
|
588
590
|
* @see https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
|
|
589
591
|
*/
|
|
592
|
+
/** Byte budget above which an old tool response is previewed, not kept whole. */
|
|
593
|
+
const TOOL_RESPONSE_PREVIEW_BYTES = 2048;
|
|
594
|
+
/**
|
|
595
|
+
* Reclaim context from a Gemini-shaped loop history IN PLACE.
|
|
596
|
+
*
|
|
597
|
+
* Returns true when something was actually reclaimed, which tells the caller
|
|
598
|
+
* it is safe to continue the loop instead of stopping. Mutates `contents` so
|
|
599
|
+
* the caller's array identity (captured by the request builder) stays valid.
|
|
600
|
+
*/
|
|
601
|
+
function reclaimVertexLoopContext(contents, modelName, observedPromptTokens) {
|
|
602
|
+
const plan = planGeminiLoopReclaim({
|
|
603
|
+
contents,
|
|
604
|
+
// The usable INPUT budget, not the whole window: the window has to hold the
|
|
605
|
+
// model's output too, and the AI Studio twin already reclaims against this
|
|
606
|
+
// same definition.
|
|
607
|
+
availableInputTokens: getAvailableInputTokens("vertex", modelName),
|
|
608
|
+
provider: "vertex",
|
|
609
|
+
observedPromptTokens,
|
|
610
|
+
});
|
|
611
|
+
if (!plan) {
|
|
612
|
+
return false;
|
|
613
|
+
}
|
|
614
|
+
const dropSet = new Set(plan.drop);
|
|
615
|
+
const truncateSet = new Set(plan.truncate);
|
|
616
|
+
const rebuilt = [];
|
|
617
|
+
for (let i = 0; i < contents.length; i++) {
|
|
618
|
+
if (dropSet.has(i)) {
|
|
619
|
+
continue;
|
|
620
|
+
}
|
|
621
|
+
const content = contents[i];
|
|
622
|
+
if (truncateSet.has(i) && Array.isArray(content.parts)) {
|
|
623
|
+
rebuilt.push({
|
|
624
|
+
...content,
|
|
625
|
+
parts: content.parts.map((part) => {
|
|
626
|
+
if (!("functionResponse" in part)) {
|
|
627
|
+
return part;
|
|
628
|
+
}
|
|
629
|
+
const fn = part.functionResponse;
|
|
630
|
+
const text = JSON.stringify(fn.response) ?? "";
|
|
631
|
+
if (text.length <= TOOL_RESPONSE_PREVIEW_BYTES) {
|
|
632
|
+
return part;
|
|
633
|
+
}
|
|
634
|
+
// Rebuilt rather than cast: `functionResponse` requires `name`, and
|
|
635
|
+
// Critical Rule 14 forbids casting through `unknown` to paper over it.
|
|
636
|
+
return {
|
|
637
|
+
functionResponse: {
|
|
638
|
+
name: fn.name,
|
|
639
|
+
response: { result: previewGeminiToolResponseText(text) },
|
|
640
|
+
},
|
|
641
|
+
};
|
|
642
|
+
}),
|
|
643
|
+
});
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
rebuilt.push(content);
|
|
647
|
+
}
|
|
648
|
+
if (dropSet.size > 0) {
|
|
649
|
+
// Gemini requires the history to start with a user turn; the note is
|
|
650
|
+
// inserted before the first surviving tool turn, never at the end where a
|
|
651
|
+
// "history was removed" cue would follow the content it refers to.
|
|
652
|
+
let noteIndex = rebuilt.findIndex((c) => Array.isArray(c.parts) &&
|
|
653
|
+
c.parts.some((part) => !!part.functionCall ||
|
|
654
|
+
!!part.functionResponse));
|
|
655
|
+
if (noteIndex < 0) {
|
|
656
|
+
noteIndex = Math.min(1, rebuilt.length);
|
|
657
|
+
}
|
|
658
|
+
rebuilt.splice(noteIndex, 0, {
|
|
659
|
+
role: "user",
|
|
660
|
+
parts: [{ text: GEMINI_ELISION_NOTE }],
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
contents.length = 0;
|
|
664
|
+
contents.push(...rebuilt);
|
|
665
|
+
return true;
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Reclaim context from a Vertex+Claude loop history IN PLACE.
|
|
669
|
+
*
|
|
670
|
+
* Same parity upgrade as the Gemini path, but this loop carries Anthropic
|
|
671
|
+
* content blocks, so it reuses the Anthropic adapter. Returns true when
|
|
672
|
+
* something was reclaimed and the loop may continue.
|
|
673
|
+
*/
|
|
674
|
+
function reclaimVertexAnthropicContext(messages, modelName, observedPromptTokens) {
|
|
675
|
+
const plan = planAnthropicLoopReclaim({
|
|
676
|
+
conversation: messages,
|
|
677
|
+
// Usable input budget, matching the Gemini twin above.
|
|
678
|
+
availableInputTokens: getAvailableInputTokens("vertex", modelName),
|
|
679
|
+
fixedOverheadTokens: 0,
|
|
680
|
+
provider: "vertex",
|
|
681
|
+
observedPromptTokens,
|
|
682
|
+
// This loop plans only when its context guard trips, so the count it hands
|
|
683
|
+
// over is the guard's projection for the request about to be sent, not a
|
|
684
|
+
// previous request's total. Without saying so the planner has no
|
|
685
|
+
// denominator, calibration stays pinned at 1, and the reclaim is inert:
|
|
686
|
+
// the guard fires on real tokens at the same ratio the planner tests its
|
|
687
|
+
// smaller char estimate against, so the plan never fires and the turn stops
|
|
688
|
+
// instead of continuing.
|
|
689
|
+
observedDescribesCurrentPayload: true,
|
|
690
|
+
});
|
|
691
|
+
if (!plan) {
|
|
692
|
+
return false;
|
|
693
|
+
}
|
|
694
|
+
const dropSet = new Set(plan.drop);
|
|
695
|
+
const truncateSet = new Set(plan.truncate);
|
|
696
|
+
const rebuilt = [];
|
|
697
|
+
for (let i = 0; i < messages.length; i++) {
|
|
698
|
+
if (dropSet.has(i)) {
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
const message = messages[i];
|
|
702
|
+
if (truncateSet.has(i) && Array.isArray(message.content)) {
|
|
703
|
+
rebuilt.push({
|
|
704
|
+
...message,
|
|
705
|
+
content: message.content.map((block) => {
|
|
706
|
+
if (block.type !== "tool_result") {
|
|
707
|
+
return block;
|
|
708
|
+
}
|
|
709
|
+
const text = typeof block.content === "string"
|
|
710
|
+
? block.content
|
|
711
|
+
: (JSON.stringify(block.content) ?? "");
|
|
712
|
+
return { ...block, content: previewAnthropicToolResultText(text) };
|
|
713
|
+
}),
|
|
714
|
+
});
|
|
715
|
+
continue;
|
|
716
|
+
}
|
|
717
|
+
rebuilt.push(message);
|
|
718
|
+
}
|
|
719
|
+
if (dropSet.size > 0) {
|
|
720
|
+
let noteIndex = rebuilt.findIndex((m) => Array.isArray(m.content) &&
|
|
721
|
+
m.content.some((b) => b.type === "tool_use" || b.type === "tool_result"));
|
|
722
|
+
if (noteIndex < 0) {
|
|
723
|
+
noteIndex = Math.min(1, rebuilt.length);
|
|
724
|
+
}
|
|
725
|
+
rebuilt.splice(noteIndex, 0, {
|
|
726
|
+
role: "user",
|
|
727
|
+
content: [{ type: "text", text: ANTHROPIC_ELISION_NOTE }],
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
messages.length = 0;
|
|
731
|
+
messages.push(...rebuilt);
|
|
732
|
+
return true;
|
|
733
|
+
}
|
|
590
734
|
export class GoogleVertexProvider extends BaseProvider {
|
|
591
735
|
projectId;
|
|
592
736
|
location;
|
|
@@ -1574,11 +1718,22 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
1574
1718
|
// conversation crosses the window threshold — synthesize from what
|
|
1575
1719
|
// we have instead of stepping into a provider rejection.
|
|
1576
1720
|
if (contextGuard.shouldStop()) {
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1721
|
+
// Parity upgrade: try to RECLAIM budget and keep going before
|
|
1722
|
+
// falling back to the historic stop-only behaviour. Ending the turn
|
|
1723
|
+
// early is safe but throws away work the model was mid-way through;
|
|
1724
|
+
// dropping the oldest complete tool exchanges usually buys enough
|
|
1725
|
+
// room to finish. Only when reclaiming changes nothing do we stop.
|
|
1726
|
+
const reclaimed = reclaimVertexLoopContext(currentContents, modelName, contextGuard.projectedNextPromptTokens);
|
|
1727
|
+
if (reclaimed) {
|
|
1728
|
+
contextGuard.resetAfterReclaim();
|
|
1729
|
+
}
|
|
1730
|
+
else {
|
|
1731
|
+
hitContextLimit = true;
|
|
1732
|
+
logger.warn(`[GoogleVertex] Gemini turn stopped by the context guard: ` +
|
|
1733
|
+
`projected prompt ~${contextGuard.projectedNextPromptTokens} tokens ` +
|
|
1734
|
+
`>= threshold ${contextGuard.thresholdTokens} (step ${step}) — synthesizing a final answer.`);
|
|
1735
|
+
break;
|
|
1736
|
+
}
|
|
1582
1737
|
}
|
|
1583
1738
|
step++;
|
|
1584
1739
|
turnClock.noteProgress();
|
|
@@ -2550,11 +2705,22 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
2550
2705
|
// conversation crosses the window threshold — synthesize from what
|
|
2551
2706
|
// we have instead of stepping into a provider rejection.
|
|
2552
2707
|
if (contextGuard.shouldStop()) {
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2708
|
+
// Parity upgrade: try to RECLAIM budget and keep going before
|
|
2709
|
+
// falling back to the historic stop-only behaviour. Ending the turn
|
|
2710
|
+
// early is safe but throws away work the model was mid-way through;
|
|
2711
|
+
// dropping the oldest complete tool exchanges usually buys enough
|
|
2712
|
+
// room to finish. Only when reclaiming changes nothing do we stop.
|
|
2713
|
+
const reclaimed = reclaimVertexLoopContext(currentContents, modelName, contextGuard.projectedNextPromptTokens);
|
|
2714
|
+
if (reclaimed) {
|
|
2715
|
+
contextGuard.resetAfterReclaim();
|
|
2716
|
+
}
|
|
2717
|
+
else {
|
|
2718
|
+
hitContextLimit = true;
|
|
2719
|
+
logger.warn(`[GoogleVertex] Gemini turn stopped by the context guard: ` +
|
|
2720
|
+
`projected prompt ~${contextGuard.projectedNextPromptTokens} tokens ` +
|
|
2721
|
+
`>= threshold ${contextGuard.thresholdTokens} (step ${step}) — synthesizing a final answer.`);
|
|
2722
|
+
break;
|
|
2723
|
+
}
|
|
2558
2724
|
}
|
|
2559
2725
|
step++;
|
|
2560
2726
|
turnClock.noteProgress();
|
|
@@ -3671,11 +3837,19 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
3671
3837
|
// Context guard: stop the tool loop before the accumulated
|
|
3672
3838
|
// conversation crosses the window threshold (see generate twin).
|
|
3673
3839
|
if (contextGuard.shouldStop()) {
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3840
|
+
// Parity upgrade: reclaim and continue where possible; the
|
|
3841
|
+
// historic stop-only behaviour remains the fallback.
|
|
3842
|
+
const reclaimed = reclaimVertexAnthropicContext(currentMessages, modelName, contextGuard.projectedNextPromptTokens);
|
|
3843
|
+
if (reclaimed) {
|
|
3844
|
+
contextGuard.resetAfterReclaim();
|
|
3845
|
+
}
|
|
3846
|
+
else {
|
|
3847
|
+
hitContextLimit = true;
|
|
3848
|
+
logger.warn(`[GoogleVertex] Anthropic stream turn stopped by the context guard: ` +
|
|
3849
|
+
`projected prompt ~${contextGuard.projectedNextPromptTokens} tokens ` +
|
|
3850
|
+
`>= threshold ${contextGuard.thresholdTokens} (step ${step}) — synthesizing a final answer.`);
|
|
3851
|
+
break;
|
|
3852
|
+
}
|
|
3679
3853
|
}
|
|
3680
3854
|
step++;
|
|
3681
3855
|
turnClock.noteProgress();
|
|
@@ -4929,11 +5103,17 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
4929
5103
|
// this step's appended tool results/output) would cross the window
|
|
4930
5104
|
// threshold — stop the tool loop and synthesize from what we have.
|
|
4931
5105
|
if (contextGuard.shouldStop()) {
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
5106
|
+
const reclaimed = reclaimVertexAnthropicContext(currentMessages, modelName, contextGuard.projectedNextPromptTokens);
|
|
5107
|
+
if (reclaimed) {
|
|
5108
|
+
contextGuard.resetAfterReclaim();
|
|
5109
|
+
}
|
|
5110
|
+
else {
|
|
5111
|
+
hitContextLimit = true;
|
|
5112
|
+
logger.warn(`[GoogleVertex] Anthropic generate turn stopped by the context guard: ` +
|
|
5113
|
+
`projected prompt ~${contextGuard.projectedNextPromptTokens} tokens ` +
|
|
5114
|
+
`>= threshold ${contextGuard.thresholdTokens} (step ${step}) — synthesizing a final answer.`);
|
|
5115
|
+
break;
|
|
5116
|
+
}
|
|
4937
5117
|
}
|
|
4938
5118
|
step++;
|
|
4939
5119
|
turnClock.noteProgress();
|
|
@@ -471,6 +471,15 @@ export type AnthropicGuardMessage = {
|
|
|
471
471
|
role: "user" | "assistant" | "system";
|
|
472
472
|
content: string | AnthropicGuardBlock[];
|
|
473
473
|
};
|
|
474
|
+
/**
|
|
475
|
+
* Structural view of one Gemini history entry, loose enough to accept both the
|
|
476
|
+
* native Vertex loop's `{ role, parts }` array and `@google/genai` contents
|
|
477
|
+
* without a cast at either call site.
|
|
478
|
+
*/
|
|
479
|
+
export type GeminiGuardContent = {
|
|
480
|
+
role: string;
|
|
481
|
+
parts: unknown[];
|
|
482
|
+
};
|
|
474
483
|
/** Tuning for {@link planLoopGuardReclaim}. */
|
|
475
484
|
export type LoopGuardPolicy = {
|
|
476
485
|
availableInputTokens: number;
|