@juspay/neurolink 12.12.11 → 12.12.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +216 -216
- package/dist/constants/contextWindows.js +1 -0
- package/dist/constants/enums.d.ts +1 -0
- package/dist/constants/enums.js +7 -0
- package/dist/core/baseProvider.js +4 -4
- package/dist/core/modules/structuredOutputPolicy.d.ts +6 -6
- package/dist/core/modules/structuredOutputPolicy.js +6 -6
- package/dist/core/nativeGenerateLoop.js +11 -0
- package/dist/core/nativeToolFormat.js +2 -2
- package/dist/factories/providerDescriptors.js +1 -1
- package/dist/factories/providerRegistry.js +4 -1
- package/dist/neurolink.js +1 -1
- package/dist/providers/amazonSagemaker.js +1 -0
- package/dist/providers/anthropic/cacheControl.d.ts +3 -3
- package/dist/providers/anthropic/cacheControl.js +3 -3
- package/dist/providers/anthropic/client.js +26 -8
- package/dist/providers/configuredOpenAICompat.d.ts +1 -1
- package/dist/providers/configuredOpenAICompat.js +1 -1
- package/dist/providers/googleAiStudio/client.js +2 -3
- package/dist/providers/googleVertex/client.js +2 -2
- package/dist/providers/nvidiaNim/client.js +11 -1
- package/dist/providers/openaiChatCompletionsBase.d.ts +4 -4
- package/dist/providers/openaiChatCompletionsBase.js +11 -10
- package/dist/types/context.d.ts +0 -22
- package/dist/types/generate.d.ts +2 -17
- package/dist/types/providers.d.ts +1 -27
- package/dist/utils/anthropicCacheBreakpoints.d.ts +6 -6
- package/dist/utils/anthropicCacheBreakpoints.js +6 -6
- package/dist/utils/json/coerce.d.ts +3 -2
- package/dist/utils/json/coerce.js +3 -2
- package/dist/utils/modelChoices.js +1 -1
- package/dist/utils/pricing.js +8 -0
- package/dist/utils/tokenUtils.d.ts +0 -1
- package/dist/utils/tokenUtils.js +0 -1
- package/dist/utils/tool.js +0 -3
- package/docs-site/static/search-index.json +1 -1
- package/package.json +3 -1
- package/dist/context/stepBudgetGuard.d.ts +0 -61
- package/dist/context/stepBudgetGuard.js +0 -357
- package/dist/core/serviceRegistry.d.ts +0 -40
- package/dist/core/serviceRegistry.js +0 -112
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "12.12.
|
|
3
|
+
"version": "12.12.13",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
|
|
6
6
|
"author": {
|
|
@@ -154,6 +154,8 @@
|
|
|
154
154
|
"test:tasks": "pnpm exec tsx test/continuous-test-suite-tasks.ts",
|
|
155
155
|
"test:auth": "pnpm exec tsx test/continuous-test-suite-auth.ts",
|
|
156
156
|
"test:autoresearch": "pnpm exec tsx test/continuous-test-suite-autoresearch.ts",
|
|
157
|
+
"test:codex-quota": "svelte-kit sync && vitest run test/codex-quota-observability.test.ts",
|
|
158
|
+
"test:agents-live": "tsx test/continuous-test-agents-live.ts",
|
|
157
159
|
"test:tool-routing-cli": "pnpm exec tsx test/continuous-test-suite-tool-routing-cli.ts",
|
|
158
160
|
"test:tool-dedup": "pnpm exec tsx test/continuous-test-suite-tool-dedup.ts",
|
|
159
161
|
"test:model-not-found-retryable": "pnpm exec tsx test/continuous-test-suite-model-not-found-retryable.ts",
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Per-step context budget guard for the AI-SDK agent loop.
|
|
3
|
-
*
|
|
4
|
-
* Pre-call budgeting (`checkContextBudget` + compaction) runs ONCE before
|
|
5
|
-
* dispatch and only sees the input/session conversation. The AI-SDK tool loop
|
|
6
|
-
* then appends assistant turns and tool results on every step — growth the
|
|
7
|
-
* pre-call pipeline never sees, which is how long agentic runs overflow the
|
|
8
|
-
* model's real context window mid-loop (provider 400s after dozens of tool
|
|
9
|
-
* calls). The googleVertex native loops already guard this via
|
|
10
|
-
* `createContextGuard`; this module brings the AI-SDK path (every provider
|
|
11
|
-
* that delegates to `generateText`) to parity — and goes one step further:
|
|
12
|
-
* instead of stopping the loop, it deterministically reclaims budget so the
|
|
13
|
-
* loop can CONTINUE.
|
|
14
|
-
*
|
|
15
|
-
* Wired in `GenerationHandler.callGenerateText` through
|
|
16
|
-
* `experimental_prepareStep`, whose result may replace the step's `messages`.
|
|
17
|
-
* The guard operates on `ModelMessage[]` natively (no lossy ChatMessage
|
|
18
|
-
* round-trip) and never makes LLM calls:
|
|
19
|
-
*
|
|
20
|
-
* Stage 1 — truncate OLD tool outputs to head/tail previews
|
|
21
|
-
* (`generateToolOutputPreview`), oldest first, outside the
|
|
22
|
-
* protected recent tail.
|
|
23
|
-
* Stage 2 — drop the oldest complete tool exchanges (assistant tool-call
|
|
24
|
-
* message + its following tool-result messages, as a unit, so
|
|
25
|
-
* call/result pairing stays intact), replacing them with a single
|
|
26
|
-
* elision note.
|
|
27
|
-
*
|
|
28
|
-
* The system prompt and tool definitions ride OUTSIDE the step messages (the
|
|
29
|
-
* handler hoists system into generateText's `system` option), so their cost is
|
|
30
|
-
* passed in as `fixedOverheadTokens`. The first user message (the task) and
|
|
31
|
-
* the most recent messages are never touched.
|
|
32
|
-
*/
|
|
33
|
-
import type { ModelMessage, StepBudgetGuardConfig } from "../types/index.js";
|
|
34
|
-
/** Estimate the token cost of a step's message array. */
|
|
35
|
-
export declare function estimateStepMessagesTokens(messages: readonly ModelMessage[], provider?: string): number;
|
|
36
|
-
/**
|
|
37
|
-
* Estimate the fixed per-request overhead: hoisted system prompt + tool
|
|
38
|
-
* definitions. Mirrors `checkContextBudget`'s categories for the pieces that
|
|
39
|
-
* do not live in the step messages.
|
|
40
|
-
*/
|
|
41
|
-
export declare function estimateFixedOverheadTokens(system: unknown, tools: Record<string, unknown> | undefined, provider?: string): number;
|
|
42
|
-
/**
|
|
43
|
-
* Create a per-step budget guard. Returns a function that, given the step's
|
|
44
|
-
* messages (and optionally the REAL input-token count the provider reported
|
|
45
|
-
* for the previous step), returns a compacted replacement array when the
|
|
46
|
-
* projected request exceeds the threshold — or `undefined` when no change is
|
|
47
|
-
* needed.
|
|
48
|
-
*
|
|
49
|
-
* Two dynamic behaviours:
|
|
50
|
-
* - the available-input budget is re-resolved on EVERY invocation, so
|
|
51
|
-
* runtime window discovery (`/model/info`, overflow self-healing) that
|
|
52
|
-
* lands mid-loop takes effect immediately instead of the guard staying
|
|
53
|
-
* frozen on the value captured at loop start;
|
|
54
|
-
* - usage feedback calibrates the estimator: the ratio between the real
|
|
55
|
-
* prompt tokens of the previous step and this guard's own estimate for
|
|
56
|
-
* what that step sent scales later estimates (only UP — underestimates
|
|
57
|
-
* overflow, overestimates merely compact earlier), eliminating the
|
|
58
|
-
* char-based estimator's drift on dense code/diff content without
|
|
59
|
-
* shipping a tokenizer.
|
|
60
|
-
*/
|
|
61
|
-
export declare function createStepBudgetGuard(config: StepBudgetGuardConfig): (messages: readonly ModelMessage[], observedInputTokensLastStep?: number) => ModelMessage[] | undefined;
|
|
@@ -1,357 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Per-step context budget guard for the AI-SDK agent loop.
|
|
3
|
-
*
|
|
4
|
-
* Pre-call budgeting (`checkContextBudget` + compaction) runs ONCE before
|
|
5
|
-
* dispatch and only sees the input/session conversation. The AI-SDK tool loop
|
|
6
|
-
* then appends assistant turns and tool results on every step — growth the
|
|
7
|
-
* pre-call pipeline never sees, which is how long agentic runs overflow the
|
|
8
|
-
* model's real context window mid-loop (provider 400s after dozens of tool
|
|
9
|
-
* calls). The googleVertex native loops already guard this via
|
|
10
|
-
* `createContextGuard`; this module brings the AI-SDK path (every provider
|
|
11
|
-
* that delegates to `generateText`) to parity — and goes one step further:
|
|
12
|
-
* instead of stopping the loop, it deterministically reclaims budget so the
|
|
13
|
-
* loop can CONTINUE.
|
|
14
|
-
*
|
|
15
|
-
* Wired in `GenerationHandler.callGenerateText` through
|
|
16
|
-
* `experimental_prepareStep`, whose result may replace the step's `messages`.
|
|
17
|
-
* The guard operates on `ModelMessage[]` natively (no lossy ChatMessage
|
|
18
|
-
* round-trip) and never makes LLM calls:
|
|
19
|
-
*
|
|
20
|
-
* Stage 1 — truncate OLD tool outputs to head/tail previews
|
|
21
|
-
* (`generateToolOutputPreview`), oldest first, outside the
|
|
22
|
-
* protected recent tail.
|
|
23
|
-
* Stage 2 — drop the oldest complete tool exchanges (assistant tool-call
|
|
24
|
-
* message + its following tool-result messages, as a unit, so
|
|
25
|
-
* call/result pairing stays intact), replacing them with a single
|
|
26
|
-
* elision note.
|
|
27
|
-
*
|
|
28
|
-
* The system prompt and tool definitions ride OUTSIDE the step messages (the
|
|
29
|
-
* handler hoists system into generateText's `system` option), so their cost is
|
|
30
|
-
* passed in as `fixedOverheadTokens`. The first user message (the task) and
|
|
31
|
-
* the most recent messages are never touched.
|
|
32
|
-
*/
|
|
33
|
-
import { DEFAULT_CONTEXT_GUARD_RATIO } from "../core/constants.js";
|
|
34
|
-
import { getAvailableInputTokens } from "../constants/contextWindows.js";
|
|
35
|
-
import { estimateTokens, TOKENS_PER_MESSAGE, } from "../utils/tokenEstimation.js";
|
|
36
|
-
import { generateToolOutputPreview } from "./toolOutputLimits.js";
|
|
37
|
-
import { logger } from "../utils/logger.js";
|
|
38
|
-
/** Estimated tokens for a tool definition that fails to serialize. */
|
|
39
|
-
const TOKENS_PER_TOOL_DEFINITION = 200;
|
|
40
|
-
/**
|
|
41
|
-
* Upper bound on the usage-feedback calibration ratio. Real tokenizers count
|
|
42
|
-
* dense code/diff content at up to ~1.3× the char-based estimate; anything
|
|
43
|
-
* far beyond that indicates inconsistent provider usage reporting, and an
|
|
44
|
-
* unbounded ratio would compact the loop into uselessness.
|
|
45
|
-
*/
|
|
46
|
-
const MAX_CALIBRATION_RATIO = 3;
|
|
47
|
-
/** Messages at the end of the conversation the guard never modifies. */
|
|
48
|
-
const PROTECTED_TAIL_MESSAGES = 4;
|
|
49
|
-
/**
|
|
50
|
-
* Fraction of the context window the guard reclaims DOWN TO once it fires.
|
|
51
|
-
*
|
|
52
|
-
* The high-water mark (`thresholdRatio`) decides *when* to act; this low-water
|
|
53
|
-
* mark decides *how far*. Reclaiming only back to the threshold meant the very
|
|
54
|
-
* next step — which appends an assistant turn plus its tool results — crossed
|
|
55
|
-
* it again, so the guard mutated the message prefix on every single step of a
|
|
56
|
-
* long agentic run. Each of those mutations invalidates the Anthropic
|
|
57
|
-
* `cache_control` prefix from the edit point onward (see
|
|
58
|
-
* anthropicCacheBreakpoints), turning a ~0.1x cached read into full-price
|
|
59
|
-
* input every step.
|
|
60
|
-
*
|
|
61
|
-
* One deeper reclaim every N steps saves the same tokens and leaves the prefix
|
|
62
|
-
* stable in between, which is what makes the cache worth having.
|
|
63
|
-
*/
|
|
64
|
-
const CONTEXT_GUARD_LOW_WATER_RATIO = 0.6;
|
|
65
|
-
/** Stage-1 preview budget for an old tool output (bytes). */
|
|
66
|
-
const OLD_TOOL_OUTPUT_PREVIEW_BYTES = 2_048;
|
|
67
|
-
/** Stage-1 preview budget for an old tool output (lines). */
|
|
68
|
-
const OLD_TOOL_OUTPUT_PREVIEW_LINES = 60;
|
|
69
|
-
/**
|
|
70
|
-
* Serialize any ModelMessage content to text for estimation. Tool-call args
|
|
71
|
-
* and tool-result outputs are JSON-stringified; unserializable values fall
|
|
72
|
-
* back to a fixed-size placeholder so estimation never throws.
|
|
73
|
-
*/
|
|
74
|
-
function contentToText(content) {
|
|
75
|
-
if (typeof content === "string") {
|
|
76
|
-
return content;
|
|
77
|
-
}
|
|
78
|
-
try {
|
|
79
|
-
return JSON.stringify(content) ?? "";
|
|
80
|
-
}
|
|
81
|
-
catch {
|
|
82
|
-
return "x".repeat(TOKENS_PER_TOOL_DEFINITION * 4);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
/** Estimate the token cost of a step's message array. */
|
|
86
|
-
export function estimateStepMessagesTokens(messages, provider) {
|
|
87
|
-
let total = 0;
|
|
88
|
-
for (const message of messages) {
|
|
89
|
-
total +=
|
|
90
|
-
estimateTokens(contentToText(message.content), provider) +
|
|
91
|
-
TOKENS_PER_MESSAGE;
|
|
92
|
-
}
|
|
93
|
-
return total;
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Estimate the fixed per-request overhead: hoisted system prompt + tool
|
|
97
|
-
* definitions. Mirrors `checkContextBudget`'s categories for the pieces that
|
|
98
|
-
* do not live in the step messages.
|
|
99
|
-
*/
|
|
100
|
-
export function estimateFixedOverheadTokens(system, tools, provider) {
|
|
101
|
-
let total = system
|
|
102
|
-
? estimateTokens(contentToText(system), provider) + TOKENS_PER_MESSAGE
|
|
103
|
-
: 0;
|
|
104
|
-
for (const tool of Object.values(tools ?? {})) {
|
|
105
|
-
try {
|
|
106
|
-
total += estimateTokens(JSON.stringify(tool) ?? "", provider);
|
|
107
|
-
}
|
|
108
|
-
catch {
|
|
109
|
-
total += TOKENS_PER_TOOL_DEFINITION;
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
return total;
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Serialize a ToolResultOutput to the text the MODEL should see in a preview.
|
|
116
|
-
* Variant-aware: `text`/`error-text` carry their payload in `.value` directly —
|
|
117
|
-
* stringifying the wrapper would put escaped `{"type":"text","value":…}` JSON
|
|
118
|
-
* in front of the model instead of the actual output. `json`/`error-json`/
|
|
119
|
-
* `content` serialize their value; unknown shapes fall back to the wrapper.
|
|
120
|
-
*/
|
|
121
|
-
function toolResultOutputToText(output) {
|
|
122
|
-
const variant = output;
|
|
123
|
-
if (variant && typeof variant === "object" && "type" in variant) {
|
|
124
|
-
if ((variant.type === "text" || variant.type === "error-text") &&
|
|
125
|
-
typeof variant.value === "string") {
|
|
126
|
-
return variant.value;
|
|
127
|
-
}
|
|
128
|
-
if (variant.type === "json" ||
|
|
129
|
-
variant.type === "error-json" ||
|
|
130
|
-
variant.type === "content") {
|
|
131
|
-
return contentToText(variant.value);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
return contentToText(output);
|
|
135
|
-
}
|
|
136
|
-
/** True when the message is an assistant message that issues tool calls. */
|
|
137
|
-
function isToolCallAssistantMessage(message) {
|
|
138
|
-
return (message.role === "assistant" &&
|
|
139
|
-
Array.isArray(message.content) &&
|
|
140
|
-
message.content.some((part) => part?.type === "tool-call"));
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Stage 1: replace large tool-result outputs outside the protected tail with
|
|
144
|
-
* head/tail previews. Returns the new array plus how many outputs shrank.
|
|
145
|
-
*/
|
|
146
|
-
function truncateOldToolOutputs(messages) {
|
|
147
|
-
const cutoff = Math.max(0, messages.length - PROTECTED_TAIL_MESSAGES);
|
|
148
|
-
let truncated = 0;
|
|
149
|
-
const next = messages.map((message, index) => {
|
|
150
|
-
if (index >= cutoff || message.role !== "tool") {
|
|
151
|
-
return message;
|
|
152
|
-
}
|
|
153
|
-
if (!Array.isArray(message.content)) {
|
|
154
|
-
return message;
|
|
155
|
-
}
|
|
156
|
-
let changed = false;
|
|
157
|
-
const content = message.content.map((part) => {
|
|
158
|
-
const resultPart = part;
|
|
159
|
-
if (resultPart?.type !== "tool-result") {
|
|
160
|
-
return part;
|
|
161
|
-
}
|
|
162
|
-
const serialized = toolResultOutputToText(resultPart.output);
|
|
163
|
-
if (serialized.length <= OLD_TOOL_OUTPUT_PREVIEW_BYTES) {
|
|
164
|
-
return part;
|
|
165
|
-
}
|
|
166
|
-
const { preview } = generateToolOutputPreview(serialized, {
|
|
167
|
-
maxBytes: OLD_TOOL_OUTPUT_PREVIEW_BYTES,
|
|
168
|
-
maxLines: OLD_TOOL_OUTPUT_PREVIEW_LINES,
|
|
169
|
-
});
|
|
170
|
-
changed = true;
|
|
171
|
-
truncated += 1;
|
|
172
|
-
return {
|
|
173
|
-
...resultPart,
|
|
174
|
-
output: { type: "text", value: preview },
|
|
175
|
-
};
|
|
176
|
-
});
|
|
177
|
-
return changed ? { ...message, content } : message;
|
|
178
|
-
});
|
|
179
|
-
return { messages: next, truncated };
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Stage 2: drop the oldest complete tool exchanges — an assistant tool-call
|
|
183
|
-
* message together with ALL directly-following `tool` messages — until the
|
|
184
|
-
* estimate fits or only the protected head/tail remains. The first
|
|
185
|
-
* non-assistant message run (the task) is never dropped. A single elision
|
|
186
|
-
* note replaces everything removed so the model knows history was elided.
|
|
187
|
-
*/
|
|
188
|
-
function dropOldestToolExchanges(messages, budgetTokens, fixedOverheadTokens, provider) {
|
|
189
|
-
const result = [...messages];
|
|
190
|
-
let droppedExchanges = 0;
|
|
191
|
-
// Running-total accounting: estimate each message ONCE, keep the estimates
|
|
192
|
-
// array in lockstep with `result`, and subtract dropped blocks — instead of
|
|
193
|
-
// re-estimating the whole array on every iteration (O(n²) with many drops).
|
|
194
|
-
const estimates = result.map((message) => estimateTokens(contentToText(message.content), provider) +
|
|
195
|
-
TOKENS_PER_MESSAGE);
|
|
196
|
-
let currentTokens = fixedOverheadTokens + estimates.reduce((sum, tokens) => sum + tokens, 0);
|
|
197
|
-
while (currentTokens > budgetTokens) {
|
|
198
|
-
// Find the FIRST (oldest) droppable exchange outside the protected tail.
|
|
199
|
-
const tailStart = Math.max(0, result.length - PROTECTED_TAIL_MESSAGES);
|
|
200
|
-
let exchangeStart = -1;
|
|
201
|
-
for (let i = 0; i < tailStart; i++) {
|
|
202
|
-
if (isToolCallAssistantMessage(result[i])) {
|
|
203
|
-
exchangeStart = i;
|
|
204
|
-
break;
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
if (exchangeStart === -1) {
|
|
208
|
-
break; // nothing left the guard is allowed to drop
|
|
209
|
-
}
|
|
210
|
-
let exchangeEnd = exchangeStart + 1;
|
|
211
|
-
while (exchangeEnd < result.length && result[exchangeEnd].role === "tool") {
|
|
212
|
-
exchangeEnd++;
|
|
213
|
-
}
|
|
214
|
-
if (exchangeEnd > tailStart) {
|
|
215
|
-
// The oldest remaining exchange bleeds into the protected tail. Because
|
|
216
|
-
// the scan is oldest-first, every exchange after this one STARTS inside
|
|
217
|
-
// the tail (this one's result chain reaches it), and every exchange
|
|
218
|
-
// before it was already dropped by earlier iterations — so there is
|
|
219
|
-
// nothing else the guard may remove. Stop.
|
|
220
|
-
break;
|
|
221
|
-
}
|
|
222
|
-
const dropped = estimates
|
|
223
|
-
.slice(exchangeStart, exchangeEnd)
|
|
224
|
-
.reduce((sum, tokens) => sum + tokens, 0);
|
|
225
|
-
result.splice(exchangeStart, exchangeEnd - exchangeStart);
|
|
226
|
-
estimates.splice(exchangeStart, exchangeEnd - exchangeStart);
|
|
227
|
-
currentTokens -= dropped;
|
|
228
|
-
droppedExchanges++;
|
|
229
|
-
}
|
|
230
|
-
if (droppedExchanges > 0) {
|
|
231
|
-
// Insert one elision note where history was removed: after the leading
|
|
232
|
-
// non-exchange messages (typically the first user/task message), but
|
|
233
|
-
// never after the protected tail — when every droppable exchange was
|
|
234
|
-
// removed, an uncapped scan would append the note at the END, where the
|
|
235
|
-
// "history was removed" cue lands after the content it refers to.
|
|
236
|
-
let noteIndex = 0;
|
|
237
|
-
while (noteIndex < result.length &&
|
|
238
|
-
!isToolCallAssistantMessage(result[noteIndex])) {
|
|
239
|
-
noteIndex++;
|
|
240
|
-
}
|
|
241
|
-
const tailBoundary = Math.max(0, result.length - PROTECTED_TAIL_MESSAGES);
|
|
242
|
-
result.splice(Math.min(noteIndex, tailBoundary), 0, {
|
|
243
|
-
role: "user",
|
|
244
|
-
content: [
|
|
245
|
-
{
|
|
246
|
-
type: "text",
|
|
247
|
-
text: `[context truncated: ${droppedExchanges} earlier tool exchange(s) were removed to fit the model's context window. Continue from the remaining context.]`,
|
|
248
|
-
},
|
|
249
|
-
],
|
|
250
|
-
});
|
|
251
|
-
}
|
|
252
|
-
return { messages: result, droppedExchanges };
|
|
253
|
-
}
|
|
254
|
-
/**
|
|
255
|
-
* Create a per-step budget guard. Returns a function that, given the step's
|
|
256
|
-
* messages (and optionally the REAL input-token count the provider reported
|
|
257
|
-
* for the previous step), returns a compacted replacement array when the
|
|
258
|
-
* projected request exceeds the threshold — or `undefined` when no change is
|
|
259
|
-
* needed.
|
|
260
|
-
*
|
|
261
|
-
* Two dynamic behaviours:
|
|
262
|
-
* - the available-input budget is re-resolved on EVERY invocation, so
|
|
263
|
-
* runtime window discovery (`/model/info`, overflow self-healing) that
|
|
264
|
-
* lands mid-loop takes effect immediately instead of the guard staying
|
|
265
|
-
* frozen on the value captured at loop start;
|
|
266
|
-
* - usage feedback calibrates the estimator: the ratio between the real
|
|
267
|
-
* prompt tokens of the previous step and this guard's own estimate for
|
|
268
|
-
* what that step sent scales later estimates (only UP — underestimates
|
|
269
|
-
* overflow, overestimates merely compact earlier), eliminating the
|
|
270
|
-
* char-based estimator's drift on dense code/diff content without
|
|
271
|
-
* shipping a tokenizer.
|
|
272
|
-
*/
|
|
273
|
-
export function createStepBudgetGuard(config) {
|
|
274
|
-
const { provider, model, maxTokens, fixedOverheadTokens = 0, getFixedOverheadTokens, thresholdRatio = DEFAULT_CONTEXT_GUARD_RATIO, } = config;
|
|
275
|
-
// Calibration state: raw estimate for the messages the PREVIOUS guard
|
|
276
|
-
// invocation let through (what was actually sent), and the current ratio.
|
|
277
|
-
let lastRawEstimate = 0;
|
|
278
|
-
let calibration = 1;
|
|
279
|
-
return function guardStepMessages(messages, observedInputTokensLastStep) {
|
|
280
|
-
const availableInput = getAvailableInputTokens(provider, model, maxTokens);
|
|
281
|
-
const thresholdTokens = Math.floor(availableInput * thresholdRatio);
|
|
282
|
-
// Resolve overhead per invocation: the tool set can GROW mid-loop
|
|
283
|
-
// (search_tools hydration adds discovered tools between steps), so a
|
|
284
|
-
// once-captured value would undercount later steps.
|
|
285
|
-
const overheadTokens = getFixedOverheadTokens?.() ?? fixedOverheadTokens;
|
|
286
|
-
const rawEstimate = overheadTokens + estimateStepMessagesTokens(messages, provider);
|
|
287
|
-
if (observedInputTokensLastStep !== undefined &&
|
|
288
|
-
observedInputTokensLastStep > 0 &&
|
|
289
|
-
lastRawEstimate > 0) {
|
|
290
|
-
calibration = Math.min(MAX_CALIBRATION_RATIO, Math.max(1, observedInputTokensLastStep / lastRawEstimate));
|
|
291
|
-
}
|
|
292
|
-
// Apply calibration to the THRESHOLD instead of every estimate so the
|
|
293
|
-
// compaction stages keep operating on raw numbers.
|
|
294
|
-
const effectiveThreshold = Math.floor(thresholdTokens / calibration);
|
|
295
|
-
// Logger Guard: per-step diagnostics for debugging why a long run does
|
|
296
|
-
// (or does not) trigger compaction — gated so nothing is serialized when
|
|
297
|
-
// debug logging is off.
|
|
298
|
-
if (logger.shouldLog("debug")) {
|
|
299
|
-
logger.debug("[StepBudgetGuard] step estimate", {
|
|
300
|
-
provider,
|
|
301
|
-
model,
|
|
302
|
-
messageCount: messages.length,
|
|
303
|
-
estimatedTokens: rawEstimate,
|
|
304
|
-
thresholdTokens: effectiveThreshold,
|
|
305
|
-
calibration,
|
|
306
|
-
observedInputTokensLastStep,
|
|
307
|
-
willCompact: rawEstimate > effectiveThreshold,
|
|
308
|
-
});
|
|
309
|
-
}
|
|
310
|
-
if (rawEstimate <= effectiveThreshold) {
|
|
311
|
-
lastRawEstimate = rawEstimate;
|
|
312
|
-
return undefined;
|
|
313
|
-
}
|
|
314
|
-
// Reclaim down to the LOW-WATER mark, not merely back under the threshold.
|
|
315
|
-
// See CONTEXT_GUARD_LOW_WATER_RATIO: stopping at the threshold guaranteed
|
|
316
|
-
// the next step crossed it again, mutating the cached prefix every step.
|
|
317
|
-
const lowWaterTokens = Math.floor((availableInput * CONTEXT_GUARD_LOW_WATER_RATIO) / calibration);
|
|
318
|
-
// Stage 1: shrink old tool outputs to previews.
|
|
319
|
-
const stage1 = truncateOldToolOutputs([...messages]);
|
|
320
|
-
let compacted = stage1.messages;
|
|
321
|
-
let newEstimate = overheadTokens + estimateStepMessagesTokens(compacted, provider);
|
|
322
|
-
// Stage 2: drop oldest complete tool exchanges until under the low-water
|
|
323
|
-
// mark. Stage order is deliberately unchanged — truncating first preserves
|
|
324
|
-
// a preview of each output, and since BOTH stages edit the oldest messages
|
|
325
|
-
// the cache prefix is invalidated at roughly the same point either way.
|
|
326
|
-
// Frequency, not stage order, is what governs cache retention here.
|
|
327
|
-
let droppedExchanges = 0;
|
|
328
|
-
if (newEstimate > lowWaterTokens) {
|
|
329
|
-
const stage2 = dropOldestToolExchanges(compacted, lowWaterTokens, overheadTokens, provider);
|
|
330
|
-
compacted = stage2.messages;
|
|
331
|
-
droppedExchanges = stage2.droppedExchanges;
|
|
332
|
-
newEstimate =
|
|
333
|
-
overheadTokens + estimateStepMessagesTokens(compacted, provider);
|
|
334
|
-
}
|
|
335
|
-
if (stage1.truncated === 0 && droppedExchanges === 0) {
|
|
336
|
-
lastRawEstimate = rawEstimate;
|
|
337
|
-
return undefined; // nothing actionable (already all-protected)
|
|
338
|
-
}
|
|
339
|
-
logger.info("[StepBudgetGuard] Compacted agent-loop step messages", {
|
|
340
|
-
provider,
|
|
341
|
-
model,
|
|
342
|
-
estimatedTokens: rawEstimate,
|
|
343
|
-
thresholdTokens: effectiveThreshold,
|
|
344
|
-
lowWaterTokens,
|
|
345
|
-
calibration,
|
|
346
|
-
afterTokens: newEstimate,
|
|
347
|
-
// Headroom reclaimed below the firing threshold. Roughly how many further
|
|
348
|
-
// steps can run before the guard mutates the prefix again — a value near
|
|
349
|
-
// zero means the cache is being invalidated every step.
|
|
350
|
-
headroomTokens: effectiveThreshold - newEstimate,
|
|
351
|
-
toolOutputsTruncated: stage1.truncated,
|
|
352
|
-
exchangesDropped: droppedExchanges,
|
|
353
|
-
});
|
|
354
|
-
lastRawEstimate = newEstimate;
|
|
355
|
-
return compacted;
|
|
356
|
-
};
|
|
357
|
-
}
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Service Registry for Dependency Injection
|
|
3
|
-
* Breaks circular dependencies by providing lazy loading and centralized service management
|
|
4
|
-
*/
|
|
5
|
-
import type { ServiceFactory } from "../types/index.js";
|
|
6
|
-
export declare class ServiceRegistry {
|
|
7
|
-
private static services;
|
|
8
|
-
private static initializing;
|
|
9
|
-
/**
|
|
10
|
-
* Register a service with optional singleton behavior
|
|
11
|
-
*/
|
|
12
|
-
static register<T>(name: string, factory: ServiceFactory<T>, options?: {
|
|
13
|
-
singleton?: boolean;
|
|
14
|
-
}): void;
|
|
15
|
-
/**
|
|
16
|
-
* Get a service instance with circular dependency detection
|
|
17
|
-
*/
|
|
18
|
-
static get<T>(name: string): Promise<T>;
|
|
19
|
-
/**
|
|
20
|
-
* Get a service synchronously (throws if async initialization required)
|
|
21
|
-
*/
|
|
22
|
-
static getSync<T>(name: string): T;
|
|
23
|
-
/**
|
|
24
|
-
* Check if a service is registered
|
|
25
|
-
*/
|
|
26
|
-
static has(name: string): boolean;
|
|
27
|
-
/**
|
|
28
|
-
* Clear all services (useful for testing)
|
|
29
|
-
*/
|
|
30
|
-
static clear(): void;
|
|
31
|
-
/**
|
|
32
|
-
* Get all registered service names
|
|
33
|
-
*/
|
|
34
|
-
static getRegisteredServices(): string[];
|
|
35
|
-
/**
|
|
36
|
-
* Register multiple services at once
|
|
37
|
-
*/
|
|
38
|
-
static registerBatch(services: Record<string, ServiceFactory>): void;
|
|
39
|
-
}
|
|
40
|
-
export declare const serviceRegistry: typeof ServiceRegistry;
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Service Registry for Dependency Injection
|
|
3
|
-
* Breaks circular dependencies by providing lazy loading and centralized service management
|
|
4
|
-
*/
|
|
5
|
-
import { logger } from "../utils/logger.js";
|
|
6
|
-
export class ServiceRegistry {
|
|
7
|
-
static services = new Map();
|
|
8
|
-
static initializing = new Set();
|
|
9
|
-
/**
|
|
10
|
-
* Register a service with optional singleton behavior
|
|
11
|
-
*/
|
|
12
|
-
static register(name, factory, options = {}) {
|
|
13
|
-
if (this.services.has(name)) {
|
|
14
|
-
logger.warn(`Service ${name} is already registered. Overwriting.`);
|
|
15
|
-
}
|
|
16
|
-
this.services.set(name, {
|
|
17
|
-
factory,
|
|
18
|
-
singleton: options.singleton ?? true,
|
|
19
|
-
instance: undefined,
|
|
20
|
-
});
|
|
21
|
-
logger.debug(`Service registered: ${name} (singleton: ${options.singleton ?? true})`);
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Get a service instance with circular dependency detection
|
|
25
|
-
*/
|
|
26
|
-
static async get(name) {
|
|
27
|
-
const registration = this.services.get(name);
|
|
28
|
-
if (!registration) {
|
|
29
|
-
throw new Error(`Service ${name} not registered. Available services: ${Array.from(this.services.keys()).join(", ")}`);
|
|
30
|
-
}
|
|
31
|
-
// Check for circular dependency
|
|
32
|
-
if (this.initializing.has(name)) {
|
|
33
|
-
throw new Error(`Circular dependency detected: ${name} is already being initialized. Chain: ${Array.from(this.initializing).join(" -> ")} -> ${name}`);
|
|
34
|
-
}
|
|
35
|
-
// Return existing singleton instance if available
|
|
36
|
-
if (registration.singleton && registration.instance !== undefined) {
|
|
37
|
-
return registration.instance;
|
|
38
|
-
}
|
|
39
|
-
try {
|
|
40
|
-
// Mark as initializing to detect circular dependencies
|
|
41
|
-
this.initializing.add(name);
|
|
42
|
-
logger.debug(`Initializing service: ${name}`);
|
|
43
|
-
// Create new instance
|
|
44
|
-
const instance = await registration.factory();
|
|
45
|
-
// Store singleton instance
|
|
46
|
-
if (registration.singleton) {
|
|
47
|
-
registration.instance = instance;
|
|
48
|
-
}
|
|
49
|
-
logger.debug(`Service initialized: ${name}`);
|
|
50
|
-
return instance;
|
|
51
|
-
}
|
|
52
|
-
catch (error) {
|
|
53
|
-
logger.error(`Failed to initialize service ${name}:`, error);
|
|
54
|
-
throw error;
|
|
55
|
-
}
|
|
56
|
-
finally {
|
|
57
|
-
// Remove from initializing set
|
|
58
|
-
this.initializing.delete(name);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Get a service synchronously (throws if async initialization required)
|
|
63
|
-
*/
|
|
64
|
-
static getSync(name) {
|
|
65
|
-
const registration = this.services.get(name);
|
|
66
|
-
if (!registration) {
|
|
67
|
-
throw new Error(`Service ${name} not registered`);
|
|
68
|
-
}
|
|
69
|
-
if (registration.singleton && registration.instance !== undefined) {
|
|
70
|
-
return registration.instance;
|
|
71
|
-
}
|
|
72
|
-
// Try synchronous initialization
|
|
73
|
-
const result = registration.factory();
|
|
74
|
-
if (result instanceof Promise) {
|
|
75
|
-
throw new Error(`Service ${name} requires asynchronous initialization. Use get() instead.`);
|
|
76
|
-
}
|
|
77
|
-
if (registration.singleton) {
|
|
78
|
-
registration.instance = result;
|
|
79
|
-
}
|
|
80
|
-
return result;
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Check if a service is registered
|
|
84
|
-
*/
|
|
85
|
-
static has(name) {
|
|
86
|
-
return this.services.has(name);
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Clear all services (useful for testing)
|
|
90
|
-
*/
|
|
91
|
-
static clear() {
|
|
92
|
-
this.services.clear();
|
|
93
|
-
this.initializing.clear();
|
|
94
|
-
logger.debug("Service registry cleared");
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* Get all registered service names
|
|
98
|
-
*/
|
|
99
|
-
static getRegisteredServices() {
|
|
100
|
-
return Array.from(this.services.keys());
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* Register multiple services at once
|
|
104
|
-
*/
|
|
105
|
-
static registerBatch(services) {
|
|
106
|
-
for (const [name, factory] of Object.entries(services)) {
|
|
107
|
-
this.register(name, factory);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
// Export singleton instance for convenience
|
|
112
|
-
export const serviceRegistry = ServiceRegistry;
|