@juspay/neurolink 10.10.4 → 10.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +387 -386
- package/dist/context/anthropicLoopGuard.d.ts +46 -0
- package/dist/context/anthropicLoopGuard.js +167 -0
- package/dist/context/loopGuardCore.d.ts +43 -0
- package/dist/context/loopGuardCore.js +145 -0
- package/dist/context/openaiCompatLoopGuard.d.ts +42 -0
- package/dist/context/openaiCompatLoopGuard.js +172 -0
- package/dist/context/toolOutputLimits.d.ts +11 -0
- package/dist/context/toolOutputLimits.js +16 -0
- package/dist/lib/context/anthropicLoopGuard.d.ts +46 -0
- package/dist/lib/context/anthropicLoopGuard.js +168 -0
- package/dist/lib/context/loopGuardCore.d.ts +43 -0
- package/dist/lib/context/loopGuardCore.js +146 -0
- package/dist/lib/context/openaiCompatLoopGuard.d.ts +42 -0
- package/dist/lib/context/openaiCompatLoopGuard.js +173 -0
- package/dist/lib/context/toolOutputLimits.d.ts +11 -0
- package/dist/lib/context/toolOutputLimits.js +16 -0
- package/dist/lib/providers/anthropic/client.js +98 -0
- package/dist/lib/providers/openaiChatCompletionsBase.js +38 -1
- package/dist/lib/types/context.d.ts +73 -0
- package/dist/providers/anthropic/client.js +98 -0
- package/dist/providers/openaiChatCompletionsBase.js +38 -1
- package/dist/types/context.d.ts +73 -0
- package/package.json +4 -1
|
@@ -14,6 +14,9 @@ import { createProxyFetch } from "../../proxy/proxyFetch.js";
|
|
|
14
14
|
import { getCapturedLimitSnapshot, getCapturedResponseHeaders, logClaudeLimitSnapshot, runInLimitCaptureScope, setLimitSpanAttributes, withLimitCapture, wrapFetchWithLimitCapture, } from "./rateLimitCapture.js";
|
|
15
15
|
import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
16
16
|
import { logger } from "../../utils/logger.js";
|
|
17
|
+
import { ANTHROPIC_ELISION_NOTE, planAnthropicLoopReclaim, previewAnthropicToolResultText, } from "../../context/anthropicLoopGuard.js";
|
|
18
|
+
import { getAvailableInputTokens } from "../../constants/contextWindows.js";
|
|
19
|
+
import { estimateTokens } from "../../utils/tokenEstimation.js";
|
|
17
20
|
import { redactUrlCredentials } from "../../utils/logSanitize.js";
|
|
18
21
|
import { ANTHROPIC_MAX_CACHE_BREAKPOINTS, applyAnthropicHistoryCacheBreakpoints, countAnthropicCacheMarkers, } from "../../utils/anthropicCacheBreakpoints.js";
|
|
19
22
|
import { calculateCost } from "../../utils/pricing.js";
|
|
@@ -1509,8 +1512,30 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1509
1512
|
// and stay fully incremental.
|
|
1510
1513
|
let bufferedText = "";
|
|
1511
1514
|
let finalResultText;
|
|
1515
|
+
/** System prompt + tool definitions: they ride outside `messages`. */
|
|
1516
|
+
const estimateAnthropicFixedOverhead = (system, tools) => {
|
|
1517
|
+
const text = (value) => {
|
|
1518
|
+
if (typeof value === "string") {
|
|
1519
|
+
return value;
|
|
1520
|
+
}
|
|
1521
|
+
try {
|
|
1522
|
+
return JSON.stringify(value) ?? "";
|
|
1523
|
+
}
|
|
1524
|
+
catch {
|
|
1525
|
+
return "";
|
|
1526
|
+
}
|
|
1527
|
+
};
|
|
1528
|
+
return (estimateTokens(text(system), "anthropic") +
|
|
1529
|
+
estimateTokens(text(tools), "anthropic"));
|
|
1530
|
+
};
|
|
1512
1531
|
const runLoop = async () => {
|
|
1513
1532
|
const conversation = payload.messages.slice();
|
|
1533
|
+
// The provider's REAL prompt-token count for the previous step,
|
|
1534
|
+
// calibrating the guard's char-based estimate for free, paired with the
|
|
1535
|
+
// guard's own estimate for that same request — a ratio between counts of
|
|
1536
|
+
// two different payloads would be meaningless.
|
|
1537
|
+
let lastObservedPromptTokens;
|
|
1538
|
+
let lastSentEstimate;
|
|
1514
1539
|
for (let step = 0; step < maxSteps; step++) {
|
|
1515
1540
|
// Mid-turn discovery sync: search_tools (tools.discovery) hydrates
|
|
1516
1541
|
// new tools into toolsRecord between steps; Claude only calls tools
|
|
@@ -1524,6 +1549,71 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1524
1549
|
logger.info(`[Anthropic] ${Object.keys(hydrated).length} tool(s) hydrated mid-turn via discovery: ${Object.keys(hydrated).join(", ")}`);
|
|
1525
1550
|
}
|
|
1526
1551
|
}
|
|
1552
|
+
// In-turn context guard. This loop appends an assistant tool_use
|
|
1553
|
+
// message plus a user tool_result message every step — growth the
|
|
1554
|
+
// pre-dispatch budget check never sees. Without it a long agentic run
|
|
1555
|
+
// overflows the window mid-loop and loses every completed step.
|
|
1556
|
+
// Returns undefined while the request still fits, leaving the history
|
|
1557
|
+
// byte-identical so the rolling cache prefix below stays valid.
|
|
1558
|
+
const reclaim = planAnthropicLoopReclaim({
|
|
1559
|
+
conversation,
|
|
1560
|
+
availableInputTokens: getAvailableInputTokens("anthropic", modelId, options.maxTokens ?? undefined),
|
|
1561
|
+
fixedOverheadTokens: estimateAnthropicFixedOverhead(payload.system, anthropicTools),
|
|
1562
|
+
provider: "anthropic",
|
|
1563
|
+
observedPromptTokens: lastObservedPromptTokens,
|
|
1564
|
+
// Both halves of the calibration ratio must describe the same
|
|
1565
|
+
// request: the tokens the provider reported, and this guard's own
|
|
1566
|
+
// estimate for what was sent to earn them.
|
|
1567
|
+
previousSentEstimate: lastSentEstimate,
|
|
1568
|
+
onSentEstimate: (tokens) => {
|
|
1569
|
+
lastSentEstimate = tokens;
|
|
1570
|
+
},
|
|
1571
|
+
});
|
|
1572
|
+
if (reclaim) {
|
|
1573
|
+
// Applied HERE, in the loop's own concrete types: the guard decides,
|
|
1574
|
+
// the caller mutates. Dropping an assistant tool_use message together
|
|
1575
|
+
// with its user tool_result message is what keeps blocks paired.
|
|
1576
|
+
const dropSet = new Set(reclaim.drop);
|
|
1577
|
+
const truncateSet = new Set(reclaim.truncate);
|
|
1578
|
+
const rebuilt = [];
|
|
1579
|
+
for (let i = 0; i < conversation.length; i++) {
|
|
1580
|
+
if (dropSet.has(i)) {
|
|
1581
|
+
continue;
|
|
1582
|
+
}
|
|
1583
|
+
const message = conversation[i];
|
|
1584
|
+
if (truncateSet.has(i) && Array.isArray(message.content)) {
|
|
1585
|
+
rebuilt.push({
|
|
1586
|
+
...message,
|
|
1587
|
+
content: message.content.map((block) => block.type === "tool_result"
|
|
1588
|
+
? {
|
|
1589
|
+
...block,
|
|
1590
|
+
content: previewAnthropicToolResultText(typeof block.content === "string"
|
|
1591
|
+
? block.content
|
|
1592
|
+
: (JSON.stringify(block.content) ?? "")),
|
|
1593
|
+
}
|
|
1594
|
+
: block),
|
|
1595
|
+
});
|
|
1596
|
+
continue;
|
|
1597
|
+
}
|
|
1598
|
+
rebuilt.push(message);
|
|
1599
|
+
}
|
|
1600
|
+
if (dropSet.size > 0) {
|
|
1601
|
+
// Anthropic requires user/assistant alternation around tool blocks;
|
|
1602
|
+
// the note is a user turn placed immediately before the first
|
|
1603
|
+
// surviving assistant tool_use turn, which preserves it.
|
|
1604
|
+
let noteIndex = rebuilt.findIndex((m) => Array.isArray(m.content) &&
|
|
1605
|
+
m.content.some((b) => b.type === "tool_use" || b.type === "tool_result"));
|
|
1606
|
+
if (noteIndex < 0) {
|
|
1607
|
+
noteIndex = Math.min(1, rebuilt.length);
|
|
1608
|
+
}
|
|
1609
|
+
rebuilt.splice(noteIndex, 0, {
|
|
1610
|
+
role: "user",
|
|
1611
|
+
content: [{ type: "text", text: ANTHROPIC_ELISION_NOTE }],
|
|
1612
|
+
});
|
|
1613
|
+
}
|
|
1614
|
+
conversation.length = 0;
|
|
1615
|
+
conversation.push(...rebuilt);
|
|
1616
|
+
}
|
|
1527
1617
|
// Prompt-cache parity with the native Vertex+Claude path — rolling
|
|
1528
1618
|
// history breakpoints, re-applied per step so the stable prefix
|
|
1529
1619
|
// stays byte-identical while the breakpoint follows the growing
|
|
@@ -1587,6 +1677,14 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1587
1677
|
totalCacheRead += event.message.usage.cache_read_input_tokens ?? 0;
|
|
1588
1678
|
totalCacheWrite +=
|
|
1589
1679
|
event.message.usage.cache_creation_input_tokens ?? 0;
|
|
1680
|
+
// Calibration signal for the in-turn guard: the FULL prompt size,
|
|
1681
|
+
// which on this path means uncached input plus both cache tiers.
|
|
1682
|
+
// Using input_tokens alone would read a cache-hit step as tiny and
|
|
1683
|
+
// let the guard drift far under the real cost.
|
|
1684
|
+
lastObservedPromptTokens =
|
|
1685
|
+
(event.message.usage.input_tokens ?? 0) +
|
|
1686
|
+
(event.message.usage.cache_read_input_tokens ?? 0) +
|
|
1687
|
+
(event.message.usage.cache_creation_input_tokens ?? 0);
|
|
1590
1688
|
}
|
|
1591
1689
|
else if (event.type === "content_block_start") {
|
|
1592
1690
|
blockTypes.set(event.index, event.content_block.type);
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
* Nothing here imports from "ai" or "@ai-sdk/*". The base class is a
|
|
18
18
|
* direct HTTP client + multi-step tool-execution loop driven by SSE.
|
|
19
19
|
*/
|
|
20
|
-
import { getRuntimeContextWindow, getRuntimeOutputCeiling, registerRuntimeContextWindow, } from "../constants/contextWindows.js";
|
|
20
|
+
import { getAvailableInputTokens, getRuntimeContextWindow, getRuntimeOutputCeiling, registerRuntimeContextWindow, } from "../constants/contextWindows.js";
|
|
21
|
+
import { guardOpenAICompatConversation } from "../context/openaiCompatLoopGuard.js";
|
|
21
22
|
import { isContextOverflowError, parseProviderOverflowDetails, } from "../context/errorDetection.js";
|
|
22
23
|
import { ContextBudgetExceededError } from "../context/errors.js";
|
|
23
24
|
import { BaseProvider } from "../core/baseProvider.js";
|
|
@@ -770,6 +771,12 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
770
771
|
// May grow mid-turn: hydrated tools with wire-unsafe names need
|
|
771
772
|
// reverse-mapping even when the initial name set required none.
|
|
772
773
|
let effectiveToolNameFromWire = toolNameFromWire;
|
|
774
|
+
// The provider's REAL prompt-token count for the previous step, used to
|
|
775
|
+
// calibrate the guard's char-based estimate for free, paired with the
|
|
776
|
+
// guard's own estimate for that same request — a ratio between counts of
|
|
777
|
+
// two different payloads would be meaningless.
|
|
778
|
+
let lastObservedPromptTokens;
|
|
779
|
+
let lastSentEstimate;
|
|
773
780
|
for (let step = 0; step < maxSteps; step++) {
|
|
774
781
|
// Mid-turn discovery sync: search_tools (tools.discovery) hydrates
|
|
775
782
|
// new tools into toolsRecord between steps. Dispatch already re-reads
|
|
@@ -795,6 +802,35 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
795
802
|
logger.info(`${this.providerName}: ${Object.keys(hydrated).length} tool(s) hydrated mid-turn via discovery: ${Object.keys(hydrated).join(", ")}`);
|
|
796
803
|
}
|
|
797
804
|
}
|
|
805
|
+
// In-turn context guard. This loop appends an assistant tool-call
|
|
806
|
+
// message plus one tool message per result on every step — growth the
|
|
807
|
+
// pre-dispatch budget check never sees. Without this, a long agentic
|
|
808
|
+
// run walks into a provider "context length exceeded" and loses every
|
|
809
|
+
// completed step. Shares its reclaim policy with the other provider
|
|
810
|
+
// loops via loopGuardCore; returns undefined (leaving `conversation`
|
|
811
|
+
// byte-identical) whenever the request still fits, so a loop that fits
|
|
812
|
+
// never pays a prompt-cache invalidation.
|
|
813
|
+
const guarded = guardOpenAICompatConversation({
|
|
814
|
+
conversation,
|
|
815
|
+
availableInputTokens: getAvailableInputTokens(this.providerName, modelId, options.maxTokens ?? undefined),
|
|
816
|
+
// Tool definitions ride outside the message array. Passing an empty
|
|
817
|
+
// message list yields the tools-only overhead, and reuses the same
|
|
818
|
+
// estimator the wire path already trusts.
|
|
819
|
+
fixedOverheadTokens: estimateWireTokens([], openAITools, this.providerName),
|
|
820
|
+
provider: this.providerName,
|
|
821
|
+
observedPromptTokens: lastObservedPromptTokens,
|
|
822
|
+
// Both halves of the calibration ratio must describe the same
|
|
823
|
+
// request: the tokens the provider reported, and this guard's own
|
|
824
|
+
// estimate for what was sent to earn them.
|
|
825
|
+
previousSentEstimate: lastSentEstimate,
|
|
826
|
+
onSentEstimate: (tokens) => {
|
|
827
|
+
lastSentEstimate = tokens;
|
|
828
|
+
},
|
|
829
|
+
});
|
|
830
|
+
if (guarded) {
|
|
831
|
+
conversation.length = 0;
|
|
832
|
+
conversation.push(...guarded);
|
|
833
|
+
}
|
|
798
834
|
const stepResult = await this.streamOneStep({
|
|
799
835
|
modelId,
|
|
800
836
|
url,
|
|
@@ -806,6 +842,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
806
842
|
openAIToolChoice,
|
|
807
843
|
pushChunk,
|
|
808
844
|
});
|
|
845
|
+
lastObservedPromptTokens = stepResult.usage?.prompt_tokens;
|
|
809
846
|
stepFinish = stepResult.finishReason;
|
|
810
847
|
if (stepResult.usage) {
|
|
811
848
|
stepUsage = mergeUsage(stepUsage, stepResult.usage);
|
|
@@ -412,6 +412,79 @@ export type RepairResult = {
|
|
|
412
412
|
orphanedCallsFixed: number;
|
|
413
413
|
orphanedResultsFixed: number;
|
|
414
414
|
};
|
|
415
|
+
/**
|
|
416
|
+
* Provider-neutral view of ONE message in an agent loop's history.
|
|
417
|
+
*
|
|
418
|
+
* Every native provider loop keeps its history in a different shape (AI-SDK
|
|
419
|
+
* `ModelMessage`, OpenAI-compatible `{role,tool_calls}`, Gemini `contents`
|
|
420
|
+
* parts, Anthropic content blocks). The reclaim POLICY is identical across all
|
|
421
|
+
* of them, so adapters map their own shape onto this view, ask the core what to
|
|
422
|
+
* do, and apply the answer themselves.
|
|
423
|
+
*/
|
|
424
|
+
export type LoopGuardEntry = {
|
|
425
|
+
/** `toolCall` and `toolResult` form the batches the policy keeps intact. */
|
|
426
|
+
kind: "other" | "toolCall" | "toolResult";
|
|
427
|
+
/** Estimated tokens this entry currently costs. */
|
|
428
|
+
tokens: number;
|
|
429
|
+
/**
|
|
430
|
+
* Tokens this entry would cost with its payload replaced by a head/tail
|
|
431
|
+
* preview. Omitted when the entry cannot usefully shrink — which is exactly
|
|
432
|
+
* the case that forces the policy to drop batches instead.
|
|
433
|
+
*/
|
|
434
|
+
previewTokens?: number;
|
|
435
|
+
};
|
|
436
|
+
/** What the caller should do to reclaim budget. Indices refer to the input array. */
|
|
437
|
+
export type LoopGuardPlan = {
|
|
438
|
+
/** False when the loop is under threshold and nothing should change. */
|
|
439
|
+
fire: boolean;
|
|
440
|
+
/** Entries whose payload should be replaced by a preview. */
|
|
441
|
+
truncate: number[];
|
|
442
|
+
/** Entries to remove entirely — always whole batches, never a partial pair. */
|
|
443
|
+
drop: number[];
|
|
444
|
+
/** Estimated total after applying the plan, including fixed overhead. */
|
|
445
|
+
projectedTokens: number;
|
|
446
|
+
};
|
|
447
|
+
/**
|
|
448
|
+
* Structural view of one Anthropic content block, loose enough to accept the
|
|
449
|
+
* official SDK's `ContentBlockParam` union and NeuroLink's own
|
|
450
|
+
* `VertexAnthropicMessage` blocks without a cast at either call site.
|
|
451
|
+
*/
|
|
452
|
+
export type AnthropicGuardBlock = {
|
|
453
|
+
type: string;
|
|
454
|
+
/** Payload of a `tool_result` block. Other block kinds carry other fields. */
|
|
455
|
+
content?: unknown;
|
|
456
|
+
/** Text of a `text` block. */
|
|
457
|
+
text?: string;
|
|
458
|
+
};
|
|
459
|
+
/**
|
|
460
|
+
* Structural view of one Anthropic-shaped message, as used by both the direct
|
|
461
|
+
* Anthropic loop and the native Vertex+Claude path. Tool calls ride as
|
|
462
|
+
* `tool_use` blocks on an assistant message; their answers ride as
|
|
463
|
+
* `tool_result` blocks on the following user message.
|
|
464
|
+
*/
|
|
465
|
+
export type AnthropicGuardMessage = {
|
|
466
|
+
/**
|
|
467
|
+
* `system` is included because the installed `@anthropic-ai/sdk` widens
|
|
468
|
+
* `MessageParam["role"]` to accept it; narrowing here would make the SDK's
|
|
469
|
+
* own array unassignable at the call site.
|
|
470
|
+
*/
|
|
471
|
+
role: "user" | "assistant" | "system";
|
|
472
|
+
content: string | AnthropicGuardBlock[];
|
|
473
|
+
};
|
|
474
|
+
/** Tuning for {@link planLoopGuardReclaim}. */
|
|
475
|
+
export type LoopGuardPolicy = {
|
|
476
|
+
availableInputTokens: number;
|
|
477
|
+
/** System prompt + tool definitions — rides outside the message array. */
|
|
478
|
+
fixedOverheadTokens: number;
|
|
479
|
+
/** Fraction of the window at which the guard fires. */
|
|
480
|
+
thresholdRatio?: number;
|
|
481
|
+
/** Fraction of the window the guard reclaims down to once it fires. */
|
|
482
|
+
lowWaterRatio?: number;
|
|
483
|
+
/** Newest entries the guard must never modify. */
|
|
484
|
+
protectedTailCount?: number;
|
|
485
|
+
/** Observed/estimated token ratio, used to tighten both marks. */
|
|
486
|
+
calibration?: number;
|
|
487
|
+
};
|
|
415
488
|
/**
|
|
416
489
|
* One contiguous tool batch: the run of `tool_call` messages emitted by a
|
|
417
490
|
* single agent step, plus the run of `tool_result` messages that follows it.
|
|
@@ -14,6 +14,9 @@ import { createProxyFetch } from "../../proxy/proxyFetch.js";
|
|
|
14
14
|
import { getCapturedLimitSnapshot, getCapturedResponseHeaders, logClaudeLimitSnapshot, runInLimitCaptureScope, setLimitSpanAttributes, withLimitCapture, wrapFetchWithLimitCapture, } from "./rateLimitCapture.js";
|
|
15
15
|
import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
16
16
|
import { logger } from "../../utils/logger.js";
|
|
17
|
+
import { ANTHROPIC_ELISION_NOTE, planAnthropicLoopReclaim, previewAnthropicToolResultText, } from "../../context/anthropicLoopGuard.js";
|
|
18
|
+
import { getAvailableInputTokens } from "../../constants/contextWindows.js";
|
|
19
|
+
import { estimateTokens } from "../../utils/tokenEstimation.js";
|
|
17
20
|
import { redactUrlCredentials } from "../../utils/logSanitize.js";
|
|
18
21
|
import { ANTHROPIC_MAX_CACHE_BREAKPOINTS, applyAnthropicHistoryCacheBreakpoints, countAnthropicCacheMarkers, } from "../../utils/anthropicCacheBreakpoints.js";
|
|
19
22
|
import { calculateCost } from "../../utils/pricing.js";
|
|
@@ -1509,8 +1512,30 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1509
1512
|
// and stay fully incremental.
|
|
1510
1513
|
let bufferedText = "";
|
|
1511
1514
|
let finalResultText;
|
|
1515
|
+
/** System prompt + tool definitions: they ride outside `messages`. */
|
|
1516
|
+
const estimateAnthropicFixedOverhead = (system, tools) => {
|
|
1517
|
+
const text = (value) => {
|
|
1518
|
+
if (typeof value === "string") {
|
|
1519
|
+
return value;
|
|
1520
|
+
}
|
|
1521
|
+
try {
|
|
1522
|
+
return JSON.stringify(value) ?? "";
|
|
1523
|
+
}
|
|
1524
|
+
catch {
|
|
1525
|
+
return "";
|
|
1526
|
+
}
|
|
1527
|
+
};
|
|
1528
|
+
return (estimateTokens(text(system), "anthropic") +
|
|
1529
|
+
estimateTokens(text(tools), "anthropic"));
|
|
1530
|
+
};
|
|
1512
1531
|
const runLoop = async () => {
|
|
1513
1532
|
const conversation = payload.messages.slice();
|
|
1533
|
+
// The provider's REAL prompt-token count for the previous step,
|
|
1534
|
+
// calibrating the guard's char-based estimate for free, paired with the
|
|
1535
|
+
// guard's own estimate for that same request — a ratio between counts of
|
|
1536
|
+
// two different payloads would be meaningless.
|
|
1537
|
+
let lastObservedPromptTokens;
|
|
1538
|
+
let lastSentEstimate;
|
|
1514
1539
|
for (let step = 0; step < maxSteps; step++) {
|
|
1515
1540
|
// Mid-turn discovery sync: search_tools (tools.discovery) hydrates
|
|
1516
1541
|
// new tools into toolsRecord between steps; Claude only calls tools
|
|
@@ -1524,6 +1549,71 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1524
1549
|
logger.info(`[Anthropic] ${Object.keys(hydrated).length} tool(s) hydrated mid-turn via discovery: ${Object.keys(hydrated).join(", ")}`);
|
|
1525
1550
|
}
|
|
1526
1551
|
}
|
|
1552
|
+
// In-turn context guard. This loop appends an assistant tool_use
|
|
1553
|
+
// message plus a user tool_result message every step — growth the
|
|
1554
|
+
// pre-dispatch budget check never sees. Without it a long agentic run
|
|
1555
|
+
// overflows the window mid-loop and loses every completed step.
|
|
1556
|
+
// Returns undefined while the request still fits, leaving the history
|
|
1557
|
+
// byte-identical so the rolling cache prefix below stays valid.
|
|
1558
|
+
const reclaim = planAnthropicLoopReclaim({
|
|
1559
|
+
conversation,
|
|
1560
|
+
availableInputTokens: getAvailableInputTokens("anthropic", modelId, options.maxTokens ?? undefined),
|
|
1561
|
+
fixedOverheadTokens: estimateAnthropicFixedOverhead(payload.system, anthropicTools),
|
|
1562
|
+
provider: "anthropic",
|
|
1563
|
+
observedPromptTokens: lastObservedPromptTokens,
|
|
1564
|
+
// Both halves of the calibration ratio must describe the same
|
|
1565
|
+
// request: the tokens the provider reported, and this guard's own
|
|
1566
|
+
// estimate for what was sent to earn them.
|
|
1567
|
+
previousSentEstimate: lastSentEstimate,
|
|
1568
|
+
onSentEstimate: (tokens) => {
|
|
1569
|
+
lastSentEstimate = tokens;
|
|
1570
|
+
},
|
|
1571
|
+
});
|
|
1572
|
+
if (reclaim) {
|
|
1573
|
+
// Applied HERE, in the loop's own concrete types: the guard decides,
|
|
1574
|
+
// the caller mutates. Dropping an assistant tool_use message together
|
|
1575
|
+
// with its user tool_result message is what keeps blocks paired.
|
|
1576
|
+
const dropSet = new Set(reclaim.drop);
|
|
1577
|
+
const truncateSet = new Set(reclaim.truncate);
|
|
1578
|
+
const rebuilt = [];
|
|
1579
|
+
for (let i = 0; i < conversation.length; i++) {
|
|
1580
|
+
if (dropSet.has(i)) {
|
|
1581
|
+
continue;
|
|
1582
|
+
}
|
|
1583
|
+
const message = conversation[i];
|
|
1584
|
+
if (truncateSet.has(i) && Array.isArray(message.content)) {
|
|
1585
|
+
rebuilt.push({
|
|
1586
|
+
...message,
|
|
1587
|
+
content: message.content.map((block) => block.type === "tool_result"
|
|
1588
|
+
? {
|
|
1589
|
+
...block,
|
|
1590
|
+
content: previewAnthropicToolResultText(typeof block.content === "string"
|
|
1591
|
+
? block.content
|
|
1592
|
+
: (JSON.stringify(block.content) ?? "")),
|
|
1593
|
+
}
|
|
1594
|
+
: block),
|
|
1595
|
+
});
|
|
1596
|
+
continue;
|
|
1597
|
+
}
|
|
1598
|
+
rebuilt.push(message);
|
|
1599
|
+
}
|
|
1600
|
+
if (dropSet.size > 0) {
|
|
1601
|
+
// Anthropic requires user/assistant alternation around tool blocks;
|
|
1602
|
+
// the note is a user turn placed immediately before the first
|
|
1603
|
+
// surviving assistant tool_use turn, which preserves it.
|
|
1604
|
+
let noteIndex = rebuilt.findIndex((m) => Array.isArray(m.content) &&
|
|
1605
|
+
m.content.some((b) => b.type === "tool_use" || b.type === "tool_result"));
|
|
1606
|
+
if (noteIndex < 0) {
|
|
1607
|
+
noteIndex = Math.min(1, rebuilt.length);
|
|
1608
|
+
}
|
|
1609
|
+
rebuilt.splice(noteIndex, 0, {
|
|
1610
|
+
role: "user",
|
|
1611
|
+
content: [{ type: "text", text: ANTHROPIC_ELISION_NOTE }],
|
|
1612
|
+
});
|
|
1613
|
+
}
|
|
1614
|
+
conversation.length = 0;
|
|
1615
|
+
conversation.push(...rebuilt);
|
|
1616
|
+
}
|
|
1527
1617
|
// Prompt-cache parity with the native Vertex+Claude path — rolling
|
|
1528
1618
|
// history breakpoints, re-applied per step so the stable prefix
|
|
1529
1619
|
// stays byte-identical while the breakpoint follows the growing
|
|
@@ -1587,6 +1677,14 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1587
1677
|
totalCacheRead += event.message.usage.cache_read_input_tokens ?? 0;
|
|
1588
1678
|
totalCacheWrite +=
|
|
1589
1679
|
event.message.usage.cache_creation_input_tokens ?? 0;
|
|
1680
|
+
// Calibration signal for the in-turn guard: the FULL prompt size,
|
|
1681
|
+
// which on this path means uncached input plus both cache tiers.
|
|
1682
|
+
// Using input_tokens alone would read a cache-hit step as tiny and
|
|
1683
|
+
// let the guard drift far under the real cost.
|
|
1684
|
+
lastObservedPromptTokens =
|
|
1685
|
+
(event.message.usage.input_tokens ?? 0) +
|
|
1686
|
+
(event.message.usage.cache_read_input_tokens ?? 0) +
|
|
1687
|
+
(event.message.usage.cache_creation_input_tokens ?? 0);
|
|
1590
1688
|
}
|
|
1591
1689
|
else if (event.type === "content_block_start") {
|
|
1592
1690
|
blockTypes.set(event.index, event.content_block.type);
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
* Nothing here imports from "ai" or "@ai-sdk/*". The base class is a
|
|
18
18
|
* direct HTTP client + multi-step tool-execution loop driven by SSE.
|
|
19
19
|
*/
|
|
20
|
-
import { getRuntimeContextWindow, getRuntimeOutputCeiling, registerRuntimeContextWindow, } from "../constants/contextWindows.js";
|
|
20
|
+
import { getAvailableInputTokens, getRuntimeContextWindow, getRuntimeOutputCeiling, registerRuntimeContextWindow, } from "../constants/contextWindows.js";
|
|
21
|
+
import { guardOpenAICompatConversation } from "../context/openaiCompatLoopGuard.js";
|
|
21
22
|
import { isContextOverflowError, parseProviderOverflowDetails, } from "../context/errorDetection.js";
|
|
22
23
|
import { ContextBudgetExceededError } from "../context/errors.js";
|
|
23
24
|
import { BaseProvider } from "../core/baseProvider.js";
|
|
@@ -770,6 +771,12 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
770
771
|
// May grow mid-turn: hydrated tools with wire-unsafe names need
|
|
771
772
|
// reverse-mapping even when the initial name set required none.
|
|
772
773
|
let effectiveToolNameFromWire = toolNameFromWire;
|
|
774
|
+
// The provider's REAL prompt-token count for the previous step, used to
|
|
775
|
+
// calibrate the guard's char-based estimate for free, paired with the
|
|
776
|
+
// guard's own estimate for that same request — a ratio between counts of
|
|
777
|
+
// two different payloads would be meaningless.
|
|
778
|
+
let lastObservedPromptTokens;
|
|
779
|
+
let lastSentEstimate;
|
|
773
780
|
for (let step = 0; step < maxSteps; step++) {
|
|
774
781
|
// Mid-turn discovery sync: search_tools (tools.discovery) hydrates
|
|
775
782
|
// new tools into toolsRecord between steps. Dispatch already re-reads
|
|
@@ -795,6 +802,35 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
795
802
|
logger.info(`${this.providerName}: ${Object.keys(hydrated).length} tool(s) hydrated mid-turn via discovery: ${Object.keys(hydrated).join(", ")}`);
|
|
796
803
|
}
|
|
797
804
|
}
|
|
805
|
+
// In-turn context guard. This loop appends an assistant tool-call
|
|
806
|
+
// message plus one tool message per result on every step — growth the
|
|
807
|
+
// pre-dispatch budget check never sees. Without this, a long agentic
|
|
808
|
+
// run walks into a provider "context length exceeded" and loses every
|
|
809
|
+
// completed step. Shares its reclaim policy with the other provider
|
|
810
|
+
// loops via loopGuardCore; returns undefined (leaving `conversation`
|
|
811
|
+
// byte-identical) whenever the request still fits, so a loop that fits
|
|
812
|
+
// never pays a prompt-cache invalidation.
|
|
813
|
+
const guarded = guardOpenAICompatConversation({
|
|
814
|
+
conversation,
|
|
815
|
+
availableInputTokens: getAvailableInputTokens(this.providerName, modelId, options.maxTokens ?? undefined),
|
|
816
|
+
// Tool definitions ride outside the message array. Passing an empty
|
|
817
|
+
// message list yields the tools-only overhead, and reuses the same
|
|
818
|
+
// estimator the wire path already trusts.
|
|
819
|
+
fixedOverheadTokens: estimateWireTokens([], openAITools, this.providerName),
|
|
820
|
+
provider: this.providerName,
|
|
821
|
+
observedPromptTokens: lastObservedPromptTokens,
|
|
822
|
+
// Both halves of the calibration ratio must describe the same
|
|
823
|
+
// request: the tokens the provider reported, and this guard's own
|
|
824
|
+
// estimate for what was sent to earn them.
|
|
825
|
+
previousSentEstimate: lastSentEstimate,
|
|
826
|
+
onSentEstimate: (tokens) => {
|
|
827
|
+
lastSentEstimate = tokens;
|
|
828
|
+
},
|
|
829
|
+
});
|
|
830
|
+
if (guarded) {
|
|
831
|
+
conversation.length = 0;
|
|
832
|
+
conversation.push(...guarded);
|
|
833
|
+
}
|
|
798
834
|
const stepResult = await this.streamOneStep({
|
|
799
835
|
modelId,
|
|
800
836
|
url,
|
|
@@ -806,6 +842,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
806
842
|
openAIToolChoice,
|
|
807
843
|
pushChunk,
|
|
808
844
|
});
|
|
845
|
+
lastObservedPromptTokens = stepResult.usage?.prompt_tokens;
|
|
809
846
|
stepFinish = stepResult.finishReason;
|
|
810
847
|
if (stepResult.usage) {
|
|
811
848
|
stepUsage = mergeUsage(stepUsage, stepResult.usage);
|
package/dist/types/context.d.ts
CHANGED
|
@@ -412,6 +412,79 @@ export type RepairResult = {
|
|
|
412
412
|
orphanedCallsFixed: number;
|
|
413
413
|
orphanedResultsFixed: number;
|
|
414
414
|
};
|
|
415
|
+
/**
|
|
416
|
+
* Provider-neutral view of ONE message in an agent loop's history.
|
|
417
|
+
*
|
|
418
|
+
* Every native provider loop keeps its history in a different shape (AI-SDK
|
|
419
|
+
* `ModelMessage`, OpenAI-compatible `{role,tool_calls}`, Gemini `contents`
|
|
420
|
+
* parts, Anthropic content blocks). The reclaim POLICY is identical across all
|
|
421
|
+
* of them, so adapters map their own shape onto this view, ask the core what to
|
|
422
|
+
* do, and apply the answer themselves.
|
|
423
|
+
*/
|
|
424
|
+
export type LoopGuardEntry = {
|
|
425
|
+
/** `toolCall` and `toolResult` form the batches the policy keeps intact. */
|
|
426
|
+
kind: "other" | "toolCall" | "toolResult";
|
|
427
|
+
/** Estimated tokens this entry currently costs. */
|
|
428
|
+
tokens: number;
|
|
429
|
+
/**
|
|
430
|
+
* Tokens this entry would cost with its payload replaced by a head/tail
|
|
431
|
+
* preview. Omitted when the entry cannot usefully shrink — which is exactly
|
|
432
|
+
* the case that forces the policy to drop batches instead.
|
|
433
|
+
*/
|
|
434
|
+
previewTokens?: number;
|
|
435
|
+
};
|
|
436
|
+
/** What the caller should do to reclaim budget. Indices refer to the input array. */
|
|
437
|
+
export type LoopGuardPlan = {
|
|
438
|
+
/** False when the loop is under threshold and nothing should change. */
|
|
439
|
+
fire: boolean;
|
|
440
|
+
/** Entries whose payload should be replaced by a preview. */
|
|
441
|
+
truncate: number[];
|
|
442
|
+
/** Entries to remove entirely — always whole batches, never a partial pair. */
|
|
443
|
+
drop: number[];
|
|
444
|
+
/** Estimated total after applying the plan, including fixed overhead. */
|
|
445
|
+
projectedTokens: number;
|
|
446
|
+
};
|
|
447
|
+
/**
|
|
448
|
+
* Structural view of one Anthropic content block, loose enough to accept the
|
|
449
|
+
* official SDK's `ContentBlockParam` union and NeuroLink's own
|
|
450
|
+
* `VertexAnthropicMessage` blocks without a cast at either call site.
|
|
451
|
+
*/
|
|
452
|
+
export type AnthropicGuardBlock = {
|
|
453
|
+
type: string;
|
|
454
|
+
/** Payload of a `tool_result` block. Other block kinds carry other fields. */
|
|
455
|
+
content?: unknown;
|
|
456
|
+
/** Text of a `text` block. */
|
|
457
|
+
text?: string;
|
|
458
|
+
};
|
|
459
|
+
/**
|
|
460
|
+
* Structural view of one Anthropic-shaped message, as used by both the direct
|
|
461
|
+
* Anthropic loop and the native Vertex+Claude path. Tool calls ride as
|
|
462
|
+
* `tool_use` blocks on an assistant message; their answers ride as
|
|
463
|
+
* `tool_result` blocks on the following user message.
|
|
464
|
+
*/
|
|
465
|
+
export type AnthropicGuardMessage = {
|
|
466
|
+
/**
|
|
467
|
+
* `system` is included because the installed `@anthropic-ai/sdk` widens
|
|
468
|
+
* `MessageParam["role"]` to accept it; narrowing here would make the SDK's
|
|
469
|
+
* own array unassignable at the call site.
|
|
470
|
+
*/
|
|
471
|
+
role: "user" | "assistant" | "system";
|
|
472
|
+
content: string | AnthropicGuardBlock[];
|
|
473
|
+
};
|
|
474
|
+
/** Tuning for {@link planLoopGuardReclaim}. */
|
|
475
|
+
export type LoopGuardPolicy = {
|
|
476
|
+
availableInputTokens: number;
|
|
477
|
+
/** System prompt + tool definitions — rides outside the message array. */
|
|
478
|
+
fixedOverheadTokens: number;
|
|
479
|
+
/** Fraction of the window at which the guard fires. */
|
|
480
|
+
thresholdRatio?: number;
|
|
481
|
+
/** Fraction of the window the guard reclaims down to once it fires. */
|
|
482
|
+
lowWaterRatio?: number;
|
|
483
|
+
/** Newest entries the guard must never modify. */
|
|
484
|
+
protectedTailCount?: number;
|
|
485
|
+
/** Observed/estimated token ratio, used to tighten both marks. */
|
|
486
|
+
calibration?: number;
|
|
487
|
+
};
|
|
415
488
|
/**
|
|
416
489
|
* One contiguous tool batch: the run of `tool_call` messages emitted by a
|
|
417
490
|
* single agent step, plus the run of `tool_result` messages that follows it.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "10.10.
|
|
3
|
+
"version": "10.10.5",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
|
|
6
6
|
"author": {
|
|
@@ -85,6 +85,9 @@
|
|
|
85
85
|
"test:tool-pairing": "npx tsx test/continuous-test-suite-tool-pairing.ts",
|
|
86
86
|
"test:token-accounting": "npx tsx test/continuous-test-suite-token-accounting.ts",
|
|
87
87
|
"test:step-guard": "npx tsx test/continuous-test-suite-step-guard.ts",
|
|
88
|
+
"test:loop-guard-core": "npx tsx test/continuous-test-suite-loop-guard-core.ts",
|
|
89
|
+
"test:openai-compat-guard": "npx tsx test/continuous-test-suite-openai-compat-guard.ts",
|
|
90
|
+
"test:anthropic-guard": "npx tsx test/continuous-test-suite-anthropic-guard.ts",
|
|
88
91
|
"test:middleware": "npx tsx test/continuous-test-suite-middleware.ts",
|
|
89
92
|
"test:observability": "npx tsx test/continuous-test-suite-observability.ts",
|
|
90
93
|
"test:ppt": "npx tsx test/continuous-test-suite-ppt.ts",
|