@juspay/neurolink 9.78.0 → 9.79.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +314 -314
- package/dist/lib/neurolink.js +14 -5
- package/dist/lib/providers/googleVertex.js +81 -3
- package/dist/lib/tasks/autoresearchTaskExecutor.js +1 -1
- package/dist/lib/types/providers.d.ts +46 -0
- package/dist/lib/utils/anthropicCacheBreakpoints.d.ts +15 -0
- package/dist/lib/utils/anthropicCacheBreakpoints.js +98 -0
- package/dist/lib/workflow/core/ensembleExecutor.js +0 -2
- package/dist/neurolink.js +14 -5
- package/dist/providers/googleVertex.js +81 -3
- package/dist/tasks/autoresearchTaskExecutor.js +1 -1
- package/dist/types/providers.d.ts +46 -0
- package/dist/utils/anthropicCacheBreakpoints.d.ts +15 -0
- package/dist/utils/anthropicCacheBreakpoints.js +97 -0
- package/dist/workflow/core/ensembleExecutor.js +0 -2
- package/package.json +2 -1
package/dist/lib/neurolink.js
CHANGED
|
@@ -340,7 +340,7 @@ export class NeuroLink {
|
|
|
340
340
|
* @param result - The result of the tool execution (optional)
|
|
341
341
|
* @param error - The error if execution failed (optional)
|
|
342
342
|
*/
|
|
343
|
-
emitToolEndEvent(toolName, startTime, success, result, error) {
|
|
343
|
+
emitToolEndEvent(toolName, startTime, success, result, error, executionId) {
|
|
344
344
|
// Emit tool end event (NeuroLink format - enhanced with result/error)
|
|
345
345
|
// Serialize error to string for consumer compatibility (event listeners
|
|
346
346
|
// commonly check `typeof event.error === "string"`).
|
|
@@ -350,6 +350,7 @@ export class NeuroLink {
|
|
|
350
350
|
timestamp: Date.now(),
|
|
351
351
|
result,
|
|
352
352
|
error: error ? error.message : undefined,
|
|
353
|
+
executionId,
|
|
353
354
|
}));
|
|
354
355
|
}
|
|
355
356
|
// Conversation memory support
|
|
@@ -8576,9 +8577,16 @@ Current user's request: ${currentInput}`;
|
|
|
8576
8577
|
: params
|
|
8577
8578
|
? JSON.stringify(params)
|
|
8578
8579
|
: "";
|
|
8580
|
+
const executionStartTime = Date.now();
|
|
8581
|
+
// Per-invocation id so consumers can correlate a tool:start with its matching
|
|
8582
|
+
// tool:end even when the same tool runs multiple times concurrently.
|
|
8583
|
+
const executionId = `${toolName}-${executionStartTime}-${Math.random()
|
|
8584
|
+
.toString(36)
|
|
8585
|
+
.slice(2, 11)}`;
|
|
8579
8586
|
return {
|
|
8580
8587
|
functionTag: "NeuroLink.executeTool",
|
|
8581
|
-
executionStartTime
|
|
8588
|
+
executionStartTime,
|
|
8589
|
+
executionId,
|
|
8582
8590
|
externalTool,
|
|
8583
8591
|
toolType,
|
|
8584
8592
|
inputSize: inputStr.length,
|
|
@@ -8636,6 +8644,7 @@ Current user's request: ${currentInput}`;
|
|
|
8636
8644
|
this.emitter.emit("tool:start", createToolEventPayload(toolName, {
|
|
8637
8645
|
timestamp: executionContext.executionStartTime,
|
|
8638
8646
|
input: params,
|
|
8647
|
+
executionId: executionContext.executionId,
|
|
8639
8648
|
}));
|
|
8640
8649
|
const toolInfo = this.toolRegistry.getToolInfo(toolName);
|
|
8641
8650
|
const finalOptions = {
|
|
@@ -8801,7 +8810,7 @@ Current user's request: ${currentInput}`;
|
|
|
8801
8810
|
prepared.metrics.errorCategories[mappedCategory] =
|
|
8802
8811
|
(prepared.metrics.errorCategories[mappedCategory] || 0) + 1;
|
|
8803
8812
|
}
|
|
8804
|
-
this.emitToolEndEvent(toolName, executionContext.executionStartTime, !isToolError, result, isToolError && errorText ? new Error(errorText) : undefined);
|
|
8813
|
+
this.emitToolEndEvent(toolName, executionContext.executionStartTime, !isToolError, result, isToolError && errorText ? new Error(errorText) : undefined, executionContext.executionId);
|
|
8805
8814
|
toolSpan.setAttribute("tool.result.status", isToolError ? "error" : "success");
|
|
8806
8815
|
toolSpan.setAttribute("tool.duration_ms", executionTime);
|
|
8807
8816
|
return result;
|
|
@@ -8820,7 +8829,7 @@ Current user's request: ${currentInput}`;
|
|
|
8820
8829
|
});
|
|
8821
8830
|
prepared.metrics.errorCategories[ErrorCategory.EXECUTION] =
|
|
8822
8831
|
(prepared.metrics.errorCategories[ErrorCategory.EXECUTION] || 0) + 1;
|
|
8823
|
-
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, new Error(`Circuit breaker open for ${toolName} (state=${error.breakerState}, failures=${error.failureCount})`));
|
|
8832
|
+
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, new Error(`Circuit breaker open for ${toolName} (state=${error.breakerState}, failures=${error.failureCount})`), executionContext.executionId);
|
|
8824
8833
|
toolSpan.setAttribute("tool.result.status", "circuit_breaker_open");
|
|
8825
8834
|
toolSpan.setAttribute("tool.duration_ms", executionTime);
|
|
8826
8835
|
toolSpan.setAttribute("tool.circuit_breaker.state", error.breakerState);
|
|
@@ -8875,7 +8884,7 @@ Current user's request: ${currentInput}`;
|
|
|
8875
8884
|
const category = structuredError.category || ErrorCategory.EXECUTION;
|
|
8876
8885
|
prepared.metrics.errorCategories[category] =
|
|
8877
8886
|
(prepared.metrics.errorCategories[category] || 0) + 1;
|
|
8878
|
-
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, structuredError);
|
|
8887
|
+
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, structuredError, executionContext.executionId);
|
|
8879
8888
|
// Gate on listenerCount: Node EventEmitter rethrows the original error
|
|
8880
8889
|
// from emit("error", e) when no listener is registered, which would
|
|
8881
8890
|
// short-circuit the surrounding flow and surface as an unhandled
|
|
@@ -10,6 +10,7 @@ import { ModelConfigurationManager } from "../core/modelConfiguration.js";
|
|
|
10
10
|
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
11
11
|
import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
|
|
12
12
|
import { ERROR_CODES, NeuroLinkError } from "../utils/errorHandling.js";
|
|
13
|
+
import { applyVertexAnthropicCacheBreakpoints } from "../utils/anthropicCacheBreakpoints.js";
|
|
13
14
|
import { FileDetector } from "../utils/fileDetector.js";
|
|
14
15
|
import { processUnifiedFilesArray } from "../utils/messageBuilder.js";
|
|
15
16
|
import { logger } from "../utils/logger.js";
|
|
@@ -2491,9 +2492,27 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
2491
2492
|
}, turnContext);
|
|
2492
2493
|
let response;
|
|
2493
2494
|
try {
|
|
2495
|
+
// Vertex has no automatic prompt caching — place explicit
|
|
2496
|
+
// cache_control breakpoints (system, tools, rolling history) so the
|
|
2497
|
+
// conversation prefix is cached across turns instead of re-billed as
|
|
2498
|
+
// fresh input every call. Re-applied per step: the stable prefix
|
|
2499
|
+
// stays byte-identical (consistent cache key) while the rolling
|
|
2500
|
+
// breakpoint follows the growing tail.
|
|
2501
|
+
const cachedStream = applyVertexAnthropicCacheBreakpoints({
|
|
2502
|
+
system: systemPromptWithSchema,
|
|
2503
|
+
tools,
|
|
2504
|
+
messages: currentMessages,
|
|
2505
|
+
});
|
|
2494
2506
|
const stream = await client.messages.stream({
|
|
2495
2507
|
...requestParams,
|
|
2496
|
-
|
|
2508
|
+
...(cachedStream.system !== undefined && {
|
|
2509
|
+
system: cachedStream.system,
|
|
2510
|
+
}),
|
|
2511
|
+
...(cachedStream.tools &&
|
|
2512
|
+
cachedStream.tools.length > 0 && {
|
|
2513
|
+
tools: cachedStream.tools,
|
|
2514
|
+
}),
|
|
2515
|
+
messages: cachedStream.messages,
|
|
2497
2516
|
});
|
|
2498
2517
|
activeStream = stream;
|
|
2499
2518
|
// Forward each text delta as it arrives — the Anthropic SDK fires
|
|
@@ -2750,6 +2769,17 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
2750
2769
|
}
|
|
2751
2770
|
metadata.responseTime = Date.now() - startTime;
|
|
2752
2771
|
metadata.totalToolExecutions = allToolCalls.filter((tc) => tc.toolName !== "final_result").length;
|
|
2772
|
+
// Surface cache metrics once, after the agentic loop, from the
|
|
2773
|
+
// cumulative turnCacheUsage running totals (accumulated via += on every
|
|
2774
|
+
// step above). Assigned here — not per-iteration — so the final values
|
|
2775
|
+
// are unambiguous: analytics sees caching is working and calculateCost
|
|
2776
|
+
// can price the cache-read/write tiers instead of full input rate.
|
|
2777
|
+
if (turnCacheUsage.read > 0) {
|
|
2778
|
+
usage.cacheReadTokens = turnCacheUsage.read;
|
|
2779
|
+
}
|
|
2780
|
+
if (turnCacheUsage.creation > 0) {
|
|
2781
|
+
usage.cacheCreationTokens = turnCacheUsage.creation;
|
|
2782
|
+
}
|
|
2753
2783
|
const turnOutputAttribute = spanJsonAttribute({
|
|
2754
2784
|
text: aggregatedTurnText,
|
|
2755
2785
|
...(structuredOutputRef.value
|
|
@@ -3110,6 +3140,12 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
3110
3140
|
const allToolCalls = [];
|
|
3111
3141
|
let totalInputTokens = 0;
|
|
3112
3142
|
let totalOutputTokens = 0;
|
|
3143
|
+
// Cache metrics (Anthropic reports these separately from input_tokens, which
|
|
3144
|
+
// is the uncached remainder). Surfaced on the result so analytics can see
|
|
3145
|
+
// caching is working and calculateCost can price the ~0.1x cache-read /
|
|
3146
|
+
// ~1.25x cache-write tiers instead of billing everything at full input rate.
|
|
3147
|
+
let totalCacheReadTokens = 0;
|
|
3148
|
+
let totalCacheCreationTokens = 0;
|
|
3113
3149
|
// Track the final Anthropic stop_reason so we can surface finishReason
|
|
3114
3150
|
// (notably "length" on token truncation) — the legacy native path always
|
|
3115
3151
|
// reported "stop", hiding truncation from callers.
|
|
@@ -3121,13 +3157,36 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
3121
3157
|
// Bound the SDK wait so a stalled Vertex/Anthropic call can't hang
|
|
3122
3158
|
// generate forever. options.timeout wins if set, otherwise default
|
|
3123
3159
|
// to 5 min — generous for tool-heavy turns.
|
|
3160
|
+
// Vertex has no automatic prompt caching — place explicit cache_control
|
|
3161
|
+
// breakpoints (system, tools, rolling history) so the conversation
|
|
3162
|
+
// prefix is cached across turns instead of re-billed as fresh input
|
|
3163
|
+
// every call. Re-applied per step: the stable prefix stays
|
|
3164
|
+
// byte-identical (consistent cache key) while the rolling breakpoint
|
|
3165
|
+
// follows the growing tail.
|
|
3166
|
+
const cachedGenerate = applyVertexAnthropicCacheBreakpoints({
|
|
3167
|
+
system: systemPromptWithSchema,
|
|
3168
|
+
tools,
|
|
3169
|
+
messages: currentMessages,
|
|
3170
|
+
});
|
|
3124
3171
|
const response = await withTimeout(client.messages.create({
|
|
3125
3172
|
...requestParams,
|
|
3126
|
-
|
|
3173
|
+
...(cachedGenerate.system !== undefined && {
|
|
3174
|
+
system: cachedGenerate.system,
|
|
3175
|
+
}),
|
|
3176
|
+
...(cachedGenerate.tools &&
|
|
3177
|
+
cachedGenerate.tools.length > 0 && {
|
|
3178
|
+
tools: cachedGenerate.tools,
|
|
3179
|
+
}),
|
|
3180
|
+
messages: cachedGenerate.messages,
|
|
3127
3181
|
}), generateTimeoutMs, "Anthropic generate timed out");
|
|
3128
|
-
// Update token counts
|
|
3182
|
+
// Update token counts. input_tokens is the uncached remainder; cache
|
|
3183
|
+
// reads/writes are reported separately and accumulated here so the
|
|
3184
|
+
// result reflects the full picture.
|
|
3129
3185
|
totalInputTokens += response.usage?.input_tokens || 0;
|
|
3130
3186
|
totalOutputTokens += response.usage?.output_tokens || 0;
|
|
3187
|
+
totalCacheReadTokens += response.usage?.cache_read_input_tokens || 0;
|
|
3188
|
+
totalCacheCreationTokens +=
|
|
3189
|
+
response.usage?.cache_creation_input_tokens || 0;
|
|
3131
3190
|
lastStopReason = response.stop_reason;
|
|
3132
3191
|
// Check if we need to handle tool use
|
|
3133
3192
|
const toolUseBlocks = response.content.filter((block) => block.type === "tool_use");
|
|
@@ -3287,6 +3346,12 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
3287
3346
|
input: totalInputTokens,
|
|
3288
3347
|
output: totalOutputTokens,
|
|
3289
3348
|
total: totalInputTokens + totalOutputTokens,
|
|
3349
|
+
...(totalCacheReadTokens > 0 && {
|
|
3350
|
+
cacheReadTokens: totalCacheReadTokens,
|
|
3351
|
+
}),
|
|
3352
|
+
...(totalCacheCreationTokens > 0 && {
|
|
3353
|
+
cacheCreationTokens: totalCacheCreationTokens,
|
|
3354
|
+
}),
|
|
3290
3355
|
},
|
|
3291
3356
|
responseTime,
|
|
3292
3357
|
toolsUsed: externalToolCalls.map((tc) => tc.toolName),
|
|
@@ -3761,6 +3826,8 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
3761
3826
|
const inputTokens = usage.input ?? usage.inputTokens ?? 0;
|
|
3762
3827
|
const outputTokens = usage.output ?? usage.outputTokens ?? 0;
|
|
3763
3828
|
const totalTokens = usage.total ?? usage.totalTokens ?? inputTokens + outputTokens;
|
|
3829
|
+
const cacheReadTokens = usage.cacheReadTokens ?? 0;
|
|
3830
|
+
const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
|
|
3764
3831
|
if (inputTokens > 0) {
|
|
3765
3832
|
span.setAttribute("gen_ai.usage.input_tokens", inputTokens);
|
|
3766
3833
|
}
|
|
@@ -3770,11 +3837,22 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
3770
3837
|
if (totalTokens > 0) {
|
|
3771
3838
|
span.setAttribute("gen_ai.usage.total_tokens", totalTokens);
|
|
3772
3839
|
}
|
|
3840
|
+
if (cacheReadTokens > 0) {
|
|
3841
|
+
span.setAttribute("gen_ai.usage.cache_read_input_tokens", cacheReadTokens);
|
|
3842
|
+
}
|
|
3843
|
+
if (cacheCreationTokens > 0) {
|
|
3844
|
+
span.setAttribute("gen_ai.usage.cache_creation_input_tokens", cacheCreationTokens);
|
|
3845
|
+
}
|
|
3773
3846
|
try {
|
|
3847
|
+
// Pass cache tokens through so calculateCost prices the ~0.1x cache-read
|
|
3848
|
+
// and ~1.25x cache-write tiers instead of billing cached content at the
|
|
3849
|
+
// full input rate.
|
|
3774
3850
|
const cost = calculateCost(this.providerName, modelName, {
|
|
3775
3851
|
input: inputTokens,
|
|
3776
3852
|
output: outputTokens,
|
|
3777
3853
|
total: totalTokens,
|
|
3854
|
+
cacheReadTokens,
|
|
3855
|
+
cacheCreationTokens,
|
|
3778
3856
|
});
|
|
3779
3857
|
if (typeof cost === "number" && cost > 0) {
|
|
3780
3858
|
span.setAttribute("neurolink.cost", cost);
|
|
@@ -220,7 +220,7 @@ export async function executeAutoresearchTick(task, neurolink, emitter) {
|
|
|
220
220
|
...(phasePolicy.forcedTool
|
|
221
221
|
? {
|
|
222
222
|
prepareStep: ({ stepNumber }) => {
|
|
223
|
-
if (stepNumber === 0) {
|
|
223
|
+
if (stepNumber === 0 && phasePolicy.forcedTool) {
|
|
224
224
|
return {
|
|
225
225
|
toolChoice: {
|
|
226
226
|
type: "tool",
|
|
@@ -1797,11 +1797,21 @@ export type VertexGenaiFunctionDeclaration = {
|
|
|
1797
1797
|
* Message payload passed to the Anthropic Vertex SDK — mirrors the Anthropic
|
|
1798
1798
|
* Messages API shape (role + structured content blocks).
|
|
1799
1799
|
*/
|
|
1800
|
+
/**
|
|
1801
|
+
* Anthropic ephemeral prompt-cache breakpoint marker. Placed on a content
|
|
1802
|
+
* block / tool / system block to make the rendered prefix up to that point a
|
|
1803
|
+
* cache breakpoint. Vertex has NO automatic caching, so these explicit markers
|
|
1804
|
+
* are the only way the conversation prefix is cached across turns.
|
|
1805
|
+
*/
|
|
1806
|
+
export type VertexAnthropicCacheControl = {
|
|
1807
|
+
type: "ephemeral";
|
|
1808
|
+
};
|
|
1800
1809
|
export type VertexAnthropicMessage = {
|
|
1801
1810
|
role: "user" | "assistant";
|
|
1802
1811
|
content: string | Array<{
|
|
1803
1812
|
type: "text";
|
|
1804
1813
|
text: string;
|
|
1814
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1805
1815
|
} | {
|
|
1806
1816
|
type: "image";
|
|
1807
1817
|
source: {
|
|
@@ -1809,6 +1819,7 @@ export type VertexAnthropicMessage = {
|
|
|
1809
1819
|
media_type: string;
|
|
1810
1820
|
data: string;
|
|
1811
1821
|
};
|
|
1822
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1812
1823
|
} | {
|
|
1813
1824
|
type: "document";
|
|
1814
1825
|
source: {
|
|
@@ -1816,23 +1827,38 @@ export type VertexAnthropicMessage = {
|
|
|
1816
1827
|
media_type: string;
|
|
1817
1828
|
data: string;
|
|
1818
1829
|
};
|
|
1830
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1819
1831
|
} | {
|
|
1820
1832
|
type: "tool_use";
|
|
1821
1833
|
id: string;
|
|
1822
1834
|
name: string;
|
|
1823
1835
|
input: unknown;
|
|
1836
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1824
1837
|
} | {
|
|
1825
1838
|
type: "tool_result";
|
|
1826
1839
|
tool_use_id: string;
|
|
1827
1840
|
content: string;
|
|
1841
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1828
1842
|
} | {
|
|
1829
1843
|
type: "thinking";
|
|
1830
1844
|
thinking: string;
|
|
1845
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1831
1846
|
} | {
|
|
1832
1847
|
type: "redacted_thinking";
|
|
1833
1848
|
data: string;
|
|
1849
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1834
1850
|
}>;
|
|
1835
1851
|
};
|
|
1852
|
+
/**
|
|
1853
|
+
* System prompt block form accepted by the Anthropic Vertex SDK. Used instead
|
|
1854
|
+
* of a bare string when a `cache_control` breakpoint must ride on the system
|
|
1855
|
+
* prompt (a string `system` cannot carry one).
|
|
1856
|
+
*/
|
|
1857
|
+
export type VertexAnthropicSystemBlock = {
|
|
1858
|
+
type: "text";
|
|
1859
|
+
text: string;
|
|
1860
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1861
|
+
};
|
|
1836
1862
|
/** Tool definition accepted by the Anthropic Vertex SDK. */
|
|
1837
1863
|
export type VertexAnthropicTool = {
|
|
1838
1864
|
name: string;
|
|
@@ -1842,6 +1868,26 @@ export type VertexAnthropicTool = {
|
|
|
1842
1868
|
properties?: Record<string, unknown>;
|
|
1843
1869
|
required?: string[];
|
|
1844
1870
|
};
|
|
1871
|
+
cache_control?: VertexAnthropicCacheControl;
|
|
1872
|
+
};
|
|
1873
|
+
/** Input to `applyVertexAnthropicCacheBreakpoints`. */
|
|
1874
|
+
export type VertexAnthropicCacheInput = {
|
|
1875
|
+
system?: string;
|
|
1876
|
+
tools?: VertexAnthropicTool[];
|
|
1877
|
+
messages: VertexAnthropicMessage[];
|
|
1878
|
+
/**
|
|
1879
|
+
* Cap on how many of the most-recent messages receive a rolling history
|
|
1880
|
+
* breakpoint. Defaults to "use the remaining budget". Two or more gives
|
|
1881
|
+
* cross-turn continuity and resilience against Anthropic's 20-block cache
|
|
1882
|
+
* lookback window on tool-heavy turns.
|
|
1883
|
+
*/
|
|
1884
|
+
maxHistoryBreakpoints?: number;
|
|
1885
|
+
};
|
|
1886
|
+
/** Output of `applyVertexAnthropicCacheBreakpoints` — a cache-annotated request. */
|
|
1887
|
+
export type VertexAnthropicCacheOutput = {
|
|
1888
|
+
system?: string | VertexAnthropicSystemBlock[];
|
|
1889
|
+
tools?: VertexAnthropicTool[];
|
|
1890
|
+
messages: VertexAnthropicMessage[];
|
|
1845
1891
|
};
|
|
1846
1892
|
/**
|
|
1847
1893
|
* Content block variants returned by the Anthropic Vertex SDK during streaming
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { VertexAnthropicCacheInput, VertexAnthropicCacheOutput } from "../types/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Annotate a native Vertex+Claude request with prompt-cache breakpoints.
|
|
4
|
+
*
|
|
5
|
+
* Budget allocation (max 4 markers):
|
|
6
|
+
* 1. The stable prefix — the last system block when a system prompt is
|
|
7
|
+
* present (this single marker caches `tools + system`, since system
|
|
8
|
+
* renders after tools); otherwise the last tool definition.
|
|
9
|
+
* 2-4. A rolling breakpoint on the last few messages, so the
|
|
10
|
+
* growing-but-now-stable conversation history is cached and only the
|
|
11
|
+
* newest turn is billed as fresh input.
|
|
12
|
+
*
|
|
13
|
+
* Pure: the inputs are cloned, never mutated.
|
|
14
|
+
*/
|
|
15
|
+
export declare function applyVertexAnthropicCacheBreakpoints(input: VertexAnthropicCacheInput): VertexAnthropicCacheOutput;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic prompt-cache breakpoint placement for the native Vertex+Claude
|
|
3
|
+
* request path.
|
|
4
|
+
*
|
|
5
|
+
* Vertex does NOT support automatic prompt caching — the only way the
|
|
6
|
+
* conversation prefix gets cached across turns is explicit `cache_control`
|
|
7
|
+
* markers in the request. Anthropic renders a request as `tools → system →
|
|
8
|
+
* messages` and caches the prefix up to each marker, with a hard ceiling of
|
|
9
|
+
* four markers.
|
|
10
|
+
*
|
|
11
|
+
* Without a marker on the message history, the entire (growing, often
|
|
12
|
+
* tool-result-heavy) conversation falls *after* the last breakpoint and is
|
|
13
|
+
* re-billed at full input price on every turn. That is the regression this
|
|
14
|
+
* fixes: it gives the history a rolling breakpoint so the stable prefix is
|
|
15
|
+
* cached at ~0.1x and only the newest turn is fresh.
|
|
16
|
+
*/
|
|
17
|
+
const EPHEMERAL = { type: "ephemeral" };
|
|
18
|
+
/** Anthropic allows at most four `cache_control` breakpoints per request. */
|
|
19
|
+
const MAX_BREAKPOINTS = 4;
|
|
20
|
+
/**
|
|
21
|
+
* Annotate a native Vertex+Claude request with prompt-cache breakpoints.
|
|
22
|
+
*
|
|
23
|
+
* Budget allocation (max 4 markers):
|
|
24
|
+
* 1. The stable prefix — the last system block when a system prompt is
|
|
25
|
+
* present (this single marker caches `tools + system`, since system
|
|
26
|
+
* renders after tools); otherwise the last tool definition.
|
|
27
|
+
* 2-4. A rolling breakpoint on the last few messages, so the
|
|
28
|
+
* growing-but-now-stable conversation history is cached and only the
|
|
29
|
+
* newest turn is billed as fresh input.
|
|
30
|
+
*
|
|
31
|
+
* Pure: the inputs are cloned, never mutated.
|
|
32
|
+
*/
|
|
33
|
+
export function applyVertexAnthropicCacheBreakpoints(input) {
|
|
34
|
+
let budget = MAX_BREAKPOINTS;
|
|
35
|
+
// 1. Stable prefix. A bare string `system` cannot carry cache_control, so
|
|
36
|
+
// convert it to block form. Marking the last system block caches the whole
|
|
37
|
+
// tools + system prefix; only fall back to marking the last tool when there
|
|
38
|
+
// is no system prompt to mark.
|
|
39
|
+
let system = input.system;
|
|
40
|
+
let tools = input.tools;
|
|
41
|
+
const hasSystem = !!input.system && input.system.trim().length > 0;
|
|
42
|
+
if (hasSystem) {
|
|
43
|
+
system = [
|
|
44
|
+
{ type: "text", text: input.system, cache_control: EPHEMERAL },
|
|
45
|
+
];
|
|
46
|
+
budget--;
|
|
47
|
+
}
|
|
48
|
+
else if (input.tools && input.tools.length > 0) {
|
|
49
|
+
const lastIndex = input.tools.length - 1;
|
|
50
|
+
tools = input.tools.map((tool, i) => i === lastIndex ? { ...tool, cache_control: EPHEMERAL } : tool);
|
|
51
|
+
budget--;
|
|
52
|
+
}
|
|
53
|
+
// 2. Rolling history breakpoints over the tail of the conversation.
|
|
54
|
+
const messages = input.messages.map((m) => ({ ...m }));
|
|
55
|
+
let remaining = Math.min(budget, input.maxHistoryBreakpoints ?? MAX_BREAKPOINTS);
|
|
56
|
+
for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
|
|
57
|
+
if (markLastContentBlock(messages, i)) {
|
|
58
|
+
remaining--;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return { system, tools, messages };
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Place a cache breakpoint on the last content block of `messages[i]`.
|
|
65
|
+
* Anthropic attaches `cache_control` to a content block, not the message
|
|
66
|
+
* envelope, so a string content body is first converted to block form.
|
|
67
|
+
* Returns false when the message has no markable block (caller then walks to
|
|
68
|
+
* an earlier message), so an empty turn never silently consumes a breakpoint.
|
|
69
|
+
*/
|
|
70
|
+
function markLastContentBlock(messages, i) {
|
|
71
|
+
const message = messages[i];
|
|
72
|
+
if (typeof message.content === "string") {
|
|
73
|
+
if (message.content.length === 0) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
messages[i] = {
|
|
77
|
+
...message,
|
|
78
|
+
content: [
|
|
79
|
+
{ type: "text", text: message.content, cache_control: EPHEMERAL },
|
|
80
|
+
],
|
|
81
|
+
};
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
if (!Array.isArray(message.content) || message.content.length === 0) {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
// Shallow clone is sufficient: we only ever add a top-level `cache_control`
|
|
88
|
+
// field to the last block and never mutate nested members (e.g. an image's
|
|
89
|
+
// `source`). Those nested objects stay shared with the input by reference,
|
|
90
|
+
// which is safe because they are never written to. If a future block shape
|
|
91
|
+
// requires mutating nested members, deep-clone that block instead.
|
|
92
|
+
const content = message.content.map((block) => ({ ...block }));
|
|
93
|
+
const lastIndex = content.length - 1;
|
|
94
|
+
content[lastIndex] = { ...content[lastIndex], cache_control: EPHEMERAL };
|
|
95
|
+
messages[i] = { ...message, content };
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=anthropicCacheBreakpoints.js.map
|
|
@@ -128,7 +128,6 @@ async function executeModel(options, workflowDefaultSystemPrompt) {
|
|
|
128
128
|
let result;
|
|
129
129
|
let attempts = 0;
|
|
130
130
|
const MAX_ATTEMPTS = 2;
|
|
131
|
-
/* eslint-disable no-constant-condition */
|
|
132
131
|
while (true) {
|
|
133
132
|
attempts++;
|
|
134
133
|
result = await executeWithTimeout(async () => {
|
|
@@ -145,7 +144,6 @@ async function executeModel(options, workflowDefaultSystemPrompt) {
|
|
|
145
144
|
}
|
|
146
145
|
logger.warn(`[${functionTag}] Model returned empty content — retrying once`, { provider: model.provider, model: model.model, attempt: attempts });
|
|
147
146
|
}
|
|
148
|
-
/* eslint-enable no-constant-condition */
|
|
149
147
|
const responseTime = Date.now() - startTime;
|
|
150
148
|
logger.debug(`[${functionTag}] Model execution successful`, {
|
|
151
149
|
provider: model.provider,
|
package/dist/neurolink.js
CHANGED
|
@@ -340,7 +340,7 @@ export class NeuroLink {
|
|
|
340
340
|
* @param result - The result of the tool execution (optional)
|
|
341
341
|
* @param error - The error if execution failed (optional)
|
|
342
342
|
*/
|
|
343
|
-
emitToolEndEvent(toolName, startTime, success, result, error) {
|
|
343
|
+
emitToolEndEvent(toolName, startTime, success, result, error, executionId) {
|
|
344
344
|
// Emit tool end event (NeuroLink format - enhanced with result/error)
|
|
345
345
|
// Serialize error to string for consumer compatibility (event listeners
|
|
346
346
|
// commonly check `typeof event.error === "string"`).
|
|
@@ -350,6 +350,7 @@ export class NeuroLink {
|
|
|
350
350
|
timestamp: Date.now(),
|
|
351
351
|
result,
|
|
352
352
|
error: error ? error.message : undefined,
|
|
353
|
+
executionId,
|
|
353
354
|
}));
|
|
354
355
|
}
|
|
355
356
|
// Conversation memory support
|
|
@@ -8576,9 +8577,16 @@ Current user's request: ${currentInput}`;
|
|
|
8576
8577
|
: params
|
|
8577
8578
|
? JSON.stringify(params)
|
|
8578
8579
|
: "";
|
|
8580
|
+
const executionStartTime = Date.now();
|
|
8581
|
+
// Per-invocation id so consumers can correlate a tool:start with its matching
|
|
8582
|
+
// tool:end even when the same tool runs multiple times concurrently.
|
|
8583
|
+
const executionId = `${toolName}-${executionStartTime}-${Math.random()
|
|
8584
|
+
.toString(36)
|
|
8585
|
+
.slice(2, 11)}`;
|
|
8579
8586
|
return {
|
|
8580
8587
|
functionTag: "NeuroLink.executeTool",
|
|
8581
|
-
executionStartTime
|
|
8588
|
+
executionStartTime,
|
|
8589
|
+
executionId,
|
|
8582
8590
|
externalTool,
|
|
8583
8591
|
toolType,
|
|
8584
8592
|
inputSize: inputStr.length,
|
|
@@ -8636,6 +8644,7 @@ Current user's request: ${currentInput}`;
|
|
|
8636
8644
|
this.emitter.emit("tool:start", createToolEventPayload(toolName, {
|
|
8637
8645
|
timestamp: executionContext.executionStartTime,
|
|
8638
8646
|
input: params,
|
|
8647
|
+
executionId: executionContext.executionId,
|
|
8639
8648
|
}));
|
|
8640
8649
|
const toolInfo = this.toolRegistry.getToolInfo(toolName);
|
|
8641
8650
|
const finalOptions = {
|
|
@@ -8801,7 +8810,7 @@ Current user's request: ${currentInput}`;
|
|
|
8801
8810
|
prepared.metrics.errorCategories[mappedCategory] =
|
|
8802
8811
|
(prepared.metrics.errorCategories[mappedCategory] || 0) + 1;
|
|
8803
8812
|
}
|
|
8804
|
-
this.emitToolEndEvent(toolName, executionContext.executionStartTime, !isToolError, result, isToolError && errorText ? new Error(errorText) : undefined);
|
|
8813
|
+
this.emitToolEndEvent(toolName, executionContext.executionStartTime, !isToolError, result, isToolError && errorText ? new Error(errorText) : undefined, executionContext.executionId);
|
|
8805
8814
|
toolSpan.setAttribute("tool.result.status", isToolError ? "error" : "success");
|
|
8806
8815
|
toolSpan.setAttribute("tool.duration_ms", executionTime);
|
|
8807
8816
|
return result;
|
|
@@ -8820,7 +8829,7 @@ Current user's request: ${currentInput}`;
|
|
|
8820
8829
|
});
|
|
8821
8830
|
prepared.metrics.errorCategories[ErrorCategory.EXECUTION] =
|
|
8822
8831
|
(prepared.metrics.errorCategories[ErrorCategory.EXECUTION] || 0) + 1;
|
|
8823
|
-
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, new Error(`Circuit breaker open for ${toolName} (state=${error.breakerState}, failures=${error.failureCount})`));
|
|
8832
|
+
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, new Error(`Circuit breaker open for ${toolName} (state=${error.breakerState}, failures=${error.failureCount})`), executionContext.executionId);
|
|
8824
8833
|
toolSpan.setAttribute("tool.result.status", "circuit_breaker_open");
|
|
8825
8834
|
toolSpan.setAttribute("tool.duration_ms", executionTime);
|
|
8826
8835
|
toolSpan.setAttribute("tool.circuit_breaker.state", error.breakerState);
|
|
@@ -8875,7 +8884,7 @@ Current user's request: ${currentInput}`;
|
|
|
8875
8884
|
const category = structuredError.category || ErrorCategory.EXECUTION;
|
|
8876
8885
|
prepared.metrics.errorCategories[category] =
|
|
8877
8886
|
(prepared.metrics.errorCategories[category] || 0) + 1;
|
|
8878
|
-
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, structuredError);
|
|
8887
|
+
this.emitToolEndEvent(toolName, executionContext.executionStartTime, false, undefined, structuredError, executionContext.executionId);
|
|
8879
8888
|
// Gate on listenerCount: Node EventEmitter rethrows the original error
|
|
8880
8889
|
// from emit("error", e) when no listener is registered, which would
|
|
8881
8890
|
// short-circuit the surrounding flow and surface as an unhandled
|