cascade-ai 0.72.0 → 0.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +734 -220
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +734 -220
- package/dist/cli.js.map +1 -1
- package/dist/desktop-core.cjs +722 -220
- package/dist/index.cjs +721 -219
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +89 -0
- package/dist/index.d.ts +89 -0
- package/dist/index.js +721 -219
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.cjs
CHANGED
|
@@ -1667,6 +1667,15 @@ function findPricing(modelId, provider, opts = {}) {
|
|
|
1667
1667
|
}
|
|
1668
1668
|
return null;
|
|
1669
1669
|
}
|
|
1670
|
+
function hasContextBands(model) {
|
|
1671
|
+
if (model.isLocal) return false;
|
|
1672
|
+
const fallbackProviders = model.provider === "azure" ? ["openai"] : model.provider === "gemini" ? ["vertex"] : [];
|
|
1673
|
+
const match = findPricing(model.id, model.provider, {
|
|
1674
|
+
baseModelId: model.baseModelId,
|
|
1675
|
+
fallbackProviders
|
|
1676
|
+
});
|
|
1677
|
+
return (match?.entry.tiers?.length ?? 0) > 1;
|
|
1678
|
+
}
|
|
1670
1679
|
function pricingDatasetMeta() {
|
|
1671
1680
|
const { schemaVersion, generatedAt, defaultCurrency } = DATASET;
|
|
1672
1681
|
return { schemaVersion, generatedAt, defaultCurrency };
|
|
@@ -1810,7 +1819,7 @@ var CASCADE_VERSION, CASCADE_CONFIG_FILE, CASCADE_DB_FILE, CASCADE_DASHBOARD_SEC
|
|
|
1810
1819
|
var init_constants = __esm({
|
|
1811
1820
|
"src/constants.ts"() {
|
|
1812
1821
|
init_pricing();
|
|
1813
|
-
CASCADE_VERSION = "0.
|
|
1822
|
+
CASCADE_VERSION = "0.73.0";
|
|
1814
1823
|
CASCADE_CONFIG_FILE = ".cascade/config.json";
|
|
1815
1824
|
CASCADE_DB_FILE = ".cascade/memory.db";
|
|
1816
1825
|
CASCADE_DASHBOARD_SECRET_FILE = ".cascade/dashboard-secret";
|
|
@@ -2241,8 +2250,23 @@ var init_constants = __esm({
|
|
|
2241
2250
|
});
|
|
2242
2251
|
|
|
2243
2252
|
// src/utils/cost.ts
|
|
2244
|
-
function resolveModelPricing(model) {
|
|
2253
|
+
function resolveModelPricing(model, opts = {}) {
|
|
2245
2254
|
if (model.isLocal) return { input: 0, output: 0, unknown: false };
|
|
2255
|
+
if (opts.inputTokens != null && hasContextBands(model)) {
|
|
2256
|
+
const banded = resolvePricing(model, { inputTokens: opts.inputTokens });
|
|
2257
|
+
const base = resolvePricing(model, { inputTokens: 0 });
|
|
2258
|
+
if (!banded.unknown && !base.unknown) {
|
|
2259
|
+
const scale = (stamped, baseRate) => stamped > 0 && baseRate > 0 ? stamped / baseRate : 1;
|
|
2260
|
+
const inScale = scale(model.inputCostPer1kTokens, base.input);
|
|
2261
|
+
const outScale = scale(model.outputCostPer1kTokens, base.output);
|
|
2262
|
+
return {
|
|
2263
|
+
input: banded.input * inScale,
|
|
2264
|
+
output: banded.output * outScale,
|
|
2265
|
+
unknown: false,
|
|
2266
|
+
...banded.estimatedFromProvider ? { estimatedFromProvider: banded.estimatedFromProvider } : {}
|
|
2267
|
+
};
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2246
2270
|
if (model.inputCostPer1kTokens > 0 || model.outputCostPer1kTokens > 0) {
|
|
2247
2271
|
return {
|
|
2248
2272
|
input: model.inputCostPer1kTokens,
|
|
@@ -2250,7 +2274,7 @@ function resolveModelPricing(model) {
|
|
|
2250
2274
|
unknown: false
|
|
2251
2275
|
};
|
|
2252
2276
|
}
|
|
2253
|
-
const fromDataset = resolvePricing(model);
|
|
2277
|
+
const fromDataset = resolvePricing(model, { inputTokens: opts.inputTokens });
|
|
2254
2278
|
if (!fromDataset.unknown) {
|
|
2255
2279
|
return {
|
|
2256
2280
|
input: fromDataset.input,
|
|
@@ -2272,11 +2296,11 @@ function resolveModelPricing(model) {
|
|
|
2272
2296
|
return { input: 0, output: 0, unknown: true };
|
|
2273
2297
|
}
|
|
2274
2298
|
function calculateCost(inputTokens, outputTokens, model) {
|
|
2275
|
-
const { input, output } = resolveModelPricing(model);
|
|
2299
|
+
const { input, output } = resolveModelPricing(model, { inputTokens });
|
|
2276
2300
|
return inputTokens / 1e3 * input + outputTokens / 1e3 * output;
|
|
2277
2301
|
}
|
|
2278
2302
|
function buildTokenUsage(inputTokens, outputTokens, model) {
|
|
2279
|
-
const { input, output, unknown } = resolveModelPricing(model);
|
|
2303
|
+
const { input, output, unknown } = resolveModelPricing(model, { inputTokens });
|
|
2280
2304
|
return {
|
|
2281
2305
|
inputTokens,
|
|
2282
2306
|
outputTokens,
|
|
@@ -2368,7 +2392,9 @@ var init_model_filter = __esm({
|
|
|
2368
2392
|
// src/providers/anthropic.ts
|
|
2369
2393
|
var anthropic_exports = {};
|
|
2370
2394
|
__export(anthropic_exports, {
|
|
2371
|
-
AnthropicProvider: () => AnthropicProvider
|
|
2395
|
+
AnthropicProvider: () => AnthropicProvider,
|
|
2396
|
+
toAnthropicToolUse: () => toAnthropicToolUse,
|
|
2397
|
+
toAnthropicTools: () => toAnthropicTools
|
|
2372
2398
|
});
|
|
2373
2399
|
function anthropicThinkingParam(modelId, maxTokens) {
|
|
2374
2400
|
if (!/claude-(opus|sonnet)-4/i.test(modelId)) return {};
|
|
@@ -2376,6 +2402,21 @@ function anthropicThinkingParam(modelId, maxTokens) {
|
|
|
2376
2402
|
if (budget < 1024) return {};
|
|
2377
2403
|
return { thinking: { type: "enabled", budget_tokens: budget } };
|
|
2378
2404
|
}
|
|
2405
|
+
function toAnthropicTools(tools) {
|
|
2406
|
+
return tools.map((t) => ({
|
|
2407
|
+
name: t.name,
|
|
2408
|
+
description: t.description,
|
|
2409
|
+
input_schema: t.inputSchema
|
|
2410
|
+
}));
|
|
2411
|
+
}
|
|
2412
|
+
function toAnthropicToolUse(toolCalls) {
|
|
2413
|
+
return toolCalls.map((tc) => ({
|
|
2414
|
+
type: "tool_use",
|
|
2415
|
+
id: tc.id,
|
|
2416
|
+
name: tc.name,
|
|
2417
|
+
input: tc.input
|
|
2418
|
+
}));
|
|
2419
|
+
}
|
|
2379
2420
|
var AnthropicProvider;
|
|
2380
2421
|
var init_anthropic = __esm({
|
|
2381
2422
|
"src/providers/anthropic.ts"() {
|
|
@@ -2404,11 +2445,7 @@ var init_anthropic = __esm({
|
|
|
2404
2445
|
}
|
|
2405
2446
|
async generateStream(options, onChunk) {
|
|
2406
2447
|
const messages = this.convertMessages(options.messages);
|
|
2407
|
-
const tools = options.tools
|
|
2408
|
-
name: t.name,
|
|
2409
|
-
description: t.description,
|
|
2410
|
-
input_schema: t.inputSchema
|
|
2411
|
-
}));
|
|
2448
|
+
const tools = options.tools && toAnthropicTools(options.tools);
|
|
2412
2449
|
let fullContent = "";
|
|
2413
2450
|
let inputTokens = 0;
|
|
2414
2451
|
let outputTokens = 0;
|
|
@@ -2536,14 +2573,7 @@ var init_anthropic = __esm({
|
|
|
2536
2573
|
const content = [];
|
|
2537
2574
|
const text = typeof m.content === "string" ? m.content : "";
|
|
2538
2575
|
if (text) content.push({ type: "text", text });
|
|
2539
|
-
|
|
2540
|
-
content.push({
|
|
2541
|
-
type: "tool_use",
|
|
2542
|
-
id: tc.id,
|
|
2543
|
-
name: tc.name,
|
|
2544
|
-
input: tc.input
|
|
2545
|
-
});
|
|
2546
|
-
}
|
|
2576
|
+
content.push(...toAnthropicToolUse(m.toolCalls ?? []));
|
|
2547
2577
|
if (content.length > 0) {
|
|
2548
2578
|
result.push({ role: "assistant", content });
|
|
2549
2579
|
}
|
|
@@ -2582,7 +2612,9 @@ var openai_exports = {};
|
|
|
2582
2612
|
__export(openai_exports, {
|
|
2583
2613
|
OpenAIProvider: () => OpenAIProvider,
|
|
2584
2614
|
isParamShapeError: () => isParamShapeError,
|
|
2585
|
-
isReasoningModel: () => isReasoningModel
|
|
2615
|
+
isReasoningModel: () => isReasoningModel,
|
|
2616
|
+
toOpenAIToolCalls: () => toOpenAIToolCalls,
|
|
2617
|
+
toOpenAITools: () => toOpenAITools
|
|
2586
2618
|
});
|
|
2587
2619
|
function isReasoningModel(id) {
|
|
2588
2620
|
const m = (id || "").toLowerCase();
|
|
@@ -2593,6 +2625,26 @@ function isParamShapeError(err) {
|
|
|
2593
2625
|
const msg = String(e?.message ?? e?.error?.message ?? "").toLowerCase();
|
|
2594
2626
|
return msg.includes("max_completion_tokens") || msg.includes("max_tokens") && (msg.includes("unsupported") || msg.includes("not supported") || msg.includes("use")) || msg.includes("temperature") && (msg.includes("unsupported") || msg.includes("does not support") || msg.includes("only the default"));
|
|
2595
2627
|
}
|
|
2628
|
+
function toOpenAITools(tools) {
|
|
2629
|
+
return tools.map((t) => ({
|
|
2630
|
+
type: "function",
|
|
2631
|
+
function: {
|
|
2632
|
+
name: t.name,
|
|
2633
|
+
description: t.description,
|
|
2634
|
+
parameters: t.inputSchema
|
|
2635
|
+
}
|
|
2636
|
+
}));
|
|
2637
|
+
}
|
|
2638
|
+
function toOpenAIToolCalls(toolCalls) {
|
|
2639
|
+
return toolCalls.map((tc) => ({
|
|
2640
|
+
id: tc.id,
|
|
2641
|
+
type: "function",
|
|
2642
|
+
function: {
|
|
2643
|
+
name: tc.name,
|
|
2644
|
+
arguments: JSON.stringify(tc.input)
|
|
2645
|
+
}
|
|
2646
|
+
}));
|
|
2647
|
+
}
|
|
2596
2648
|
var OpenAIProvider;
|
|
2597
2649
|
var init_openai = __esm({
|
|
2598
2650
|
"src/providers/openai.ts"() {
|
|
@@ -2619,14 +2671,7 @@ var init_openai = __esm({
|
|
|
2619
2671
|
}
|
|
2620
2672
|
async generateStream(options, onChunk) {
|
|
2621
2673
|
const messages = this.convertMessages(options.messages, options.systemPrompt);
|
|
2622
|
-
const tools = options.tools
|
|
2623
|
-
type: "function",
|
|
2624
|
-
function: {
|
|
2625
|
-
name: t.name,
|
|
2626
|
-
description: t.description,
|
|
2627
|
-
parameters: t.inputSchema
|
|
2628
|
-
}
|
|
2629
|
-
}));
|
|
2674
|
+
const tools = options.tools && toOpenAITools(options.tools);
|
|
2630
2675
|
let fullContent = "";
|
|
2631
2676
|
let inputTokens = 0;
|
|
2632
2677
|
let outputTokens = 0;
|
|
@@ -2772,14 +2817,7 @@ var init_openai = __esm({
|
|
|
2772
2817
|
result.push({
|
|
2773
2818
|
role: "assistant",
|
|
2774
2819
|
content: m.content || "",
|
|
2775
|
-
tool_calls: m.toolCalls
|
|
2776
|
-
id: toolCall.id,
|
|
2777
|
-
type: "function",
|
|
2778
|
-
function: {
|
|
2779
|
-
name: toolCall.name,
|
|
2780
|
-
arguments: JSON.stringify(toolCall.input)
|
|
2781
|
-
}
|
|
2782
|
-
}))
|
|
2820
|
+
tool_calls: toOpenAIToolCalls(m.toolCalls)
|
|
2783
2821
|
});
|
|
2784
2822
|
} else {
|
|
2785
2823
|
result.push({ role: m.role, content: m.content });
|
|
@@ -3046,8 +3084,30 @@ var init_gemini_schema = __esm({
|
|
|
3046
3084
|
// src/providers/gemini.ts
|
|
3047
3085
|
var gemini_exports = {};
|
|
3048
3086
|
__export(gemini_exports, {
|
|
3049
|
-
GeminiProvider: () => GeminiProvider
|
|
3087
|
+
GeminiProvider: () => GeminiProvider,
|
|
3088
|
+
toGeminiFunctionCalls: () => toGeminiFunctionCalls,
|
|
3089
|
+
toGeminiTools: () => toGeminiTools
|
|
3050
3090
|
});
|
|
3091
|
+
function toGeminiTools(tools) {
|
|
3092
|
+
return tools.map((tool) => ({
|
|
3093
|
+
name: tool.name,
|
|
3094
|
+
description: tool.description,
|
|
3095
|
+
// Convert, never cast. Gemini's parameters field is an OpenAPI subset
|
|
3096
|
+
// that 400s on unknown keys, and MCP servers ship schemas carrying their
|
|
3097
|
+
// own extensions (GitHub's adds `x-mcp-header` to every header-bound
|
|
3098
|
+
// property). Casting sent those through verbatim and every request failed
|
|
3099
|
+
// before the model saw it. See providers/gemini-schema.ts.
|
|
3100
|
+
parameters: toGeminiParameters(tool.inputSchema)
|
|
3101
|
+
}));
|
|
3102
|
+
}
|
|
3103
|
+
function toGeminiFunctionCalls(toolCalls) {
|
|
3104
|
+
return toolCalls.map((tc) => ({
|
|
3105
|
+
functionCall: {
|
|
3106
|
+
name: tc.name,
|
|
3107
|
+
args: tc.input
|
|
3108
|
+
}
|
|
3109
|
+
}));
|
|
3110
|
+
}
|
|
3051
3111
|
var GeminiProvider;
|
|
3052
3112
|
var init_gemini = __esm({
|
|
3053
3113
|
"src/providers/gemini.ts"() {
|
|
@@ -3077,7 +3137,7 @@ var init_gemini = __esm({
|
|
|
3077
3137
|
{ category: genai.HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: genai.HarmBlockThreshold.BLOCK_NONE },
|
|
3078
3138
|
{ category: genai.HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: genai.HarmBlockThreshold.BLOCK_NONE }
|
|
3079
3139
|
],
|
|
3080
|
-
tools: options.tools?.length ? [{ functionDeclarations: options.tools
|
|
3140
|
+
tools: options.tools?.length ? [{ functionDeclarations: toGeminiTools(options.tools) }] : void 0,
|
|
3081
3141
|
abortSignal: options.signal
|
|
3082
3142
|
}
|
|
3083
3143
|
});
|
|
@@ -3218,14 +3278,7 @@ var init_gemini = __esm({
|
|
|
3218
3278
|
const parts = [];
|
|
3219
3279
|
const textContent = typeof m.content === "string" ? m.content : "";
|
|
3220
3280
|
if (textContent) parts.push({ text: textContent });
|
|
3221
|
-
|
|
3222
|
-
parts.push({
|
|
3223
|
-
functionCall: {
|
|
3224
|
-
name: tc.name,
|
|
3225
|
-
args: tc.input
|
|
3226
|
-
}
|
|
3227
|
-
});
|
|
3228
|
-
}
|
|
3281
|
+
parts.push(...toGeminiFunctionCalls(m.toolCalls ?? []));
|
|
3229
3282
|
if (parts.length > 0) {
|
|
3230
3283
|
contents.push({ role: "model", parts });
|
|
3231
3284
|
}
|
|
@@ -3272,18 +3325,6 @@ var init_gemini = __esm({
|
|
|
3272
3325
|
}
|
|
3273
3326
|
return parts;
|
|
3274
3327
|
}
|
|
3275
|
-
convertTool(tool) {
|
|
3276
|
-
return {
|
|
3277
|
-
name: tool.name,
|
|
3278
|
-
description: tool.description,
|
|
3279
|
-
// Convert, never cast. Gemini's parameters field is an OpenAPI subset
|
|
3280
|
-
// that 400s on unknown keys, and MCP servers ship schemas carrying their
|
|
3281
|
-
// own extensions (GitHub's adds `x-mcp-header` to every header-bound
|
|
3282
|
-
// property). Casting sent those through verbatim and every request failed
|
|
3283
|
-
// before the model saw it. See providers/gemini-schema.ts.
|
|
3284
|
-
parameters: toGeminiParameters(tool.inputSchema)
|
|
3285
|
-
};
|
|
3286
|
-
}
|
|
3287
3328
|
};
|
|
3288
3329
|
}
|
|
3289
3330
|
});
|
|
@@ -3369,12 +3410,32 @@ var init_net = __esm({
|
|
|
3369
3410
|
// src/providers/ollama.ts
|
|
3370
3411
|
var ollama_exports = {};
|
|
3371
3412
|
__export(ollama_exports, {
|
|
3372
|
-
OllamaProvider: () => OllamaProvider
|
|
3413
|
+
OllamaProvider: () => OllamaProvider,
|
|
3414
|
+
toOllamaToolCalls: () => toOllamaToolCalls,
|
|
3415
|
+
toOllamaTools: () => toOllamaTools
|
|
3373
3416
|
});
|
|
3374
3417
|
function isToolCapable(modelName) {
|
|
3375
3418
|
const name = modelName.toLowerCase();
|
|
3376
3419
|
return TOOL_CAPABLE_FAMILIES.some((family) => name.includes(family));
|
|
3377
3420
|
}
|
|
3421
|
+
function toOllamaTools(tools) {
|
|
3422
|
+
return tools.map((t) => ({
|
|
3423
|
+
type: "function",
|
|
3424
|
+
function: {
|
|
3425
|
+
name: t.name,
|
|
3426
|
+
description: t.description,
|
|
3427
|
+
parameters: t.inputSchema
|
|
3428
|
+
}
|
|
3429
|
+
}));
|
|
3430
|
+
}
|
|
3431
|
+
function toOllamaToolCalls(toolCalls) {
|
|
3432
|
+
return toolCalls.map((tc) => ({
|
|
3433
|
+
function: {
|
|
3434
|
+
name: tc.name,
|
|
3435
|
+
arguments: tc.input
|
|
3436
|
+
}
|
|
3437
|
+
}));
|
|
3438
|
+
}
|
|
3378
3439
|
var TOOL_CAPABLE_FAMILIES, OllamaProvider;
|
|
3379
3440
|
var init_ollama = __esm({
|
|
3380
3441
|
"src/providers/ollama.ts"() {
|
|
@@ -3407,14 +3468,7 @@ var init_ollama = __esm({
|
|
|
3407
3468
|
}
|
|
3408
3469
|
async generateStream(options, onChunk) {
|
|
3409
3470
|
const messages = this.convertMessages(options.messages, options.systemPrompt);
|
|
3410
|
-
const ollamaTools = options.tools
|
|
3411
|
-
type: "function",
|
|
3412
|
-
function: {
|
|
3413
|
-
name: t.name,
|
|
3414
|
-
description: t.description,
|
|
3415
|
-
parameters: t.inputSchema
|
|
3416
|
-
}
|
|
3417
|
-
}));
|
|
3471
|
+
const ollamaTools = options.tools && toOllamaTools(options.tools);
|
|
3418
3472
|
const response = await fetch(`${this.baseUrl}/api/chat`, {
|
|
3419
3473
|
method: "POST",
|
|
3420
3474
|
headers: { "Content-Type": "application/json" },
|
|
@@ -3594,12 +3648,7 @@ var init_ollama = __esm({
|
|
|
3594
3648
|
result.push({
|
|
3595
3649
|
role: "assistant",
|
|
3596
3650
|
content: typeof m.content === "string" ? m.content : "",
|
|
3597
|
-
tool_calls: m.toolCalls
|
|
3598
|
-
function: {
|
|
3599
|
-
name: tc.name,
|
|
3600
|
-
arguments: tc.input
|
|
3601
|
-
}
|
|
3602
|
-
}))
|
|
3651
|
+
tool_calls: toOllamaToolCalls(m.toolCalls)
|
|
3603
3652
|
});
|
|
3604
3653
|
continue;
|
|
3605
3654
|
}
|
|
@@ -6593,6 +6642,21 @@ var DEFAULT_PROVIDER_TPM = {
|
|
|
6593
6642
|
"openai-compatible": 3e4,
|
|
6594
6643
|
ollama: Number.POSITIVE_INFINITY
|
|
6595
6644
|
};
|
|
6645
|
+
function abortReason(signal) {
|
|
6646
|
+
return signal.reason instanceof Error ? signal.reason : new Error("Aborted while waiting for rate-limit capacity");
|
|
6647
|
+
}
|
|
6648
|
+
function sleepUnlessAborted(ms, signal) {
|
|
6649
|
+
if (!signal) return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
6650
|
+
return new Promise((resolve2) => {
|
|
6651
|
+
const done = () => {
|
|
6652
|
+
clearTimeout(timer);
|
|
6653
|
+
signal.removeEventListener("abort", done);
|
|
6654
|
+
resolve2();
|
|
6655
|
+
};
|
|
6656
|
+
const timer = setTimeout(done, ms);
|
|
6657
|
+
signal.addEventListener("abort", done, { once: true });
|
|
6658
|
+
});
|
|
6659
|
+
}
|
|
6596
6660
|
var TpmLimiter = class {
|
|
6597
6661
|
buckets = /* @__PURE__ */ new Map();
|
|
6598
6662
|
constructor(overrides = {}) {
|
|
@@ -6611,9 +6675,10 @@ var TpmLimiter = class {
|
|
|
6611
6675
|
* are reported back via `refund` when short, or simply settled at the next
|
|
6612
6676
|
* refill.
|
|
6613
6677
|
*/
|
|
6614
|
-
async acquire(provider, estimatedTokens) {
|
|
6678
|
+
async acquire(provider, estimatedTokens, signal) {
|
|
6615
6679
|
const bucket = this.buckets.get(provider);
|
|
6616
6680
|
if (!bucket || bucket.tokensPerMinute === Number.POSITIVE_INFINITY) return;
|
|
6681
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
6617
6682
|
const want = Math.min(estimatedTokens, bucket.tokensPerMinute);
|
|
6618
6683
|
for (; ; ) {
|
|
6619
6684
|
this.refill(bucket);
|
|
@@ -6623,7 +6688,8 @@ var TpmLimiter = class {
|
|
|
6623
6688
|
}
|
|
6624
6689
|
const deficit = want - bucket.available;
|
|
6625
6690
|
const waitMs = Math.max(50, Math.ceil(deficit / bucket.tokensPerMinute * 6e4));
|
|
6626
|
-
await
|
|
6691
|
+
await sleepUnlessAborted(waitMs, signal);
|
|
6692
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
6627
6693
|
}
|
|
6628
6694
|
}
|
|
6629
6695
|
/**
|
|
@@ -6669,6 +6735,9 @@ var TpmLimiter = class {
|
|
|
6669
6735
|
};
|
|
6670
6736
|
|
|
6671
6737
|
// src/core/router/local-queue.ts
|
|
6738
|
+
function abortReason2(signal) {
|
|
6739
|
+
return signal.reason instanceof Error ? signal.reason : new Error("Aborted while waiting for a local inference slot");
|
|
6740
|
+
}
|
|
6672
6741
|
var LocalRequestQueue = class {
|
|
6673
6742
|
maxConcurrent;
|
|
6674
6743
|
active = 0;
|
|
@@ -6679,9 +6748,16 @@ var LocalRequestQueue = class {
|
|
|
6679
6748
|
/**
|
|
6680
6749
|
* Acquire a queue slot. Returns a `release` function that MUST be called
|
|
6681
6750
|
* when the inference call is done (even on error). Rejects if the slot
|
|
6682
|
-
* cannot be acquired within `timeoutMs
|
|
6751
|
+
* cannot be acquired within `timeoutMs`, or as soon as `signal` fires.
|
|
6752
|
+
*
|
|
6753
|
+
* The signal matters as much as the timeout: with concurrency of 1 a caller
|
|
6754
|
+
* can sit here for half the inference timeout — 150 seconds by default —
|
|
6755
|
+
* and until it was wired up, neither cancelling the run nor spending its
|
|
6756
|
+
* budget got it out. It waited out the window for a slot it would drop the
|
|
6757
|
+
* moment it received one.
|
|
6683
6758
|
*/
|
|
6684
|
-
async acquire(timeoutMs) {
|
|
6759
|
+
async acquire(timeoutMs, signal) {
|
|
6760
|
+
if (signal?.aborted) throw abortReason2(signal);
|
|
6685
6761
|
if (this.active < this.maxConcurrent) {
|
|
6686
6762
|
this.active++;
|
|
6687
6763
|
return this.makeRelease();
|
|
@@ -6689,23 +6765,36 @@ var LocalRequestQueue = class {
|
|
|
6689
6765
|
return new Promise((resolve2, reject) => {
|
|
6690
6766
|
let settled = false;
|
|
6691
6767
|
let timer;
|
|
6768
|
+
const cleanup = () => {
|
|
6769
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
6770
|
+
signal?.removeEventListener("abort", onAbort);
|
|
6771
|
+
};
|
|
6692
6772
|
const resolver = (release) => {
|
|
6693
6773
|
if (settled) return;
|
|
6694
6774
|
settled = true;
|
|
6695
|
-
|
|
6775
|
+
cleanup();
|
|
6696
6776
|
resolve2(release);
|
|
6697
6777
|
};
|
|
6778
|
+
const abandon = () => {
|
|
6779
|
+
if (settled) return false;
|
|
6780
|
+
settled = true;
|
|
6781
|
+
cleanup();
|
|
6782
|
+
const idx = this.queue.indexOf(resolver);
|
|
6783
|
+
if (idx !== -1) this.queue.splice(idx, 1);
|
|
6784
|
+
return true;
|
|
6785
|
+
};
|
|
6786
|
+
function onAbort() {
|
|
6787
|
+
if (abandon()) reject(abortReason2(signal));
|
|
6788
|
+
}
|
|
6698
6789
|
if (timeoutMs !== void 0 && timeoutMs > 0) {
|
|
6699
6790
|
timer = setTimeout(() => {
|
|
6700
|
-
if (
|
|
6701
|
-
settled = true;
|
|
6702
|
-
const idx = this.queue.indexOf(resolver);
|
|
6703
|
-
if (idx !== -1) this.queue.splice(idx, 1);
|
|
6791
|
+
if (!abandon()) return;
|
|
6704
6792
|
reject(new Error(
|
|
6705
6793
|
`Local model queue: timed out waiting for a free slot after ${timeoutMs}ms. Active: ${this.active}, Queued: ${this.queue.length}. Consider increasing localConcurrency or localInferenceTimeoutMs in your config.`
|
|
6706
6794
|
));
|
|
6707
6795
|
}, timeoutMs);
|
|
6708
6796
|
}
|
|
6797
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
6709
6798
|
this.queue.push(resolver);
|
|
6710
6799
|
});
|
|
6711
6800
|
}
|
|
@@ -6846,6 +6935,116 @@ function fileDeadModelPersistence(filePath) {
|
|
|
6846
6935
|
init_constants();
|
|
6847
6936
|
init_cost();
|
|
6848
6937
|
|
|
6938
|
+
// src/core/context/compaction.ts
|
|
6939
|
+
var CHARS_PER_TOKEN = 4;
|
|
6940
|
+
function estimateTokens(text) {
|
|
6941
|
+
if (!text) return 0;
|
|
6942
|
+
return Math.max(1, Math.ceil(text.length / CHARS_PER_TOKEN));
|
|
6943
|
+
}
|
|
6944
|
+
function contentToText(content) {
|
|
6945
|
+
if (typeof content === "string") return content;
|
|
6946
|
+
return content.map((c) => c.type === "text" ? c.text : c.type === "tool_result" ? c.content : `[${c.type}]`).join(" ");
|
|
6947
|
+
}
|
|
6948
|
+
function messagesTokens(messages) {
|
|
6949
|
+
return messages.reduce((n, m) => n + estimateTokens(contentToText(m.content)), 0);
|
|
6950
|
+
}
|
|
6951
|
+
function atomize(text, maxChars) {
|
|
6952
|
+
const seps = ["\n\n", "\n", ". ", " "];
|
|
6953
|
+
let units = [text];
|
|
6954
|
+
for (const sep of seps) {
|
|
6955
|
+
if (units.every((u) => u.length <= maxChars)) break;
|
|
6956
|
+
const next = [];
|
|
6957
|
+
for (const u of units) {
|
|
6958
|
+
if (u.length <= maxChars) {
|
|
6959
|
+
next.push(u);
|
|
6960
|
+
continue;
|
|
6961
|
+
}
|
|
6962
|
+
const parts = u.split(sep);
|
|
6963
|
+
parts.forEach((p, i) => next.push(i < parts.length - 1 ? p + sep : p));
|
|
6964
|
+
}
|
|
6965
|
+
units = next;
|
|
6966
|
+
}
|
|
6967
|
+
const out = [];
|
|
6968
|
+
for (const u of units) {
|
|
6969
|
+
if (u.length <= maxChars) out.push(u);
|
|
6970
|
+
else for (let i = 0; i < u.length; i += maxChars) out.push(u.slice(i, i + maxChars));
|
|
6971
|
+
}
|
|
6972
|
+
return out;
|
|
6973
|
+
}
|
|
6974
|
+
function chunkText(text, opts) {
|
|
6975
|
+
const maxChars = Math.max(1, Math.floor(opts.maxTokens * 4));
|
|
6976
|
+
const overlap = Math.min(maxChars - 1, Math.max(0, Math.floor(maxChars * (opts.overlapRatio ?? 0.1))));
|
|
6977
|
+
const trimmed = text.trim();
|
|
6978
|
+
if (!trimmed) return [];
|
|
6979
|
+
if (trimmed.length <= maxChars) return [trimmed];
|
|
6980
|
+
const units = atomize(trimmed, maxChars);
|
|
6981
|
+
const chunks = [];
|
|
6982
|
+
let cur = "";
|
|
6983
|
+
for (const u of units) {
|
|
6984
|
+
if (cur && cur.length + u.length > maxChars) {
|
|
6985
|
+
chunks.push(cur.trim());
|
|
6986
|
+
cur = overlap > 0 ? cur.slice(-overlap) : "";
|
|
6987
|
+
}
|
|
6988
|
+
cur += u;
|
|
6989
|
+
}
|
|
6990
|
+
if (cur.trim()) chunks.push(cur.trim());
|
|
6991
|
+
return chunks;
|
|
6992
|
+
}
|
|
6993
|
+
function needsCompaction(totalTokens, contextWindow, reserve = 0.2) {
|
|
6994
|
+
if (!contextWindow || contextWindow <= 0) return false;
|
|
6995
|
+
return totalTokens > contextWindow * (1 - reserve);
|
|
6996
|
+
}
|
|
6997
|
+
async function mapReduceCompact(text, opts) {
|
|
6998
|
+
const capChars = opts.capTokens * 4;
|
|
6999
|
+
let working = text;
|
|
7000
|
+
let truncated = false;
|
|
7001
|
+
if (working.length > capChars) {
|
|
7002
|
+
working = working.slice(0, capChars);
|
|
7003
|
+
truncated = true;
|
|
7004
|
+
}
|
|
7005
|
+
const instruction = opts.instruction ?? "Summarize this content, preserving key facts, names, numbers, and decisions.";
|
|
7006
|
+
let chunks = chunkText(working, { maxTokens: opts.chunkTokens });
|
|
7007
|
+
if (chunks.length <= 1) return { text: working, chunks: chunks.length, calls: 0, truncated };
|
|
7008
|
+
let calls = 0;
|
|
7009
|
+
let parts = await Promise.all(
|
|
7010
|
+
chunks.map((c, i) => {
|
|
7011
|
+
calls++;
|
|
7012
|
+
return opts.summarize(c, `${instruction} (part ${i + 1} of ${chunks.length})`);
|
|
7013
|
+
})
|
|
7014
|
+
);
|
|
7015
|
+
let combined = parts.join("\n\n");
|
|
7016
|
+
const maxPasses = opts.maxReducePasses ?? 3;
|
|
7017
|
+
for (let pass = 0; pass < maxPasses && estimateTokens(combined) > opts.targetTokens; pass++) {
|
|
7018
|
+
const groups = chunkText(combined, { maxTokens: opts.chunkTokens });
|
|
7019
|
+
if (groups.length <= 1) break;
|
|
7020
|
+
parts = await Promise.all(
|
|
7021
|
+
groups.map((g) => {
|
|
7022
|
+
calls++;
|
|
7023
|
+
return opts.summarize(g, "Merge these partial summaries into one cohesive summary; keep all distinct facts.");
|
|
7024
|
+
})
|
|
7025
|
+
);
|
|
7026
|
+
combined = parts.join("\n\n");
|
|
7027
|
+
}
|
|
7028
|
+
const note = truncated ? "\n\n[Note: input exceeded the extended-context cap and was truncated.]" : "";
|
|
7029
|
+
return { text: combined + note, chunks: chunks.length, calls, truncated };
|
|
7030
|
+
}
|
|
7031
|
+
async function rollingSummary(messages, opts) {
|
|
7032
|
+
if (messages.length <= opts.keepRecent) return messages;
|
|
7033
|
+
const older = messages.slice(0, messages.length - opts.keepRecent);
|
|
7034
|
+
const recent = messages.slice(messages.length - opts.keepRecent);
|
|
7035
|
+
const transcript = older.map((m) => `${m.role}: ${contentToText(m.content)}`).join("\n");
|
|
7036
|
+
const summary = await opts.summarize(
|
|
7037
|
+
transcript,
|
|
7038
|
+
"Summarize this earlier conversation. Preserve facts, decisions, names, numbers, and any open threads or todos."
|
|
7039
|
+
);
|
|
7040
|
+
const summaryMessage = {
|
|
7041
|
+
role: "system",
|
|
7042
|
+
content: `Summary of earlier conversation (folded to fit the context window):
|
|
7043
|
+
${summary}`
|
|
7044
|
+
};
|
|
7045
|
+
return [summaryMessage, ...recent];
|
|
7046
|
+
}
|
|
7047
|
+
|
|
6849
7048
|
// src/utils/retry.ts
|
|
6850
7049
|
var CascadeCancelledError = class extends Error {
|
|
6851
7050
|
constructor(reason) {
|
|
@@ -6867,6 +7066,162 @@ async function withTimeout(promise, timeoutMs, errorMessage = "Operation timed o
|
|
|
6867
7066
|
if (timer !== void 0) clearTimeout(timer);
|
|
6868
7067
|
}
|
|
6869
7068
|
}
|
|
7069
|
+
async function withTimeoutAbort(run, timeoutMs, errorMessage = "Operation timed out", outer) {
|
|
7070
|
+
const outers = (Array.isArray(outer) ? outer : [outer]).filter((sig) => sig !== void 0);
|
|
7071
|
+
const controller = new AbortController();
|
|
7072
|
+
let timer;
|
|
7073
|
+
let rejectRace;
|
|
7074
|
+
const settle = (reason) => {
|
|
7075
|
+
rejectRace?.(reason);
|
|
7076
|
+
controller.abort(reason);
|
|
7077
|
+
};
|
|
7078
|
+
const abortOuter = () => {
|
|
7079
|
+
const fired = outers.find((sig) => sig.aborted);
|
|
7080
|
+
settle(fired?.reason instanceof Error ? fired.reason : new CascadeCancelledError("Run cancelled"));
|
|
7081
|
+
};
|
|
7082
|
+
const failPromise = new Promise((_, reject) => {
|
|
7083
|
+
rejectRace = reject;
|
|
7084
|
+
timer = setTimeout(() => settle(new Error(errorMessage)), timeoutMs);
|
|
7085
|
+
});
|
|
7086
|
+
try {
|
|
7087
|
+
if (outers.some((sig) => sig.aborted)) {
|
|
7088
|
+
abortOuter();
|
|
7089
|
+
return await failPromise;
|
|
7090
|
+
}
|
|
7091
|
+
for (const sig of outers) sig.addEventListener("abort", abortOuter, { once: true });
|
|
7092
|
+
return await Promise.race([run(controller.signal), failPromise]);
|
|
7093
|
+
} finally {
|
|
7094
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
7095
|
+
for (const sig of outers) sig.removeEventListener("abort", abortOuter);
|
|
7096
|
+
}
|
|
7097
|
+
}
|
|
7098
|
+
function anySignal(sources) {
|
|
7099
|
+
const live = sources.filter((s) => s !== void 0);
|
|
7100
|
+
if (live.length === 0) return { release: () => {
|
|
7101
|
+
} };
|
|
7102
|
+
if (live.length === 1) return { signal: live[0], release: () => {
|
|
7103
|
+
} };
|
|
7104
|
+
const controller = new AbortController();
|
|
7105
|
+
function detach() {
|
|
7106
|
+
for (const s of live) s.removeEventListener("abort", onAbort);
|
|
7107
|
+
}
|
|
7108
|
+
function onAbort() {
|
|
7109
|
+
const fired = live.find((s) => s.aborted);
|
|
7110
|
+
controller.abort(fired?.reason instanceof Error ? fired.reason : new CascadeCancelledError("Run cancelled"));
|
|
7111
|
+
detach();
|
|
7112
|
+
}
|
|
7113
|
+
if (live.some((s) => s.aborted)) {
|
|
7114
|
+
onAbort();
|
|
7115
|
+
return { signal: controller.signal, release: detach };
|
|
7116
|
+
}
|
|
7117
|
+
for (const s of live) s.addEventListener("abort", onAbort, { once: true });
|
|
7118
|
+
return { signal: controller.signal, release: detach };
|
|
7119
|
+
}
|
|
7120
|
+
|
|
7121
|
+
// src/core/router/wire-profile.ts
|
|
7122
|
+
init_anthropic();
|
|
7123
|
+
init_gemini();
|
|
7124
|
+
init_ollama();
|
|
7125
|
+
init_openai();
|
|
7126
|
+
var textAndImageOnly = (block) => block.type === "text" || block.type === "image";
|
|
7127
|
+
var ANTHROPIC = {
|
|
7128
|
+
// convertMessages: `if (m.role === 'system') continue` — history system
|
|
7129
|
+
// messages are skipped outright; only options.systemPrompt goes as system
|
|
7130
|
+
// input. Compaction emits exactly such a message, holding a summary of the
|
|
7131
|
+
// whole conversation, so charging it refused runs over input never sent.
|
|
7132
|
+
dropsMessage: (m) => m.role === "system",
|
|
7133
|
+
blockHandling: (m) => {
|
|
7134
|
+
if (m.role === "tool") return "stringified";
|
|
7135
|
+
if (m.role === "assistant") return "dropped";
|
|
7136
|
+
return "blocks";
|
|
7137
|
+
},
|
|
7138
|
+
sendsToolCalls: (m) => m.role === "assistant",
|
|
7139
|
+
sendsBlock: textAndImageOnly,
|
|
7140
|
+
sendsToolCallId: true,
|
|
7141
|
+
sizeToolCalls: toAnthropicToolUse,
|
|
7142
|
+
sendsUrlImages: true,
|
|
7143
|
+
readsTopLevelImages: false,
|
|
7144
|
+
sizeTools: toAnthropicTools
|
|
7145
|
+
};
|
|
7146
|
+
var GEMINI = {
|
|
7147
|
+
// buildContents folds a system message into the next user turn, but only
|
|
7148
|
+
// when it has text: array content collapses to '' and the message is
|
|
7149
|
+
// skipped, as is a whitespace-only string.
|
|
7150
|
+
dropsMessage: (m) => m.role === "system" && !(typeof m.content === "string" ? m.content : "").trim(),
|
|
7151
|
+
blockHandling: (m) => {
|
|
7152
|
+
if (m.role === "tool") return "stringified";
|
|
7153
|
+
if (m.role === "assistant" || m.role === "system") return "dropped";
|
|
7154
|
+
return "blocks";
|
|
7155
|
+
},
|
|
7156
|
+
sendsToolCalls: (m) => m.role === "assistant",
|
|
7157
|
+
sendsBlock: textAndImageOnly,
|
|
7158
|
+
sendsToolCallId: true,
|
|
7159
|
+
sizeToolCalls: toGeminiFunctionCalls,
|
|
7160
|
+
sendsUrlImages: false,
|
|
7161
|
+
readsTopLevelImages: true,
|
|
7162
|
+
sizeTools: toGeminiTools
|
|
7163
|
+
};
|
|
7164
|
+
var OPENAI = {
|
|
7165
|
+
dropsMessage: () => false,
|
|
7166
|
+
blockHandling: (m) => {
|
|
7167
|
+
if (m.role === "system" || m.role === "tool") return "dropped";
|
|
7168
|
+
return "blocks";
|
|
7169
|
+
},
|
|
7170
|
+
// The tool_calls branch is inside `if (typeof m.content === 'string')`; an
|
|
7171
|
+
// assistant turn with array content falls through to the parts branch and
|
|
7172
|
+
// its tool calls are never attached.
|
|
7173
|
+
sendsToolCalls: (m) => m.role === "assistant" && typeof m.content === "string",
|
|
7174
|
+
sendsBlock: textAndImageOnly,
|
|
7175
|
+
sendsToolCallId: true,
|
|
7176
|
+
sizeToolCalls: toOpenAIToolCalls,
|
|
7177
|
+
sendsUrlImages: true,
|
|
7178
|
+
readsTopLevelImages: false,
|
|
7179
|
+
sizeTools: toOpenAITools
|
|
7180
|
+
};
|
|
7181
|
+
var OLLAMA = {
|
|
7182
|
+
dropsMessage: () => false,
|
|
7183
|
+
blockHandling: (m) => {
|
|
7184
|
+
if (m.role === "system") return "dropped";
|
|
7185
|
+
if (m.role === "tool") return "stringified";
|
|
7186
|
+
if (m.role === "assistant" && m.toolCalls?.length) return "dropped";
|
|
7187
|
+
return "blocks";
|
|
7188
|
+
},
|
|
7189
|
+
sendsToolCalls: (m) => m.role === "assistant",
|
|
7190
|
+
// The image split maps `b.image.data` with no check on `type`, so a URL is
|
|
7191
|
+
// submitted too (as a URL string in the `images` array).
|
|
7192
|
+
sendsBlock: textAndImageOnly,
|
|
7193
|
+
sendsToolCallId: false,
|
|
7194
|
+
sizeToolCalls: toOllamaToolCalls,
|
|
7195
|
+
sendsUrlImages: true,
|
|
7196
|
+
readsTopLevelImages: false,
|
|
7197
|
+
sizeTools: toOllamaTools
|
|
7198
|
+
};
|
|
7199
|
+
var PROFILES = {
|
|
7200
|
+
anthropic: ANTHROPIC,
|
|
7201
|
+
gemini: GEMINI,
|
|
7202
|
+
openai: OPENAI,
|
|
7203
|
+
// AzureOpenAIProvider and OpenAICompatibleProvider both extend OpenAIProvider
|
|
7204
|
+
// and neither overrides convertMessages.
|
|
7205
|
+
azure: OPENAI,
|
|
7206
|
+
"openai-compatible": OPENAI,
|
|
7207
|
+
ollama: OLLAMA
|
|
7208
|
+
};
|
|
7209
|
+
function wireProfile(provider) {
|
|
7210
|
+
return PROFILES[provider] ?? OPENAI;
|
|
7211
|
+
}
|
|
7212
|
+
function geminiImageCopies(messages) {
|
|
7213
|
+
for (const m of messages) {
|
|
7214
|
+
if (m.role === "user") {
|
|
7215
|
+
const laterUserTurn = messages.slice(messages.indexOf(m) + 1).some((x) => x.role === "user");
|
|
7216
|
+
return laterUserTurn ? 2 : 1;
|
|
7217
|
+
}
|
|
7218
|
+
const text = typeof m.content === "string" ? m.content : "";
|
|
7219
|
+
if (m.role === "system" && text.trim()) return 1;
|
|
7220
|
+
if (m.role === "tool") return 1;
|
|
7221
|
+
if (m.role === "assistant" && (text || m.toolCalls?.length)) return 1;
|
|
7222
|
+
}
|
|
7223
|
+
return 1;
|
|
7224
|
+
}
|
|
6870
7225
|
|
|
6871
7226
|
// src/core/router/model-profiler.ts
|
|
6872
7227
|
var SKIP_PATTERN = /embed|dall-e|whisper|tts|vision|instruct-vision|rerank/i;
|
|
@@ -7376,6 +7731,36 @@ function azureModelForTier(tier, ranked) {
|
|
|
7376
7731
|
if (tier === "T3") return ranked[ranked.length - 1];
|
|
7377
7732
|
return ranked[Math.min(1, ranked.length - 1)];
|
|
7378
7733
|
}
|
|
7734
|
+
var IMAGE_TOKENS_MAX = 2e3;
|
|
7735
|
+
var IMAGE_TOKENS_EACH = IMAGE_TOKENS_MAX;
|
|
7736
|
+
function safeJson(value) {
|
|
7737
|
+
try {
|
|
7738
|
+
return JSON.stringify(value) ?? "";
|
|
7739
|
+
} catch {
|
|
7740
|
+
return "";
|
|
7741
|
+
}
|
|
7742
|
+
}
|
|
7743
|
+
function newRunAbort() {
|
|
7744
|
+
const controller = new AbortController();
|
|
7745
|
+
EventEmitter.setMaxListeners(64, controller.signal);
|
|
7746
|
+
return controller;
|
|
7747
|
+
}
|
|
7748
|
+
var TOKENS_PER_MESSAGE_FRAMING = 4;
|
|
7749
|
+
function guardTokens(text) {
|
|
7750
|
+
if (!text) return 0;
|
|
7751
|
+
let asciiChars = 0;
|
|
7752
|
+
let denseBytes = 0;
|
|
7753
|
+
for (const ch of text) {
|
|
7754
|
+
const cp = ch.codePointAt(0);
|
|
7755
|
+
if (cp <= 127) {
|
|
7756
|
+
asciiChars++;
|
|
7757
|
+
continue;
|
|
7758
|
+
}
|
|
7759
|
+
denseBytes += cp <= 2047 ? 2 : cp <= 65535 ? 3 : 4;
|
|
7760
|
+
}
|
|
7761
|
+
if (denseBytes === 0) return estimateTokens(text);
|
|
7762
|
+
return Math.ceil(asciiChars / CHARS_PER_TOKEN) + denseBytes;
|
|
7763
|
+
}
|
|
7379
7764
|
var DISCOVERY_TTL_MS = 15 * 60 * 1e3;
|
|
7380
7765
|
var DISCOVERY_TIMEOUT_MS = 4e3;
|
|
7381
7766
|
var cloudDiscoveryCache = /* @__PURE__ */ new Map();
|
|
@@ -7428,6 +7813,33 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7428
7813
|
// start of every `cascade run`, independent of the session-wide budget.
|
|
7429
7814
|
runTokens = 0;
|
|
7430
7815
|
runCostUsd = 0;
|
|
7816
|
+
/**
|
|
7817
|
+
* Budget held for calls that are IN FLIGHT — admitted by the preflight check
|
|
7818
|
+
* but not yet counted by recordStats. Without these, concurrent calls all
|
|
7819
|
+
* measure themselves against the same unspent allowance and are all admitted.
|
|
7820
|
+
* Released when each call settles; reset by beginRun() alongside the totals.
|
|
7821
|
+
*/
|
|
7822
|
+
reservedTokens = 0;
|
|
7823
|
+
reservedCostUsd = 0;
|
|
7824
|
+
/**
|
|
7825
|
+
* Fires when this run's budget is spent, cancelling work already in flight.
|
|
7826
|
+
*
|
|
7827
|
+
* The kill switch was previously only a flag, which stops calls that have
|
|
7828
|
+
* not yet submitted. A parallel wave's earlier members are already at the
|
|
7829
|
+
* provider by then, and they carried on generating for a run whose output
|
|
7830
|
+
* would be discarded. Every provider call chains to this, so tripping the
|
|
7831
|
+
* ceiling reaches the requests that are running, not just the ones queued.
|
|
7832
|
+
*/
|
|
7833
|
+
runAbort = newRunAbort();
|
|
7834
|
+
/**
|
|
7835
|
+
* Bumped by every beginRun(). A release handle remembers the generation it
|
|
7836
|
+
* was issued in and does nothing if that has moved on: beginRun() zeroes the
|
|
7837
|
+
* counters, so a call still in flight from the previous run would otherwise
|
|
7838
|
+
* subtract its estimate from the NEW run's totals and drive them negative —
|
|
7839
|
+
* leaving later preflights believing there is more allowance than the cap
|
|
7840
|
+
* actually holds, which is worse than no gate at all.
|
|
7841
|
+
*/
|
|
7842
|
+
runGeneration = 0;
|
|
7431
7843
|
runBudgetExceeded = false;
|
|
7432
7844
|
runBudgetExceededReason;
|
|
7433
7845
|
/**
|
|
@@ -7597,7 +8009,7 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7597
8009
|
try {
|
|
7598
8010
|
const cfg = this.config.providers.find((p) => p.type === model.provider) ?? { type: model.provider };
|
|
7599
8011
|
const provider = this.createProvider(cfg, model);
|
|
7600
|
-
const result = await
|
|
8012
|
+
const result = await withTimeoutAbort((signal) => provider.generate({
|
|
7601
8013
|
messages: [{ role: "user", content: "Call the echo tool with text set to 'ping'. Use the tool; do not answer in prose." }],
|
|
7602
8014
|
tools: [{
|
|
7603
8015
|
name: "echo",
|
|
@@ -7605,7 +8017,8 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7605
8017
|
inputSchema: { type: "object", properties: { text: { type: "string", description: "Text to echo" } }, required: ["text"] }
|
|
7606
8018
|
}],
|
|
7607
8019
|
maxTokens: 80,
|
|
7608
|
-
temperature: 0
|
|
8020
|
+
temperature: 0,
|
|
8021
|
+
signal
|
|
7609
8022
|
}), 3e4, "tool-support probe timed out");
|
|
7610
8023
|
return (result.toolCalls?.length ?? 0) > 0;
|
|
7611
8024
|
} catch {
|
|
@@ -7726,9 +8139,12 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7726
8139
|
this.runBudgetExceededReason ?? "Per-task budget exceeded."
|
|
7727
8140
|
);
|
|
7728
8141
|
}
|
|
8142
|
+
const runGeneration = this.runGeneration;
|
|
8143
|
+
const runAbort = this.runAbort;
|
|
8144
|
+
const runSignal = this.runSignal;
|
|
7729
8145
|
options = applyTierLimits(options, tier, this.config?.tierLimits);
|
|
7730
|
-
if (
|
|
7731
|
-
options = { ...options, signal:
|
|
8146
|
+
if (runSignal && !options.signal) {
|
|
8147
|
+
options = { ...options, signal: runSignal };
|
|
7732
8148
|
}
|
|
7733
8149
|
if (options.model && !requireVision) {
|
|
7734
8150
|
this.ensureProvider(options.model, this.config.providers);
|
|
@@ -7750,58 +8166,84 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7750
8166
|
if (!model) throw new Error(`No model available for tier ${tier}`);
|
|
7751
8167
|
const provider = this.getProvider(model);
|
|
7752
8168
|
if (!provider) throw new Error(`No provider for model ${model.id}`);
|
|
8169
|
+
let releaseReservation = this.enforcePreflightBudget(model, options);
|
|
7753
8170
|
const requestedTokens = options.maxTokens ?? model.maxOutputTokens ?? 1024;
|
|
7754
8171
|
const estimatedTokens = Math.min(requestedTokens, model.maxOutputTokens ?? requestedTokens) + 512;
|
|
7755
8172
|
if (this.tpmLimiter) {
|
|
7756
|
-
|
|
8173
|
+
const wait = anySignal([options.signal, runAbort.signal]);
|
|
8174
|
+
try {
|
|
8175
|
+
await this.tpmLimiter.acquire(model.provider, estimatedTokens, wait.signal);
|
|
8176
|
+
} catch (err) {
|
|
8177
|
+
releaseReservation?.();
|
|
8178
|
+
throw this.asAbortFailure(err, options.signal, runSignal);
|
|
8179
|
+
} finally {
|
|
8180
|
+
wait.release();
|
|
8181
|
+
}
|
|
7757
8182
|
}
|
|
7758
8183
|
const useStream = Boolean(onChunk) && model.supportsStreaming && typeof provider.generateStream === "function";
|
|
7759
8184
|
let releaseLocalSlot;
|
|
7760
8185
|
if (model.isLocal) {
|
|
7761
8186
|
const inferenceTimeoutMs = this.config.localInferenceTimeoutMs ?? 3e5;
|
|
7762
8187
|
const queueWaitMs = Math.round(inferenceTimeoutMs / 2);
|
|
7763
|
-
|
|
8188
|
+
const wait = anySignal([options.signal, runAbort.signal]);
|
|
8189
|
+
try {
|
|
8190
|
+
releaseLocalSlot = await this.localQueue.acquire(queueWaitMs, wait.signal);
|
|
8191
|
+
} catch (err) {
|
|
8192
|
+
releaseReservation?.();
|
|
8193
|
+
throw this.asAbortFailure(err, options.signal, runSignal);
|
|
8194
|
+
} finally {
|
|
8195
|
+
wait.release();
|
|
8196
|
+
}
|
|
7764
8197
|
}
|
|
7765
8198
|
try {
|
|
8199
|
+
if (runGeneration === this.runGeneration && this.runBudgetExceeded) {
|
|
8200
|
+
this.tpmLimiter?.refund(model.provider, estimatedTokens);
|
|
8201
|
+
throw new _CascadeRouter.BudgetExceededError(
|
|
8202
|
+
this.runBudgetExceededReason ?? "Per-task budget exceeded."
|
|
8203
|
+
);
|
|
8204
|
+
}
|
|
7766
8205
|
let result;
|
|
7767
8206
|
if (model.isLocal) {
|
|
7768
8207
|
const inferenceTimeoutMs = this.config.localInferenceTimeoutMs ?? 3e5;
|
|
7769
|
-
|
|
7770
|
-
|
|
7771
|
-
|
|
7772
|
-
|
|
7773
|
-
|
|
7774
|
-
inferencePromise,
|
|
8208
|
+
result = await withTimeoutAbort(
|
|
8209
|
+
(signal) => useStream && onChunk ? provider.generateStream({ ...options, signal }, (chunk) => {
|
|
8210
|
+
const text = typeof chunk?.text === "string" ? chunk.text : "";
|
|
8211
|
+
if (text) onChunk({ ...chunk, text });
|
|
8212
|
+
}) : provider.generate({ ...options, signal }),
|
|
7775
8213
|
inferenceTimeoutMs,
|
|
7776
|
-
`Local model ${model.id} inference timed out after ${inferenceTimeoutMs}ms
|
|
8214
|
+
`Local model ${model.id} inference timed out after ${inferenceTimeoutMs}ms`,
|
|
8215
|
+
[options.signal, runAbort.signal]
|
|
7777
8216
|
);
|
|
7778
8217
|
} else if (useStream && onChunk) {
|
|
7779
8218
|
const cloudTimeoutMs = this.config.cloudInferenceTimeoutMs ?? 12e4;
|
|
7780
8219
|
try {
|
|
7781
|
-
result = await
|
|
7782
|
-
provider.generateStream(options, (chunk) => {
|
|
8220
|
+
result = await withTimeoutAbort(
|
|
8221
|
+
(signal) => provider.generateStream({ ...options, signal }, (chunk) => {
|
|
7783
8222
|
const text = typeof chunk?.text === "string" ? chunk.text : "";
|
|
7784
8223
|
if (text) onChunk({ ...chunk, text });
|
|
7785
8224
|
}),
|
|
7786
8225
|
cloudTimeoutMs,
|
|
7787
|
-
`Model ${model.id} stream timed out after ${cloudTimeoutMs}ms
|
|
8226
|
+
`Model ${model.id} stream timed out after ${cloudTimeoutMs}ms`,
|
|
8227
|
+
[options.signal, runAbort.signal]
|
|
7788
8228
|
);
|
|
7789
8229
|
} catch (streamErr) {
|
|
7790
|
-
if (streamErr instanceof Error && streamErr.name === "AbortError" ||
|
|
8230
|
+
if (streamErr instanceof Error && streamErr.name === "AbortError" || runSignal?.aborted || options.signal?.aborted) {
|
|
7791
8231
|
throw streamErr;
|
|
7792
8232
|
}
|
|
7793
|
-
result = await
|
|
7794
|
-
provider.generate(options),
|
|
8233
|
+
result = await withTimeoutAbort(
|
|
8234
|
+
(signal) => provider.generate({ ...options, signal }),
|
|
7795
8235
|
cloudTimeoutMs,
|
|
7796
|
-
`Model ${model.id} inference timed out after ${cloudTimeoutMs}ms
|
|
8236
|
+
`Model ${model.id} inference timed out after ${cloudTimeoutMs}ms`,
|
|
8237
|
+
[options.signal, runAbort.signal]
|
|
7797
8238
|
);
|
|
7798
8239
|
}
|
|
7799
8240
|
} else {
|
|
7800
8241
|
const cloudTimeoutMs = this.config.cloudInferenceTimeoutMs ?? 12e4;
|
|
7801
|
-
result = await
|
|
7802
|
-
provider.generate(options),
|
|
8242
|
+
result = await withTimeoutAbort(
|
|
8243
|
+
(signal) => provider.generate({ ...options, signal }),
|
|
7803
8244
|
cloudTimeoutMs,
|
|
7804
|
-
`Model ${model.id} inference timed out after ${cloudTimeoutMs}ms
|
|
8245
|
+
`Model ${model.id} inference timed out after ${cloudTimeoutMs}ms`,
|
|
8246
|
+
[options.signal, runAbort.signal]
|
|
7805
8247
|
);
|
|
7806
8248
|
}
|
|
7807
8249
|
const corrected = buildTokenUsage(
|
|
@@ -7820,11 +8262,12 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7820
8262
|
if (!result || typeof result.content !== "string" || !result.usage) {
|
|
7821
8263
|
throw new Error(`Provider ${model.provider}:${model.id} returned an invalid generation result.`);
|
|
7822
8264
|
}
|
|
7823
|
-
this.recordStats(tier, model, result.usage, options.featureTag);
|
|
8265
|
+
this.recordStats(tier, model, result.usage, options.featureTag, runGeneration);
|
|
7824
8266
|
this.failover.recordSuccess(model.provider);
|
|
7825
8267
|
return result;
|
|
7826
8268
|
} catch (err) {
|
|
7827
|
-
if (err instanceof
|
|
8269
|
+
if (err instanceof _CascadeRouter.BudgetExceededError) throw err;
|
|
8270
|
+
if (err instanceof Error && err.name === "AbortError" || runSignal?.aborted || options.signal?.aborted) {
|
|
7828
8271
|
throw new CascadeCancelledError("Run cancelled");
|
|
7829
8272
|
}
|
|
7830
8273
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -7842,6 +8285,8 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7842
8285
|
});
|
|
7843
8286
|
releaseLocalSlot?.();
|
|
7844
8287
|
releaseLocalSlot = void 0;
|
|
8288
|
+
releaseReservation?.();
|
|
8289
|
+
releaseReservation = void 0;
|
|
7845
8290
|
const retryOpts = options.model && options.model.id === model.id ? { ...options, model: void 0 } : options;
|
|
7846
8291
|
return this.generate(tier, retryOpts, onChunk, requireVision);
|
|
7847
8292
|
}
|
|
@@ -7863,6 +8308,8 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7863
8308
|
});
|
|
7864
8309
|
releaseLocalSlot?.();
|
|
7865
8310
|
releaseLocalSlot = void 0;
|
|
8311
|
+
releaseReservation?.();
|
|
8312
|
+
releaseReservation = void 0;
|
|
7866
8313
|
const base = options.model && options.model.id === model.id ? { ...options, model: void 0 } : options;
|
|
7867
8314
|
const retryOpts = { ...base, _notFoundDepth: depth };
|
|
7868
8315
|
return this.generate(tier, retryOpts, onChunk, requireVision);
|
|
@@ -7871,6 +8318,8 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
7871
8318
|
throw err;
|
|
7872
8319
|
} finally {
|
|
7873
8320
|
releaseLocalSlot?.();
|
|
8321
|
+
releaseReservation?.();
|
|
8322
|
+
releaseReservation = void 0;
|
|
7874
8323
|
}
|
|
7875
8324
|
}
|
|
7876
8325
|
getModelForTier(tier) {
|
|
@@ -8241,7 +8690,14 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
8241
8690
|
}
|
|
8242
8691
|
return void 0;
|
|
8243
8692
|
}
|
|
8244
|
-
|
|
8693
|
+
/**
|
|
8694
|
+
* `runGeneration` is the generation the call was ADMITTED in — see
|
|
8695
|
+
* `generate()`. Session-wide accounting always applies (the spend is real
|
|
8696
|
+
* whenever it lands), but the per-RUN ceiling is only meaningful for the run
|
|
8697
|
+
* the call belongs to, so a straggler from a finished run is left out of it.
|
|
8698
|
+
* Omitted by the direct callers in tests, which have no run to straddle.
|
|
8699
|
+
*/
|
|
8700
|
+
recordStats(tier, model, usage, featureTag, runGeneration) {
|
|
8245
8701
|
this.stats.totalTokens += usage.totalTokens;
|
|
8246
8702
|
this.stats.totalCostUsd += usage.estimatedCostUsd;
|
|
8247
8703
|
this.sessionCostUsd += usage.estimatedCostUsd;
|
|
@@ -8266,10 +8722,13 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
8266
8722
|
});
|
|
8267
8723
|
}
|
|
8268
8724
|
}
|
|
8269
|
-
|
|
8270
|
-
|
|
8725
|
+
const currentRun = runGeneration === void 0 || runGeneration === this.runGeneration;
|
|
8726
|
+
if (currentRun) {
|
|
8727
|
+
this.runTokens += usage.totalTokens;
|
|
8728
|
+
this.runCostUsd += usage.estimatedCostUsd;
|
|
8729
|
+
}
|
|
8271
8730
|
this.updateBudgetState();
|
|
8272
|
-
this.enforceRunBudget();
|
|
8731
|
+
if (currentRun) this.enforceRunBudget();
|
|
8273
8732
|
}
|
|
8274
8733
|
/**
|
|
8275
8734
|
* Resets per-run accounting at the start of each `cascade run`. Session
|
|
@@ -8279,9 +8738,158 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
8279
8738
|
beginRun() {
|
|
8280
8739
|
this.runTokens = 0;
|
|
8281
8740
|
this.runCostUsd = 0;
|
|
8741
|
+
this.reservedTokens = 0;
|
|
8742
|
+
this.reservedCostUsd = 0;
|
|
8743
|
+
this.runAbort = newRunAbort();
|
|
8744
|
+
this.runGeneration++;
|
|
8282
8745
|
this.runBudgetExceeded = false;
|
|
8283
8746
|
this.runBudgetExceededReason = void 0;
|
|
8284
8747
|
}
|
|
8748
|
+
/**
|
|
8749
|
+
* Refuse a call whose input alone cannot fit what is left of the run budget.
|
|
8750
|
+
*
|
|
8751
|
+
* `enforceRunBudget()` below runs AFTER a call returns, which makes the caps
|
|
8752
|
+
* a stop rather than a pre-authorisation: the tokens are already bought by
|
|
8753
|
+
* the time it looks. That was tolerable while a prompt could not exceed
|
|
8754
|
+
* 20,000 characters, and stopped being so when that cap was removed in
|
|
8755
|
+
* 0.72.0 — a multi-megabyte prompt is billed in full on the very first call
|
|
8756
|
+
* (the complexity classifier sends the whole thing), and no ceiling
|
|
8757
|
+
* configured downstream can give that money back.
|
|
8758
|
+
*
|
|
8759
|
+
* Deliberately narrow, because a false refusal is worse than a late stop:
|
|
8760
|
+
*
|
|
8761
|
+
* • INPUT ONLY. What a call returns is not knowable in advance, and
|
|
8762
|
+
* charging a worst-case `maxTokens` of output against the budget would
|
|
8763
|
+
* decline runs that would have finished comfortably. Input is the part
|
|
8764
|
+
* that is already determined, and it is the part the removed cap used to
|
|
8765
|
+
* bound.
|
|
8766
|
+
* • Compared against what REMAINS (`cap - spent`), not the whole cap, so a
|
|
8767
|
+
* call is judged on the budget it can actually draw from.
|
|
8768
|
+
* • Skipped when the model has no usable price. An estimate cannot be made,
|
|
8769
|
+
* and refusing on ignorance would break every self-hosted and local model
|
|
8770
|
+
* the moment a cost cap is set. The post-hoc stop still applies there.
|
|
8771
|
+
* • Skipped entirely when no cap is configured — there is nothing to
|
|
8772
|
+
* pre-authorise against.
|
|
8773
|
+
*
|
|
8774
|
+
* The message names the cap AND the estimate, because "too expensive" with
|
|
8775
|
+
* no numbers gives the user nothing to change.
|
|
8776
|
+
*/
|
|
8777
|
+
enforcePreflightBudget(model, options) {
|
|
8778
|
+
const budget = this.config?.budget;
|
|
8779
|
+
const maxCost = budget?.maxCostPerRunUsd;
|
|
8780
|
+
const maxTokens = budget?.maxTokensPerRun;
|
|
8781
|
+
if (maxCost == null && maxTokens == null) return void 0;
|
|
8782
|
+
const inputTokens = this.estimateInputTokens(options, model);
|
|
8783
|
+
if (maxTokens != null) {
|
|
8784
|
+
const remaining = maxTokens - this.runTokens - this.reservedTokens;
|
|
8785
|
+
if (inputTokens > remaining) {
|
|
8786
|
+
this.failPreflight(
|
|
8787
|
+
`This request's input is about ${inputTokens.toLocaleString()} tokens, but only ${Math.max(0, remaining).toLocaleString()} of the per-task cap of ${maxTokens.toLocaleString()} remain. Shorten the input, or raise budget.maxTokensPerRun.`
|
|
8788
|
+
);
|
|
8789
|
+
}
|
|
8790
|
+
}
|
|
8791
|
+
let estimatedUsd = 0;
|
|
8792
|
+
if (maxCost != null) {
|
|
8793
|
+
const { input: inputPer1k, unknown } = resolveModelPricing(model, { inputTokens });
|
|
8794
|
+
if (!unknown && inputPer1k > 0) {
|
|
8795
|
+
estimatedUsd = inputTokens / 1e3 * inputPer1k;
|
|
8796
|
+
const remaining = maxCost - this.runCostUsd - this.reservedCostUsd;
|
|
8797
|
+
if (estimatedUsd > remaining) {
|
|
8798
|
+
this.failPreflight(
|
|
8799
|
+
`Sending this request to ${model.provider}:${model.id} would cost about $${estimatedUsd.toFixed(4)} in input alone (~${inputTokens.toLocaleString()} tokens), and only $${Math.max(0, remaining).toFixed(4)} of the per-task cap of $${maxCost.toFixed(4)} remains. Shorten the input, choose a cheaper model, or raise budget.maxCostPerRunUsd.`
|
|
8800
|
+
);
|
|
8801
|
+
}
|
|
8802
|
+
}
|
|
8803
|
+
}
|
|
8804
|
+
this.reservedTokens += inputTokens;
|
|
8805
|
+
this.reservedCostUsd += estimatedUsd;
|
|
8806
|
+
const generation = this.runGeneration;
|
|
8807
|
+
let released = false;
|
|
8808
|
+
return () => {
|
|
8809
|
+
if (released || generation !== this.runGeneration) return;
|
|
8810
|
+
released = true;
|
|
8811
|
+
this.reservedTokens -= inputTokens;
|
|
8812
|
+
this.reservedCostUsd -= estimatedUsd;
|
|
8813
|
+
};
|
|
8814
|
+
}
|
|
8815
|
+
/**
|
|
8816
|
+
* What this call will send, in tokens.
|
|
8817
|
+
*
|
|
8818
|
+
* Text is the bulk of it, but not all of it: providers bill the serialized
|
|
8819
|
+
* TOOL DEFINITIONS on every native-tool call (Anthropic sends each name,
|
|
8820
|
+
* description and input schema), and an image is billed by its own size
|
|
8821
|
+
* rather than as the `[image]` placeholder `contentToText` reduces it to. A
|
|
8822
|
+
* tool-heavy or vision request could therefore pass either cap while its real
|
|
8823
|
+
* input already exceeded the allowance.
|
|
8824
|
+
*/
|
|
8825
|
+
estimateInputTokens(options, model) {
|
|
8826
|
+
let tokens = guardTokens(options.systemPrompt ?? "");
|
|
8827
|
+
let images = 0;
|
|
8828
|
+
const wire = wireProfile(model.provider);
|
|
8829
|
+
const countsImage = (img) => wire.sendsUrlImages || img?.type === "base64";
|
|
8830
|
+
for (const message of options.messages) {
|
|
8831
|
+
if (wire.dropsMessage(message)) continue;
|
|
8832
|
+
tokens += TOKENS_PER_MESSAGE_FRAMING;
|
|
8833
|
+
if (typeof message.content === "string") {
|
|
8834
|
+
tokens += guardTokens(message.content);
|
|
8835
|
+
} else {
|
|
8836
|
+
switch (wire.blockHandling(message)) {
|
|
8837
|
+
case "blocks":
|
|
8838
|
+
for (const block of message.content) {
|
|
8839
|
+
if (!wire.sendsBlock(block)) continue;
|
|
8840
|
+
if (block.type === "image") {
|
|
8841
|
+
if (countsImage(block.image)) images++;
|
|
8842
|
+
continue;
|
|
8843
|
+
}
|
|
8844
|
+
tokens += guardTokens(contentToText([block]));
|
|
8845
|
+
}
|
|
8846
|
+
break;
|
|
8847
|
+
case "stringified":
|
|
8848
|
+
tokens += guardTokens(safeJson(message.content));
|
|
8849
|
+
break;
|
|
8850
|
+
}
|
|
8851
|
+
}
|
|
8852
|
+
if (message.toolCalls?.length && wire.sendsToolCalls(message)) {
|
|
8853
|
+
tokens += guardTokens(safeJson(wire.sizeToolCalls(message.toolCalls)));
|
|
8854
|
+
}
|
|
8855
|
+
if (message.toolCallId && message.role === "tool" && wire.sendsToolCallId) {
|
|
8856
|
+
tokens += guardTokens(message.toolCallId);
|
|
8857
|
+
}
|
|
8858
|
+
}
|
|
8859
|
+
if (options.tools?.length) {
|
|
8860
|
+
tokens += guardTokens(safeJson(wire.sizeTools(options.tools)));
|
|
8861
|
+
}
|
|
8862
|
+
if (wire.readsTopLevelImages) {
|
|
8863
|
+
const topLevel = (options.images ?? []).filter(countsImage).length;
|
|
8864
|
+
images += topLevel * geminiImageCopies(options.messages);
|
|
8865
|
+
}
|
|
8866
|
+
tokens += images * IMAGE_TOKENS_EACH;
|
|
8867
|
+
return tokens;
|
|
8868
|
+
}
|
|
8869
|
+
/**
|
|
8870
|
+
* Report a wait that ended because something asked it to stop, the same way
|
|
8871
|
+
* the provider call below reports one.
|
|
8872
|
+
*
|
|
8873
|
+
* The queue waits sit OUTSIDE the try/catch that maps aborts, so without
|
|
8874
|
+
* this a cancellation surfaced as whatever the queue happened to throw and
|
|
8875
|
+
* was retried or failed over like an ordinary error. A budget abort keeps
|
|
8876
|
+
* saying it ran out of budget — otherwise the reason the run stopped is lost
|
|
8877
|
+
* on the way up — and anything else that was aborted becomes a cancellation.
|
|
8878
|
+
*/
|
|
8879
|
+
asAbortFailure(err, callSignal, runSignal) {
|
|
8880
|
+
if (err instanceof _CascadeRouter.BudgetExceededError) return err;
|
|
8881
|
+
if (err instanceof CascadeCancelledError) return err;
|
|
8882
|
+
if (callSignal?.aborted || runSignal?.aborted) return new CascadeCancelledError("Run cancelled");
|
|
8883
|
+
return err;
|
|
8884
|
+
}
|
|
8885
|
+
/** Trips the same run-level flag a post-hoc overrun does, and reports it the same way. */
|
|
8886
|
+
failPreflight(reason) {
|
|
8887
|
+
this.runBudgetExceeded = true;
|
|
8888
|
+
this.runBudgetExceededReason = reason;
|
|
8889
|
+
this.runAbort.abort(new _CascadeRouter.BudgetExceededError(reason));
|
|
8890
|
+
this.emit("budget:exceeded", { reason, spentUsd: this.sessionCostUsd });
|
|
8891
|
+
throw new _CascadeRouter.BudgetExceededError(reason);
|
|
8892
|
+
}
|
|
8285
8893
|
/**
|
|
8286
8894
|
* Enforce the hard per-task ceiling. Once tripped, the flag makes every
|
|
8287
8895
|
* subsequent (and concurrent) generate() call in this run fail fast.
|
|
@@ -8297,6 +8905,7 @@ var CascadeRouter = class _CascadeRouter extends EventEmitter__default.default {
|
|
|
8297
8905
|
const reason = overTokens ? `Per-task token cap of ${maxTokens.toLocaleString()} reached (used ${this.runTokens.toLocaleString()}). Stopping this run to avoid runaway cost \u2014 raise budget.maxTokensPerRun for larger jobs.` : `Per-task cost cap of $${maxCost.toFixed(4)} reached (spent $${this.runCostUsd.toFixed(4)}). Stopping this run to avoid runaway cost.`;
|
|
8298
8906
|
this.runBudgetExceeded = true;
|
|
8299
8907
|
this.runBudgetExceededReason = reason;
|
|
8908
|
+
this.runAbort.abort(new _CascadeRouter.BudgetExceededError(reason));
|
|
8300
8909
|
this.emit("budget:exceeded", { reason, spentUsd: this.sessionCostUsd });
|
|
8301
8910
|
throw new _CascadeRouter.BudgetExceededError(reason);
|
|
8302
8911
|
}
|
|
@@ -9534,6 +10143,12 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : "") + textToolSuffix,
|
|
|
9534
10143
|
maxTokens: 4096,
|
|
9535
10144
|
...subtaskModel ? { model: subtaskModel } : {},
|
|
9536
10145
|
featureTag: this.assignment?.sectionTitle,
|
|
10146
|
+
// The wave signal, so cancel-and-respawn reaches the request that is
|
|
10147
|
+
// RUNNING rather than only stopping the worker at its next checkpoint.
|
|
10148
|
+
// It is a superset of the run signal the router injects when this is
|
|
10149
|
+
// absent (T2 composes it from that signal plus its own per-wave one),
|
|
10150
|
+
// so wiring it here loses no cancellation and adds the wave's.
|
|
10151
|
+
...this.signal ? { signal: this.signal } : {},
|
|
9537
10152
|
...this.localOnlyMatch ? { forceLocal: true } : {}
|
|
9538
10153
|
};
|
|
9539
10154
|
const result = await this.router.generate(
|
|
@@ -10012,6 +10627,7 @@ ${current}`
|
|
|
10012
10627
|
HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
10013
10628
|
maxTokens: 4096,
|
|
10014
10629
|
featureTag: assignment.sectionTitle,
|
|
10630
|
+
...this.signal ? { signal: this.signal } : {},
|
|
10015
10631
|
...this.localOnlyMatch ? { forceLocal: true } : {}
|
|
10016
10632
|
});
|
|
10017
10633
|
const next = (improved.content ?? "").trim();
|
|
@@ -10073,6 +10689,7 @@ Reply with JSON: { "completeness": "pass"|"fail", "correctness": "pass"|"fail",
|
|
|
10073
10689
|
|
|
10074
10690
|
HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
10075
10691
|
featureTag: assignment.sectionTitle,
|
|
10692
|
+
...this.signal ? { signal: this.signal } : {},
|
|
10076
10693
|
...this.localOnlyMatch ? { forceLocal: true } : {}
|
|
10077
10694
|
});
|
|
10078
10695
|
try {
|
|
@@ -10112,6 +10729,7 @@ ${output.slice(0, 4e3)}`;
|
|
|
10112
10729
|
messages: [{ role: "user", content: prompt }],
|
|
10113
10730
|
maxTokens: 300,
|
|
10114
10731
|
temperature: 0,
|
|
10732
|
+
...this.signal ? { signal: this.signal } : {},
|
|
10115
10733
|
...this.localOnlyMatch ? { forceLocal: true } : {}
|
|
10116
10734
|
});
|
|
10117
10735
|
const match = /\[[\s\S]*\]/.exec(result.content);
|
|
@@ -10223,6 +10841,10 @@ Begin execution now.`;
|
|
|
10223
10841
|
}
|
|
10224
10842
|
};
|
|
10225
10843
|
var PeerBus = class extends EventEmitter__default.default {
|
|
10844
|
+
constructor() {
|
|
10845
|
+
super();
|
|
10846
|
+
this.setMaxListeners(64);
|
|
10847
|
+
}
|
|
10226
10848
|
outputs = /* @__PURE__ */ new Map();
|
|
10227
10849
|
waiters = /* @__PURE__ */ new Map();
|
|
10228
10850
|
members = /* @__PURE__ */ new Set();
|
|
@@ -11471,6 +12093,7 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
|
11471
12093
|
const waveSignal = AbortSignal.any(
|
|
11472
12094
|
[this.signal, this.waveAbortController.signal].filter(Boolean)
|
|
11473
12095
|
);
|
|
12096
|
+
EventEmitter.setMaxListeners(64, waveSignal);
|
|
11474
12097
|
const runOne = async (id) => {
|
|
11475
12098
|
const assignment = sanitizedAssignments.find((a) => a.subtaskId === id);
|
|
11476
12099
|
const worker = workerMap.get(id);
|
|
@@ -18471,115 +19094,6 @@ var GraphSearchTool = class extends BaseTool {
|
|
|
18471
19094
|
}
|
|
18472
19095
|
};
|
|
18473
19096
|
|
|
18474
|
-
// src/core/context/compaction.ts
|
|
18475
|
-
function estimateTokens(text) {
|
|
18476
|
-
if (!text) return 0;
|
|
18477
|
-
return Math.max(1, Math.ceil(text.length / 4));
|
|
18478
|
-
}
|
|
18479
|
-
function contentToText(content) {
|
|
18480
|
-
if (typeof content === "string") return content;
|
|
18481
|
-
return content.map((c) => c.type === "text" ? c.text : c.type === "tool_result" ? c.content : `[${c.type}]`).join(" ");
|
|
18482
|
-
}
|
|
18483
|
-
function messagesTokens(messages) {
|
|
18484
|
-
return messages.reduce((n, m) => n + estimateTokens(contentToText(m.content)), 0);
|
|
18485
|
-
}
|
|
18486
|
-
function atomize(text, maxChars) {
|
|
18487
|
-
const seps = ["\n\n", "\n", ". ", " "];
|
|
18488
|
-
let units = [text];
|
|
18489
|
-
for (const sep of seps) {
|
|
18490
|
-
if (units.every((u) => u.length <= maxChars)) break;
|
|
18491
|
-
const next = [];
|
|
18492
|
-
for (const u of units) {
|
|
18493
|
-
if (u.length <= maxChars) {
|
|
18494
|
-
next.push(u);
|
|
18495
|
-
continue;
|
|
18496
|
-
}
|
|
18497
|
-
const parts = u.split(sep);
|
|
18498
|
-
parts.forEach((p, i) => next.push(i < parts.length - 1 ? p + sep : p));
|
|
18499
|
-
}
|
|
18500
|
-
units = next;
|
|
18501
|
-
}
|
|
18502
|
-
const out = [];
|
|
18503
|
-
for (const u of units) {
|
|
18504
|
-
if (u.length <= maxChars) out.push(u);
|
|
18505
|
-
else for (let i = 0; i < u.length; i += maxChars) out.push(u.slice(i, i + maxChars));
|
|
18506
|
-
}
|
|
18507
|
-
return out;
|
|
18508
|
-
}
|
|
18509
|
-
function chunkText(text, opts) {
|
|
18510
|
-
const maxChars = Math.max(1, Math.floor(opts.maxTokens * 4));
|
|
18511
|
-
const overlap = Math.min(maxChars - 1, Math.max(0, Math.floor(maxChars * (opts.overlapRatio ?? 0.1))));
|
|
18512
|
-
const trimmed = text.trim();
|
|
18513
|
-
if (!trimmed) return [];
|
|
18514
|
-
if (trimmed.length <= maxChars) return [trimmed];
|
|
18515
|
-
const units = atomize(trimmed, maxChars);
|
|
18516
|
-
const chunks = [];
|
|
18517
|
-
let cur = "";
|
|
18518
|
-
for (const u of units) {
|
|
18519
|
-
if (cur && cur.length + u.length > maxChars) {
|
|
18520
|
-
chunks.push(cur.trim());
|
|
18521
|
-
cur = overlap > 0 ? cur.slice(-overlap) : "";
|
|
18522
|
-
}
|
|
18523
|
-
cur += u;
|
|
18524
|
-
}
|
|
18525
|
-
if (cur.trim()) chunks.push(cur.trim());
|
|
18526
|
-
return chunks;
|
|
18527
|
-
}
|
|
18528
|
-
function needsCompaction(totalTokens, contextWindow, reserve = 0.2) {
|
|
18529
|
-
if (!contextWindow || contextWindow <= 0) return false;
|
|
18530
|
-
return totalTokens > contextWindow * (1 - reserve);
|
|
18531
|
-
}
|
|
18532
|
-
async function mapReduceCompact(text, opts) {
|
|
18533
|
-
const capChars = opts.capTokens * 4;
|
|
18534
|
-
let working = text;
|
|
18535
|
-
let truncated = false;
|
|
18536
|
-
if (working.length > capChars) {
|
|
18537
|
-
working = working.slice(0, capChars);
|
|
18538
|
-
truncated = true;
|
|
18539
|
-
}
|
|
18540
|
-
const instruction = opts.instruction ?? "Summarize this content, preserving key facts, names, numbers, and decisions.";
|
|
18541
|
-
let chunks = chunkText(working, { maxTokens: opts.chunkTokens });
|
|
18542
|
-
if (chunks.length <= 1) return { text: working, chunks: chunks.length, calls: 0, truncated };
|
|
18543
|
-
let calls = 0;
|
|
18544
|
-
let parts = await Promise.all(
|
|
18545
|
-
chunks.map((c, i) => {
|
|
18546
|
-
calls++;
|
|
18547
|
-
return opts.summarize(c, `${instruction} (part ${i + 1} of ${chunks.length})`);
|
|
18548
|
-
})
|
|
18549
|
-
);
|
|
18550
|
-
let combined = parts.join("\n\n");
|
|
18551
|
-
const maxPasses = opts.maxReducePasses ?? 3;
|
|
18552
|
-
for (let pass = 0; pass < maxPasses && estimateTokens(combined) > opts.targetTokens; pass++) {
|
|
18553
|
-
const groups = chunkText(combined, { maxTokens: opts.chunkTokens });
|
|
18554
|
-
if (groups.length <= 1) break;
|
|
18555
|
-
parts = await Promise.all(
|
|
18556
|
-
groups.map((g) => {
|
|
18557
|
-
calls++;
|
|
18558
|
-
return opts.summarize(g, "Merge these partial summaries into one cohesive summary; keep all distinct facts.");
|
|
18559
|
-
})
|
|
18560
|
-
);
|
|
18561
|
-
combined = parts.join("\n\n");
|
|
18562
|
-
}
|
|
18563
|
-
const note = truncated ? "\n\n[Note: input exceeded the extended-context cap and was truncated.]" : "";
|
|
18564
|
-
return { text: combined + note, chunks: chunks.length, calls, truncated };
|
|
18565
|
-
}
|
|
18566
|
-
async function rollingSummary(messages, opts) {
|
|
18567
|
-
if (messages.length <= opts.keepRecent) return messages;
|
|
18568
|
-
const older = messages.slice(0, messages.length - opts.keepRecent);
|
|
18569
|
-
const recent = messages.slice(messages.length - opts.keepRecent);
|
|
18570
|
-
const transcript = older.map((m) => `${m.role}: ${contentToText(m.content)}`).join("\n");
|
|
18571
|
-
const summary = await opts.summarize(
|
|
18572
|
-
transcript,
|
|
18573
|
-
"Summarize this earlier conversation. Preserve facts, decisions, names, numbers, and any open threads or todos."
|
|
18574
|
-
);
|
|
18575
|
-
const summaryMessage = {
|
|
18576
|
-
role: "system",
|
|
18577
|
-
content: `Summary of earlier conversation (folded to fit the context window):
|
|
18578
|
-
${summary}`
|
|
18579
|
-
};
|
|
18580
|
-
return [summaryMessage, ...recent];
|
|
18581
|
-
}
|
|
18582
|
-
|
|
18583
19097
|
// src/core/steering/guidance.ts
|
|
18584
19098
|
var GuidanceQueue = class {
|
|
18585
19099
|
entries = [];
|