lynkr 9.10.0 → 9.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/lynkr-init.js +7 -0
- package/config/model-tiers.json +5 -5
- package/package.json +3 -3
- package/public/dashboard.html +29 -0
- package/src/api/router.js +34 -0
- package/src/clients/databricks.js +120 -65
- package/src/clients/gpt-utils.js +8 -1
- package/src/clients/ollama-utils.js +19 -10
- package/src/clients/prompt-cache-injection.js +93 -30
- package/src/config/index.js +79 -0
- package/src/context/compression.js +27 -8
- package/src/context/distill.js +2 -1
- package/src/dashboard/api.js +5 -1
- package/src/memory/distiller.js +350 -0
- package/src/memory/extractor.js +0 -8
- package/src/memory/format.js +17 -2
- package/src/memory/index.js +10 -0
- package/src/memory/retriever.js +7 -2
- package/src/memory/skills-cache.js +180 -0
- package/src/memory/tencentdb-launcher.js +459 -0
- package/src/memory/wiki.js +171 -0
- package/src/orchestrator/index.js +85 -2
- package/src/prompts/system.js +20 -6
- package/src/routing/affinity-store.js +68 -1
- package/src/routing/agentic-detector.js +9 -1
- package/src/routing/cache-economics.js +123 -0
- package/src/routing/cache-switch-cost.js +194 -0
- package/src/routing/calibration.js +8 -1
- package/src/routing/complexity-analyzer.js +7 -1
- package/src/routing/context-compressor.js +170 -0
- package/src/routing/difficulty-classifier.js +115 -13
- package/src/routing/index.js +161 -53
- package/src/routing/model-registry.js +35 -10
- package/src/routing/session-affinity.js +60 -0
- package/src/routing/telemetry.js +139 -2
- package/src/routing/verifier.js +41 -2
- package/src/server.js +36 -3
- package/src/tools/code-graph.js +6 -4
|
@@ -21,6 +21,7 @@ const crypto = require("crypto");
|
|
|
21
21
|
const { getSemanticCache, isSemanticCacheEnabled } = require("../cache/semantic");
|
|
22
22
|
const { areSimilarToolCalls } = require("../clients/gpt-utils");
|
|
23
23
|
const { getModelRegistrySync } = require("../routing/model-registry");
|
|
24
|
+
const sessionAffinity = require("../routing/session-affinity");
|
|
24
25
|
|
|
25
26
|
/**
|
|
26
27
|
* Get destination URL for audit logging based on provider type
|
|
@@ -433,7 +434,7 @@ function injectToolLoopStopInstruction(messages, threshold = 5) {
|
|
|
433
434
|
// Inject instruction to stop tool calls and provide a final answer
|
|
434
435
|
const stopInstruction = {
|
|
435
436
|
role: "user",
|
|
436
|
-
content: `⚠️ IMPORTANT: You have already executed ${toolResultCount} tool calls in this conversation. This is likely an infinite loop. STOP calling tools immediately and provide a direct text response to the user based on the information you have gathered. If you cannot complete the task, explain why. DO NOT call any more tools.`,
|
|
437
|
+
content: `⚠️ IMPORTANT: You have already executed ${toolResultCount} tool calls in this conversation. This is likely an infinite loop. STOP calling tools immediately and provide a direct text response to the user based on the information you have gathered. If you cannot complete the task, explain why. DO NOT call any more tools. Your response must contain your actual findings — NOT an acknowledgment or restatement of this warning.`,
|
|
437
438
|
};
|
|
438
439
|
|
|
439
440
|
return [...messages, stopInstruction];
|
|
@@ -452,6 +453,23 @@ const DEDUP_MAX_SIGNATURES = 50;
|
|
|
452
453
|
const DEDUP_WARN_THRESHOLD = 5;
|
|
453
454
|
const DEDUP_TERMINATE_THRESHOLD = 8;
|
|
454
455
|
|
|
456
|
+
/**
|
|
457
|
+
* State-management tools are called repeatedly with similar args BY DESIGN —
|
|
458
|
+
* that's their job, not a loop. Opencode calls `todowrite` after every step;
|
|
459
|
+
* Claude Code uses `TodoWrite`; OpenWorker uses `todo_write`/`propose_plan`.
|
|
460
|
+
* Counting them in the dedup tracker produced false-positive loop warnings
|
|
461
|
+
* and force-terminations on healthy agent sessions (live incident: an
|
|
462
|
+
* opencode analysis run was force-terminated into a raw tool-result dump
|
|
463
|
+
* because its todo updates hit the terminate threshold).
|
|
464
|
+
* @param {Object} toolCall - tool_use block
|
|
465
|
+
* @returns {boolean}
|
|
466
|
+
*/
|
|
467
|
+
function isDedupExemptTool(toolCall) {
|
|
468
|
+
const raw = toolCall?.function?.name ?? toolCall?.name ?? '';
|
|
469
|
+
const name = String(raw).toLowerCase().replace(/[_-]/g, '');
|
|
470
|
+
return name.includes('todo') || name === 'updateplan' || name === 'proposeplan';
|
|
471
|
+
}
|
|
472
|
+
|
|
455
473
|
/**
|
|
456
474
|
* Initialise session.metadata.toolCallDedup if missing.
|
|
457
475
|
* @param {Object} session
|
|
@@ -462,6 +480,7 @@ function ensureDedupStructure(session) {
|
|
|
462
480
|
session.metadata.toolCallDedup = {
|
|
463
481
|
signatures: {},
|
|
464
482
|
similarGroups: {},
|
|
483
|
+
seenIds: {},
|
|
465
484
|
lastResetAt: Date.now(),
|
|
466
485
|
warningInjected: false,
|
|
467
486
|
};
|
|
@@ -476,9 +495,26 @@ function ensureDedupStructure(session) {
|
|
|
476
495
|
*/
|
|
477
496
|
function recordCrossRequestToolCall(session, toolCall) {
|
|
478
497
|
if (!session?.metadata) return;
|
|
498
|
+
if (isDedupExemptTool(toolCall)) return; // repeat-by-design tools never count toward loop detection
|
|
479
499
|
ensureDedupStructure(session);
|
|
480
500
|
|
|
481
501
|
const dedup = session.metadata.toolCallDedup;
|
|
502
|
+
|
|
503
|
+
// Stateless HTTP agents (opencode, Cursor, OpenWorker) replay the FULL
|
|
504
|
+
// conversation history on every agent-loop step, so the same tool_use block
|
|
505
|
+
// arrives again on step 2, 3, 4… Counting each replay made counts grow
|
|
506
|
+
// quadratically with turn length — any exploration past ~5 steps hit the
|
|
507
|
+
// terminate threshold regardless of what it was doing (live incident:
|
|
508
|
+
// a 7-step opencode code-trace was force-terminated as a "loop").
|
|
509
|
+
// Each tool_use block carries a unique id: count each id exactly once.
|
|
510
|
+
const blockId = toolCall?.id;
|
|
511
|
+
if (blockId) {
|
|
512
|
+
if (!dedup.seenIds) dedup.seenIds = {};
|
|
513
|
+
if (dedup.seenIds[blockId]) return; // already counted this exact call
|
|
514
|
+
dedup.seenIds[blockId] = 1;
|
|
515
|
+
// Bounded: reset wipes this on each new user question; cap as a backstop.
|
|
516
|
+
if (Object.keys(dedup.seenIds).length > 500) dedup.seenIds = { [blockId]: 1 };
|
|
517
|
+
}
|
|
482
518
|
const signature = getToolCallSignature(toolCall);
|
|
483
519
|
const toolName = toolCall.function?.name ?? toolCall.name ?? 'unknown';
|
|
484
520
|
const args = toolCall.function?.arguments ?? toolCall.input;
|
|
@@ -620,6 +656,7 @@ function resetDedupTracking(session) {
|
|
|
620
656
|
session.metadata.toolCallDedup = {
|
|
621
657
|
signatures: {},
|
|
622
658
|
similarGroups: {},
|
|
659
|
+
seenIds: {},
|
|
623
660
|
lastResetAt: Date.now(),
|
|
624
661
|
warningInjected: false,
|
|
625
662
|
};
|
|
@@ -1421,6 +1458,30 @@ async function runAgentLoop({
|
|
|
1421
1458
|
|
|
1422
1459
|
|
|
1423
1460
|
if (steps === 1 && agentTimer) agentTimer.mark("preCompression");
|
|
1461
|
+
|
|
1462
|
+
// === CONVERSATION DISTILLATION (TencentDB-inspired L0-L3 pipeline) ===
|
|
1463
|
+
// Long conversations collapse older turns into one distilled block
|
|
1464
|
+
// (persona + scenario summary) before history compression runs.
|
|
1465
|
+
if (steps === 1 && config.memory?.enabled !== false && config.memory?.distillation?.enabled !== false) {
|
|
1466
|
+
try {
|
|
1467
|
+
const distiller = require('../memory/distiller');
|
|
1468
|
+
if (distiller.needsDistillation(cleanPayload.messages)) {
|
|
1469
|
+
const result = distiller.distillMessages(cleanPayload.messages, {
|
|
1470
|
+
sessionId: session?.id,
|
|
1471
|
+
});
|
|
1472
|
+
if (result.applied) {
|
|
1473
|
+
cleanPayload.messages = result.messages;
|
|
1474
|
+
logger.debug({
|
|
1475
|
+
sessionId: session?.id ?? null,
|
|
1476
|
+
...result.stats,
|
|
1477
|
+
}, '[distiller] Conversation distillation applied');
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
} catch (err) {
|
|
1481
|
+
logger.warn({ err, sessionId: session?.id }, 'Distillation failed, continuing with full history');
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1424
1485
|
if (steps === 1 && config.historyCompression?.enabled !== false) {
|
|
1425
1486
|
try {
|
|
1426
1487
|
if (historyCompression.needsCompression(cleanPayload.messages)) {
|
|
@@ -1883,6 +1944,28 @@ IMPORTANT TOOL USAGE RULES:
|
|
|
1883
1944
|
}
|
|
1884
1945
|
}
|
|
1885
1946
|
|
|
1947
|
+
// Cache-aware routing (Phase 1): persist the session's warm-prefix state
|
|
1948
|
+
// from the response's cache counters so the router can price a mid-session
|
|
1949
|
+
// model switch against the live cache clock. Best-effort — never blocks
|
|
1950
|
+
// the response path.
|
|
1951
|
+
if (session?.id && actualUsage) {
|
|
1952
|
+
try {
|
|
1953
|
+
// Prefer the ROUTED provider/model over the request-level default:
|
|
1954
|
+
// the tier router inside invokeModel may have diverged from
|
|
1955
|
+
// providerType, and the cache lives with whoever actually served
|
|
1956
|
+
// (verified live: databricks default label on ollama-served turns).
|
|
1957
|
+
const served = databricksResponse.routingDecision || {};
|
|
1958
|
+
sessionAffinity.recordCacheUsage(session.id, {
|
|
1959
|
+
provider: served.provider || providerType,
|
|
1960
|
+
model: served.model || cleanPayload.model,
|
|
1961
|
+
cacheReadTokens: actualUsage.cacheReadTokens,
|
|
1962
|
+
cacheCreationTokens: actualUsage.cacheCreationTokens,
|
|
1963
|
+
});
|
|
1964
|
+
} catch (err) {
|
|
1965
|
+
logger.debug({ err: err.message }, "[Orchestrator] cache-state update failed");
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1886
1969
|
if (auditLogger.enabled) {
|
|
1887
1970
|
const latencyMs = Date.now() - start;
|
|
1888
1971
|
|
|
@@ -2879,7 +2962,7 @@ async function processMessage({ payload, headers, session, cwd, options = {} })
|
|
|
2879
2962
|
if (Array.isArray(payload?.messages)) {
|
|
2880
2963
|
payload.messages.push({
|
|
2881
2964
|
role: "user",
|
|
2882
|
-
content: `⚠️ CRITICAL SYSTEM WARNING: You have called the "${dedupToolName}" tool ${maxCount} times with identical or similar parameters across multiple requests. This IS an infinite loop. STOP calling this tool immediately. You MUST now provide a direct text response based on the results you have received. If the tool returned "no results" or empty output, that IS the final answer - do not retry.
|
|
2965
|
+
content: `⚠️ CRITICAL SYSTEM WARNING: You have called the "${dedupToolName}" tool ${maxCount} times with identical or similar parameters across multiple requests. This IS an infinite loop. STOP calling this tool immediately. You MUST now provide a direct text response based on the results you have received. If the tool returned "no results" or empty output, that IS the final answer - do not retry. Your response must contain your actual findings from the tool results gathered so far — NOT an acknowledgment or restatement of this warning. Do not mention this warning in your response.`,
|
|
2883
2966
|
});
|
|
2884
2967
|
}
|
|
2885
2968
|
}
|
package/src/prompts/system.js
CHANGED
|
@@ -70,13 +70,27 @@ function compressToolDescriptions(tools, mode = null) {
|
|
|
70
70
|
return tools; // Return unmodified if not in minimal mode
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
// Normalize to Anthropic shape: OpenAI-format callers (OpenWorker, aisuite,
|
|
74
|
+
// any OpenAI SDK) send {type:"function", function:{name, description, parameters}}.
|
|
75
|
+
// Unwrap once here so the compression logic below can assume input_schema exists.
|
|
76
|
+
tools = tools.map(tool =>
|
|
77
|
+
tool?.type === "function" && tool.function
|
|
78
|
+
? {
|
|
79
|
+
name: tool.function.name,
|
|
80
|
+
description: tool.function.description,
|
|
81
|
+
input_schema: tool.function.parameters || { type: "object", properties: {} },
|
|
82
|
+
}
|
|
83
|
+
: tool
|
|
84
|
+
);
|
|
85
|
+
|
|
73
86
|
return tools.map(tool => {
|
|
87
|
+
const input_schema = tool.input_schema || { type: "object", properties: {} };
|
|
74
88
|
const compressed = {
|
|
75
89
|
name: tool.name,
|
|
76
90
|
input_schema: {
|
|
77
|
-
type:
|
|
91
|
+
type: input_schema.type || "object",
|
|
78
92
|
properties: {},
|
|
79
|
-
required:
|
|
93
|
+
required: input_schema.required || [],
|
|
80
94
|
}
|
|
81
95
|
};
|
|
82
96
|
|
|
@@ -86,8 +100,8 @@ function compressToolDescriptions(tools, mode = null) {
|
|
|
86
100
|
}
|
|
87
101
|
|
|
88
102
|
// Compress property descriptions
|
|
89
|
-
if (
|
|
90
|
-
for (const [key, value] of Object.entries(
|
|
103
|
+
if (input_schema.properties) {
|
|
104
|
+
for (const [key, value] of Object.entries(input_schema.properties)) {
|
|
91
105
|
compressed.input_schema.properties[key] = {
|
|
92
106
|
type: value.type,
|
|
93
107
|
};
|
|
@@ -108,8 +122,8 @@ function compressToolDescriptions(tools, mode = null) {
|
|
|
108
122
|
}
|
|
109
123
|
|
|
110
124
|
// Preserve additionalProperties if set
|
|
111
|
-
if (
|
|
112
|
-
compressed.input_schema.additionalProperties =
|
|
125
|
+
if (input_schema.additionalProperties !== undefined) {
|
|
126
|
+
compressed.input_schema.additionalProperties = input_schema.additionalProperties;
|
|
113
127
|
}
|
|
114
128
|
|
|
115
129
|
return compressed;
|
|
@@ -56,6 +56,12 @@ function _db() {
|
|
|
56
56
|
if (!cols.has("has_tool_history")) {
|
|
57
57
|
db.exec("ALTER TABLE session_pins ADD COLUMN has_tool_history INTEGER DEFAULT 0");
|
|
58
58
|
}
|
|
59
|
+
// Additive migration for cache-aware routing (Phase 1): JSON blob
|
|
60
|
+
// holding {warmPrefixTokens, provider, model, lastRequestAt, ttlMs},
|
|
61
|
+
// updated after every upstream response that reports cache usage.
|
|
62
|
+
if (!cols.has("cache_state")) {
|
|
63
|
+
db.exec("ALTER TABLE session_pins ADD COLUMN cache_state TEXT");
|
|
64
|
+
}
|
|
59
65
|
schemaEnsured = true;
|
|
60
66
|
} catch (err) {
|
|
61
67
|
degradation.record("feedback", err);
|
|
@@ -160,6 +166,67 @@ function save(sessionId, pin) {
|
|
|
160
166
|
}
|
|
161
167
|
}
|
|
162
168
|
|
|
169
|
+
/**
|
|
170
|
+
* Persist per-session prompt-cache state (Phase 1, cache-aware routing).
|
|
171
|
+
* Piggybacks on the session_pins row; creates a minimal row when the session
|
|
172
|
+
* has no pin yet (possible when sticky sessions are disabled but tracking
|
|
173
|
+
* is on). Best-effort like everything else in this module.
|
|
174
|
+
*
|
|
175
|
+
* @param {string} sessionId
|
|
176
|
+
* @param {{warmPrefixTokens:number, provider:string, model:string|null, lastRequestAt:number, ttlMs:number}} state
|
|
177
|
+
*/
|
|
178
|
+
function saveCacheState(sessionId, state) {
|
|
179
|
+
if (!sessionId || !state?.provider) return;
|
|
180
|
+
const db = _db();
|
|
181
|
+
if (!db) return;
|
|
182
|
+
try {
|
|
183
|
+
const json = JSON.stringify(state);
|
|
184
|
+
const res = _stmt(
|
|
185
|
+
db,
|
|
186
|
+
"cache_state_update",
|
|
187
|
+
"UPDATE session_pins SET cache_state = ? WHERE session_id = ?"
|
|
188
|
+
).run(json, sessionId);
|
|
189
|
+
if (res.changes === 0) {
|
|
190
|
+
_stmt(
|
|
191
|
+
db,
|
|
192
|
+
"cache_state_insert",
|
|
193
|
+
`INSERT INTO session_pins (session_id, provider, model, ts, cache_state)
|
|
194
|
+
VALUES (?, ?, ?, ?, ?)
|
|
195
|
+
ON CONFLICT(session_id) DO UPDATE SET cache_state = excluded.cache_state`
|
|
196
|
+
).run(sessionId, state.provider, state.model ?? null, Date.now(), json);
|
|
197
|
+
}
|
|
198
|
+
} catch (err) {
|
|
199
|
+
degradation.record("feedback", err);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Load per-session cache state. Returns null when absent, unparsable, or the
|
|
205
|
+
* DB is unavailable — callers treat null as "no cache signal for this
|
|
206
|
+
* session" (provider doesn't report cache usage, or no response seen yet).
|
|
207
|
+
*
|
|
208
|
+
* @param {string} sessionId
|
|
209
|
+
* @returns {{warmPrefixTokens:number, provider:string, model:string|null, lastRequestAt:number, ttlMs:number}|null}
|
|
210
|
+
*/
|
|
211
|
+
function loadCacheState(sessionId) {
|
|
212
|
+
if (!sessionId) return null;
|
|
213
|
+
const db = _db();
|
|
214
|
+
if (!db) return null;
|
|
215
|
+
try {
|
|
216
|
+
const row = _stmt(
|
|
217
|
+
db,
|
|
218
|
+
"cache_state_load",
|
|
219
|
+
"SELECT cache_state FROM session_pins WHERE session_id = ?"
|
|
220
|
+
).get(sessionId);
|
|
221
|
+
if (!row?.cache_state) return null;
|
|
222
|
+
const parsed = JSON.parse(row.cache_state);
|
|
223
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
224
|
+
} catch (err) {
|
|
225
|
+
degradation.record("feedback", err);
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
163
230
|
/**
|
|
164
231
|
* Remove a pin.
|
|
165
232
|
* @param {string} sessionId
|
|
@@ -205,4 +272,4 @@ function _clear() {
|
|
|
205
272
|
} catch { /* best-effort */ }
|
|
206
273
|
}
|
|
207
274
|
|
|
208
|
-
module.exports = { load, save, remove, cleanup, _clear };
|
|
275
|
+
module.exports = { load, save, remove, cleanup, saveCacheState, loadCacheState, _clear };
|
|
@@ -224,13 +224,21 @@ class AgenticDetector {
|
|
|
224
224
|
const agentType = this._classifyAgentType(score, signals);
|
|
225
225
|
const isAgentic = score >= 25;
|
|
226
226
|
|
|
227
|
+
// AGENTIC_DISABLE_FLOOR=1 bypasses the per-agent-type minTier — the raw
|
|
228
|
+
// complexity score decides the tier without the tools-based bump. Useful
|
|
229
|
+
// for demos/tests where tool-heavy clients (OpenWorker, aisuite) send 25+
|
|
230
|
+
// tools on every request and otherwise floor everything to MEDIUM+.
|
|
231
|
+
const flooredMinTier = process.env.AGENTIC_DISABLE_FLOOR === '1'
|
|
232
|
+
? 'SIMPLE'
|
|
233
|
+
: AGENT_TYPES[agentType].minTier;
|
|
234
|
+
|
|
227
235
|
const result = {
|
|
228
236
|
isAgentic,
|
|
229
237
|
agentType,
|
|
230
238
|
confidence: Math.min(score / 100, 1),
|
|
231
239
|
score,
|
|
232
240
|
signals,
|
|
233
|
-
minTier:
|
|
241
|
+
minTier: flooredMinTier,
|
|
234
242
|
scoreBoost: AGENT_TYPES[agentType].scoreBoost,
|
|
235
243
|
description: AGENT_TYPES[agentType].description,
|
|
236
244
|
clientProfile: profile?.name ?? null,
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider prompt-cache economics.
|
|
3
|
+
*
|
|
4
|
+
* Single resolution point for "what does the prompt cache cost on this
|
|
5
|
+
* provider/model": read price, write price, TTL, and mechanism. The
|
|
6
|
+
* switch-cost math in `cache-switch-cost.js` and the cache-state tracker in
|
|
7
|
+
* `session-affinity.js` read from here — decision code never hardcodes
|
|
8
|
+
* per-model numbers.
|
|
9
|
+
*
|
|
10
|
+
* Resolution order per field:
|
|
11
|
+
* 1. model-registry entry (models.dev carries absolute cacheRead/cacheWrite
|
|
12
|
+
* $/1M for many models)
|
|
13
|
+
* 2. provider-type fallback table below (multipliers of the model's input
|
|
14
|
+
* price), researched Aug 2026.
|
|
15
|
+
*
|
|
16
|
+
* Mechanisms:
|
|
17
|
+
* - 'explicit' — cache_control breakpoints, paid writes (Anthropic-style).
|
|
18
|
+
* TTL refreshes on every read, so lastRequestAt + ttlMs is a live clock.
|
|
19
|
+
* - 'automatic' — provider caches transparently, writes are free
|
|
20
|
+
* (OpenAI/DeepSeek/Gemini-style). TTL is best-effort.
|
|
21
|
+
* - 'local' — no dollar cost at all; the "cost" of a cold prefix is
|
|
22
|
+
* prefill latency, handled separately in cache-switch-cost.js.
|
|
23
|
+
*
|
|
24
|
+
* @module routing/cache-economics
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const logger = require('../logger');
|
|
28
|
+
|
|
29
|
+
// Fallback economics keyed by Lynkr provider type. `readMult`/`writeMult`
|
|
30
|
+
// are multipliers of the model's per-1M input price.
|
|
31
|
+
const PROVIDER_CACHE_DEFAULTS = {
|
|
32
|
+
// Anthropic-hosted (explicit cache_control, 1.25x write for the 5-min TTL,
|
|
33
|
+
// TTL refreshed on every read).
|
|
34
|
+
'azure-anthropic': { readMult: 0.1, writeMult: 1.25, ttlMs: 5 * 60 * 1000, mechanism: 'explicit' },
|
|
35
|
+
bedrock: { readMult: 0.1, writeMult: 1.25, ttlMs: 5 * 60 * 1000, mechanism: 'explicit' },
|
|
36
|
+
databricks: { readMult: 0.1, writeMult: 1.25, ttlMs: 5 * 60 * 1000, mechanism: 'explicit' },
|
|
37
|
+
|
|
38
|
+
// OpenAI automatic prefix caching: writes free, ~5-10 min idle eviction.
|
|
39
|
+
openai: { readMult: 0.1, writeMult: 0, ttlMs: 10 * 60 * 1000, mechanism: 'automatic' },
|
|
40
|
+
'azure-openai': { readMult: 0.1, writeMult: 0, ttlMs: 10 * 60 * 1000, mechanism: 'automatic' },
|
|
41
|
+
|
|
42
|
+
// Gemini implicit caching / GLM / Kimi / aggregators: free automatic
|
|
43
|
+
// caching, best-effort TTL. Aggregators (openrouter/edenai) depend on the
|
|
44
|
+
// underlying model; models.dev per-model data wins when present.
|
|
45
|
+
vertex: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
46
|
+
zai: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
47
|
+
moonshot: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
48
|
+
openrouter: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
49
|
+
edenai: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
50
|
+
// DeepSeek direct (via aggregators today, kept for model-level matches):
|
|
51
|
+
deepseek: { readMult: 0.02, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
52
|
+
|
|
53
|
+
// Local runtimes: no dollars; KV cache lives as long as the model stays
|
|
54
|
+
// loaded (ollama keep_alive default ~5 min).
|
|
55
|
+
ollama: { readMult: 0, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'local' },
|
|
56
|
+
llamacpp: { readMult: 0, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'local' },
|
|
57
|
+
lmstudio: { readMult: 0, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'local' },
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const GENERIC_DEFAULT = { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' };
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Fallback cache economics for a provider type (no model lookup).
|
|
64
|
+
* @param {string|null} providerType
|
|
65
|
+
* @returns {{readMult:number, writeMult:number, ttlMs:number, mechanism:string}}
|
|
66
|
+
*/
|
|
67
|
+
function getProviderCacheDefaults(providerType) {
|
|
68
|
+
return PROVIDER_CACHE_DEFAULTS[providerType] || GENERIC_DEFAULT;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Resolve full cache economics for a provider/model pair.
|
|
73
|
+
*
|
|
74
|
+
* @param {string|null} providerType - Lynkr provider type (e.g. 'databricks')
|
|
75
|
+
* @param {string|null} model - model name for registry lookup
|
|
76
|
+
* @returns {{
|
|
77
|
+
* inputPerM:number, outputPerM:number,
|
|
78
|
+
* cacheReadPerM:number, cacheWritePerM:number,
|
|
79
|
+
* ttlMs:number, mechanism:string, unknownPricing:boolean
|
|
80
|
+
* }} All prices are USD per 1M tokens.
|
|
81
|
+
*/
|
|
82
|
+
function resolveCacheEconomics(providerType, model) {
|
|
83
|
+
const defaults = getProviderCacheDefaults(providerType);
|
|
84
|
+
|
|
85
|
+
let cost = null;
|
|
86
|
+
try {
|
|
87
|
+
const { getModelRegistrySync } = require('./model-registry');
|
|
88
|
+
cost = model ? getModelRegistrySync().getCost(model) : null;
|
|
89
|
+
} catch (err) {
|
|
90
|
+
logger.debug({ err: err.message }, '[CacheEconomics] registry lookup failed');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const inputPerM = typeof cost?.input === 'number' ? cost.input : 0;
|
|
94
|
+
const outputPerM = typeof cost?.output === 'number' ? cost.output : 0;
|
|
95
|
+
|
|
96
|
+
// Registry cacheTtlMs/cacheMechanism (Phase 4 entries) win over the
|
|
97
|
+
// provider fallback; models.dev absolute cache prices win over multipliers.
|
|
98
|
+
const mechanism = cost?.cacheMechanism || defaults.mechanism;
|
|
99
|
+
const ttlMs = typeof cost?.cacheTtlMs === 'number' ? cost.cacheTtlMs : defaults.ttlMs;
|
|
100
|
+
|
|
101
|
+
const cacheReadPerM = mechanism === 'local'
|
|
102
|
+
? 0
|
|
103
|
+
: (typeof cost?.cacheRead === 'number' ? cost.cacheRead : inputPerM * defaults.readMult);
|
|
104
|
+
const cacheWritePerM = mechanism === 'local'
|
|
105
|
+
? 0
|
|
106
|
+
: (typeof cost?.cacheWrite === 'number' ? cost.cacheWrite : inputPerM * defaults.writeMult);
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
inputPerM,
|
|
110
|
+
outputPerM,
|
|
111
|
+
cacheReadPerM,
|
|
112
|
+
cacheWritePerM,
|
|
113
|
+
ttlMs,
|
|
114
|
+
mechanism,
|
|
115
|
+
unknownPricing: !!cost?.unknown,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
module.exports = {
|
|
120
|
+
PROVIDER_CACHE_DEFAULTS,
|
|
121
|
+
getProviderCacheDefaults,
|
|
122
|
+
resolveCacheEconomics,
|
|
123
|
+
};
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Switch-cost break-even model (Phase 3, cache-aware routing).
|
|
3
|
+
*
|
|
4
|
+
* Answers one question for a session that holds a warm prompt-cache prefix
|
|
5
|
+
* on its pinned model: does switching to a cheaper target model pay for the
|
|
6
|
+
* cache it breaks, within the turns this session is expected to have left?
|
|
7
|
+
*
|
|
8
|
+
* stayPerTurn = warm*cacheRead(cur) + new*input(cur) + out*output(cur)
|
|
9
|
+
* switchOnce = (warm + new) * cacheWrite(target) // one-time
|
|
10
|
+
* switchPerTurn = warm*cacheRead(tgt) + new*input(tgt) + out*output(tgt)
|
|
11
|
+
* breakEvenTurns = switchOnce / (stayPerTurn - switchPerTurn)
|
|
12
|
+
*
|
|
13
|
+
* Switch iff breakEvenTurns <= expectedRemainingTurns.
|
|
14
|
+
*
|
|
15
|
+
* The output-token term is deliberately included: for de-escalation the
|
|
16
|
+
* dominant per-turn saving is often the output-price spread (e.g. Opus $25/M
|
|
17
|
+
* vs Haiku $5/M out), and omitting it makes the router hold expensive pins
|
|
18
|
+
* long past the point the economics justify.
|
|
19
|
+
*
|
|
20
|
+
* Scope: this module prices COST-MOTIVATED switches (de-escalator, bandit
|
|
21
|
+
* exploration, economic downgrades). Hard triggers — risk keywords, force
|
|
22
|
+
* phrases, context overflow, vision, upward drift — are never gated here;
|
|
23
|
+
* correctness beats cost and the math can never favor a pricier model anyway
|
|
24
|
+
* (savings would be negative).
|
|
25
|
+
*
|
|
26
|
+
* Local models have no dollar cost; the price of a cold prefix is prefill
|
|
27
|
+
* latency, so a warm local prefix above a size threshold holds the pin.
|
|
28
|
+
*
|
|
29
|
+
* All pricing comes from cache-economics.js (registry-first, provider
|
|
30
|
+
* fallback) — no per-model numbers live in this file.
|
|
31
|
+
*
|
|
32
|
+
* @module routing/cache-switch-cost
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
const config = require('../config');
|
|
36
|
+
const logger = require('../logger');
|
|
37
|
+
|
|
38
|
+
const PER_TOKEN = 1 / 1_000_000; // registry prices are USD per 1M tokens
|
|
39
|
+
|
|
40
|
+
function _cfg() {
|
|
41
|
+
const c = config.routing?.cacheAware || {};
|
|
42
|
+
return {
|
|
43
|
+
enabled: c.enabled !== false,
|
|
44
|
+
defaultRemainingTurns: c.defaultRemainingTurns ?? 10,
|
|
45
|
+
newTokensPerTurn: c.newTokensPerTurn ?? 2000,
|
|
46
|
+
outputTokensPerTurn: c.outputTokensPerTurn ?? 800,
|
|
47
|
+
localMaxSwitchPrefixTokens: c.localMaxSwitchPrefixTokens ?? 16000,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Evaluate whether a cost-motivated switch away from the session's warm
|
|
53
|
+
* prefix should be allowed.
|
|
54
|
+
*
|
|
55
|
+
* @param {Object} args
|
|
56
|
+
* @param {Object|null} args.cacheState - from sessionAffinity.getCacheState:
|
|
57
|
+
* {warmPrefixTokens, provider, model, lastRequestAt, ttlMs, cold} or null.
|
|
58
|
+
* @param {{provider:string, model:string|null}} args.current - pinned target.
|
|
59
|
+
* @param {{provider:string, model:string|null}} args.target - proposed target.
|
|
60
|
+
* @param {number} [args.newTokensPerTurn] - est. fresh input tokens per turn.
|
|
61
|
+
* @param {number} [args.outputTokensPerTurn] - est. output tokens per turn.
|
|
62
|
+
* @param {number|null} [args.expectedRemainingTurns] - median remaining
|
|
63
|
+
* turns; falls back to the conservative config default when null.
|
|
64
|
+
* @param {Object} [args.deps] - test injection: {resolveCacheEconomics}.
|
|
65
|
+
* @returns {{
|
|
66
|
+
* switchAllowed: boolean,
|
|
67
|
+
* reason: string,
|
|
68
|
+
* breakEvenTurns: number|null,
|
|
69
|
+
* expectedRemainingTurns: number,
|
|
70
|
+
* warmPrefixTokens: number,
|
|
71
|
+
* switchOnceUsd: number|null,
|
|
72
|
+
* stayPerTurnUsd: number|null,
|
|
73
|
+
* switchPerTurnUsd: number|null,
|
|
74
|
+
* projectedStaySavingsUsd: number|null,
|
|
75
|
+
* }}
|
|
76
|
+
*/
|
|
77
|
+
function evaluateSwitch({
|
|
78
|
+
cacheState,
|
|
79
|
+
current,
|
|
80
|
+
target,
|
|
81
|
+
newTokensPerTurn,
|
|
82
|
+
outputTokensPerTurn,
|
|
83
|
+
expectedRemainingTurns,
|
|
84
|
+
deps = {},
|
|
85
|
+
} = {}) {
|
|
86
|
+
const cfg = _cfg();
|
|
87
|
+
const remaining = Number.isFinite(expectedRemainingTurns) && expectedRemainingTurns > 0
|
|
88
|
+
? expectedRemainingTurns
|
|
89
|
+
: cfg.defaultRemainingTurns;
|
|
90
|
+
|
|
91
|
+
const base = {
|
|
92
|
+
switchAllowed: true,
|
|
93
|
+
reason: 'no_cache_state',
|
|
94
|
+
breakEvenTurns: null,
|
|
95
|
+
expectedRemainingTurns: remaining,
|
|
96
|
+
warmPrefixTokens: cacheState?.warmPrefixTokens ?? 0,
|
|
97
|
+
switchOnceUsd: null,
|
|
98
|
+
stayPerTurnUsd: null,
|
|
99
|
+
switchPerTurnUsd: null,
|
|
100
|
+
projectedStaySavingsUsd: null,
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
if (!cfg.enabled) return { ...base, reason: 'feature_disabled' };
|
|
104
|
+
if (!current?.provider || !target?.provider) return base;
|
|
105
|
+
if (current.provider === target.provider && current.model === target.model) {
|
|
106
|
+
return { ...base, reason: 'same_model' };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// No tracked state → nothing warm to protect (provider reports no cache
|
|
110
|
+
// usage, or no response seen yet). Switching is cache-free.
|
|
111
|
+
if (!cacheState || !(cacheState.warmPrefixTokens > 0)) return base;
|
|
112
|
+
|
|
113
|
+
// State recorded for a different model than the pin we're defending —
|
|
114
|
+
// stale after a prior switch; treat as no protection.
|
|
115
|
+
if (cacheState.model && current.model && cacheState.model !== current.model) {
|
|
116
|
+
return { ...base, reason: 'stale_cache_state' };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Phase 2 — TTL clock. Anthropic-style caches refresh on every read, so
|
|
120
|
+
// lastRequestAt + ttlMs going stale means the prefix is already cold and
|
|
121
|
+
// the switch costs nothing extra.
|
|
122
|
+
if (cacheState.cold) return { ...base, reason: 'cache_cold' };
|
|
123
|
+
|
|
124
|
+
let resolve = deps.resolveCacheEconomics;
|
|
125
|
+
if (typeof resolve !== 'function') {
|
|
126
|
+
({ resolveCacheEconomics: resolve } = require('./cache-economics'));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const econCur = resolve(current.provider, current.model);
|
|
130
|
+
const econTgt = resolve(target.provider, target.model);
|
|
131
|
+
|
|
132
|
+
const W = cacheState.warmPrefixTokens;
|
|
133
|
+
const N = Number.isFinite(newTokensPerTurn) && newTokensPerTurn >= 0
|
|
134
|
+
? newTokensPerTurn : cfg.newTokensPerTurn;
|
|
135
|
+
const O = Number.isFinite(outputTokensPerTurn) && outputTokensPerTurn >= 0
|
|
136
|
+
? outputTokensPerTurn : cfg.outputTokensPerTurn;
|
|
137
|
+
|
|
138
|
+
// Local pin: no dollars in either column — the cost of abandoning the
|
|
139
|
+
// warm prefix is prefill latency on whatever serves the next turn.
|
|
140
|
+
// Hold while the prefix is large; small prefixes re-fill fast enough.
|
|
141
|
+
if (econCur.mechanism === 'local' && econTgt.mechanism === 'local') {
|
|
142
|
+
const allowed = W <= cfg.localMaxSwitchPrefixTokens;
|
|
143
|
+
return {
|
|
144
|
+
...base,
|
|
145
|
+
switchAllowed: allowed,
|
|
146
|
+
reason: allowed ? 'local_prefix_small' : 'local_prefill_hold',
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const stayPerTurn = (W * econCur.cacheReadPerM + N * econCur.inputPerM + O * econCur.outputPerM) * PER_TOKEN;
|
|
151
|
+
const switchOnce = (W + N) * econTgt.cacheWritePerM * PER_TOKEN;
|
|
152
|
+
const switchPerTurn = (W * econTgt.cacheReadPerM + N * econTgt.inputPerM + O * econTgt.outputPerM) * PER_TOKEN;
|
|
153
|
+
const savingsPerTurn = stayPerTurn - switchPerTurn;
|
|
154
|
+
|
|
155
|
+
const priced = {
|
|
156
|
+
...base,
|
|
157
|
+
switchOnceUsd: switchOnce,
|
|
158
|
+
stayPerTurnUsd: stayPerTurn,
|
|
159
|
+
switchPerTurnUsd: switchPerTurn,
|
|
160
|
+
projectedStaySavingsUsd: savingsPerTurn,
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// Switching to a same-or-more-expensive target never pays on economics.
|
|
164
|
+
// (Escalations don't route through this gate, so blocking is safe.)
|
|
165
|
+
if (savingsPerTurn <= 0) {
|
|
166
|
+
return {
|
|
167
|
+
...priced,
|
|
168
|
+
switchAllowed: false,
|
|
169
|
+
reason: 'never_profitable',
|
|
170
|
+
breakEvenTurns: Infinity,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const breakEvenTurns = switchOnce / savingsPerTurn;
|
|
175
|
+
const allowed = breakEvenTurns <= remaining;
|
|
176
|
+
|
|
177
|
+
logger.debug({
|
|
178
|
+
current: `${current.provider}:${current.model}`,
|
|
179
|
+
target: `${target.provider}:${target.model}`,
|
|
180
|
+
warmPrefixTokens: W,
|
|
181
|
+
breakEvenTurns: Number(breakEvenTurns.toFixed(2)),
|
|
182
|
+
expectedRemainingTurns: remaining,
|
|
183
|
+
allowed,
|
|
184
|
+
}, '[CacheSwitchCost] break-even evaluated');
|
|
185
|
+
|
|
186
|
+
return {
|
|
187
|
+
...priced,
|
|
188
|
+
switchAllowed: allowed,
|
|
189
|
+
reason: allowed ? 'break_even_cleared' : 'break_even_blocked',
|
|
190
|
+
breakEvenTurns,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
module.exports = { evaluateSwitch };
|
|
@@ -146,7 +146,8 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
|
|
|
146
146
|
continue;
|
|
147
147
|
}
|
|
148
148
|
const ordered = Array.from(tierBuckets.entries()).sort((a, b) => a[0] - b[0]);
|
|
149
|
-
|
|
149
|
+
const [defaultLower, defaultUpper] = DEFAULT_RANGES[tier];
|
|
150
|
+
let suggestedUpper = defaultUpper;
|
|
150
151
|
const bucketsSummary = [];
|
|
151
152
|
for (const [lo, vals] of ordered) {
|
|
152
153
|
if (vals.length < 5) {
|
|
@@ -155,6 +156,12 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
|
|
|
155
156
|
}
|
|
156
157
|
const med = _median(vals);
|
|
157
158
|
bucketsSummary.push({ bucket: lo, samples: vals.length, median: med });
|
|
159
|
+
// Buckets outside the tier's own default range come from fallback-served
|
|
160
|
+
// requests (another tier's scores recorded under the serving tier). They
|
|
161
|
+
// say nothing about this tier's real range, and letting them shrink the
|
|
162
|
+
// upper bound below the lower bound collapses the tier to a single point
|
|
163
|
+
// in re-stitching (observed 2026-07-26: MEDIUM → [20,20]).
|
|
164
|
+
if (lo + 4 < defaultLower || lo > defaultUpper) continue;
|
|
158
165
|
if (med < floor && lo + 4 < suggestedUpper) {
|
|
159
166
|
// shrink tier upper bound just below the failing bucket
|
|
160
167
|
suggestedUpper = lo + 4;
|
|
@@ -795,7 +795,13 @@ async function analyzeComplexity(payload, options = {}) {
|
|
|
795
795
|
threshold,
|
|
796
796
|
mode: 'weighted',
|
|
797
797
|
recommendation,
|
|
798
|
-
|
|
798
|
+
// taskType rides alongside the numeric dimensions. The legacy path
|
|
799
|
+
// has always carried breakdown.taskType, and both the deescalator
|
|
800
|
+
// (tier + request_type evidence key) and the telemetry request_type
|
|
801
|
+
// column derive from it — without it here, weighted-mode requests
|
|
802
|
+
// (the default) recorded request_type NULL and could never
|
|
803
|
+
// accumulate demotion evidence.
|
|
804
|
+
breakdown: { ...weighted.dimensions, taskType: taskTypeResult },
|
|
799
805
|
weights: weighted.weights,
|
|
800
806
|
meta: weighted.meta,
|
|
801
807
|
forceReason: taskTypeResult.reason?.startsWith('force_') ? taskTypeResult.reason : null,
|