lynkr 9.10.0 → 9.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -11
- package/bin/lynkr-init.js +7 -0
- package/bin/statusline.js +80 -0
- package/bin/wrap.js +10 -6
- package/config/model-tiers.json +5 -5
- package/package.json +9 -4
- package/public/dashboard.html +354 -1
- package/src/api/openai-router.js +120 -2
- package/src/api/router.js +34 -0
- package/src/clients/databricks.js +148 -66
- package/src/clients/gpt-utils.js +8 -1
- package/src/clients/ollama-utils.js +19 -10
- package/src/clients/prompt-cache-injection.js +93 -30
- package/src/config/index.js +79 -0
- package/src/context/compression.js +27 -8
- package/src/context/distill.js +2 -1
- package/src/dashboard/api.js +82 -2
- package/src/dashboard/recommendations/cache-weakspots.js +88 -0
- package/src/dashboard/recommendations/deadweight.js +82 -0
- package/src/dashboard/recommendations/downsize.js +109 -0
- package/src/dashboard/recommendations/index.js +106 -0
- package/src/dashboard/recommendations/side-requests.js +68 -0
- package/src/dashboard/recommendations/wilson.js +24 -0
- package/src/dashboard/router.js +5 -0
- package/src/memory/distiller.js +350 -0
- package/src/memory/extractor.js +0 -8
- package/src/memory/format.js +17 -2
- package/src/memory/index.js +10 -0
- package/src/memory/retriever.js +7 -2
- package/src/memory/skills-cache.js +180 -0
- package/src/memory/tencentdb-launcher.js +459 -0
- package/src/memory/wiki.js +171 -0
- package/src/orchestrator/index.js +85 -2
- package/src/orchestrator/sse-transformer.js +9 -3
- package/src/prompts/system.js +20 -6
- package/src/routing/affinity-store.js +68 -1
- package/src/routing/agentic-detector.js +9 -1
- package/src/routing/cache-economics.js +123 -0
- package/src/routing/cache-switch-cost.js +194 -0
- package/src/routing/calibration.js +8 -1
- package/src/routing/complexity-analyzer.js +37 -10
- package/src/routing/context-compressor.js +170 -0
- package/src/routing/difficulty-classifier.js +115 -13
- package/src/routing/index.js +161 -53
- package/src/routing/model-registry.js +35 -10
- package/src/routing/session-affinity.js +60 -0
- package/src/routing/telemetry.js +343 -2
- package/src/routing/verifier.js +41 -2
- package/src/server.js +36 -3
- package/src/tools/code-graph.js +6 -4
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wiki — Large-Block Content Registry (TencentDB-Agent-Memory inspired)
|
|
3
|
+
*
|
|
4
|
+
* Registers large content blocks (docs, configs, big tool outputs) the
|
|
5
|
+
* first time they appear in old conversation history, then replaces later
|
|
6
|
+
* near-duplicates with a short reference + summary instead of resending
|
|
7
|
+
* the full block. Entries persist in the memories table (type='wiki') so
|
|
8
|
+
* dedup works across sessions.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const store = require("./store");
|
|
12
|
+
const distill = require("../context/distill");
|
|
13
|
+
const config = require("../config");
|
|
14
|
+
const logger = require("../logger");
|
|
15
|
+
|
|
16
|
+
const CHARS_PER_TOKEN = 4;
|
|
17
|
+
const MAX_CACHED_ENTRIES = 500;
|
|
18
|
+
const MAX_SIGNATURE_LINES = 200;
|
|
19
|
+
|
|
20
|
+
// In-memory entry cache: [{ id, summary, signature: Set<string> }]
|
|
21
|
+
let entryCache = null;
|
|
22
|
+
|
|
23
|
+
function wikiConfig() {
|
|
24
|
+
return config.memory?.wiki ?? {};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function minChars() {
|
|
28
|
+
return (wikiConfig().minTokens ?? 500) * CHARS_PER_TOKEN;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Load persisted wiki entries into the in-memory cache (lazy, once).
|
|
33
|
+
*/
|
|
34
|
+
function loadCache() {
|
|
35
|
+
if (entryCache) return entryCache;
|
|
36
|
+
|
|
37
|
+
entryCache = [];
|
|
38
|
+
try {
|
|
39
|
+
const rows = store.getMemoriesByType("wiki", MAX_CACHED_ENTRIES);
|
|
40
|
+
for (const row of rows) {
|
|
41
|
+
const meta = typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata;
|
|
42
|
+
if (!meta?.signatureLines?.length) continue;
|
|
43
|
+
entryCache.push({
|
|
44
|
+
id: row.id,
|
|
45
|
+
summary: row.content,
|
|
46
|
+
signature: new Set(meta.signatureLines),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
} catch (err) {
|
|
50
|
+
logger.warn({ err }, "[wiki] Failed to load wiki entries, starting empty");
|
|
51
|
+
}
|
|
52
|
+
return entryCache;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Build a one-line summary for a content block: first heading or first
|
|
57
|
+
* meaningful line, plus a size note.
|
|
58
|
+
*/
|
|
59
|
+
function makeSummary(text) {
|
|
60
|
+
const lines = distill.normalizeText(text).split("\n").filter(Boolean);
|
|
61
|
+
const heading = lines.find(l => /^#{1,4}\s|^[A-Z][^a-z]*$/.test(l.trim()));
|
|
62
|
+
const first = (heading || lines[0] || "").trim().slice(0, 120);
|
|
63
|
+
return `${first} (${lines.length} lines, ~${Math.ceil(text.length / CHARS_PER_TOKEN)} tokens)`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Register a large block, or return a compact reference if a similar
|
|
68
|
+
* block is already known.
|
|
69
|
+
*
|
|
70
|
+
* @param {string} text - Content block from old history
|
|
71
|
+
* @returns {{ref: string, id: number, saved: number}|null} Reference when a
|
|
72
|
+
* similar entry exists; null when the block was registered or is too small
|
|
73
|
+
*/
|
|
74
|
+
function registerOrDereference(text) {
|
|
75
|
+
if (wikiConfig().enabled === false) return null;
|
|
76
|
+
if (!text || text.length < minChars()) return null;
|
|
77
|
+
|
|
78
|
+
const threshold = wikiConfig().similarityThreshold ?? 0.85;
|
|
79
|
+
const signature = distill.extractSignature(text);
|
|
80
|
+
const cache = loadCache();
|
|
81
|
+
|
|
82
|
+
for (const entry of cache) {
|
|
83
|
+
const sim = distill.jaccardSimilarity(signature, entry.signature);
|
|
84
|
+
if (sim >= threshold) {
|
|
85
|
+
const ref = `[wiki:${entry.id} — ${(sim * 100).toFixed(0)}% match] ${entry.summary}`;
|
|
86
|
+
return { ref, id: entry.id, saved: text.length - ref.length };
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// No match — register for future dedup
|
|
91
|
+
try {
|
|
92
|
+
const summary = makeSummary(text);
|
|
93
|
+
const memory = store.createMemory({
|
|
94
|
+
sessionId: null, // wiki entries are global
|
|
95
|
+
content: summary,
|
|
96
|
+
type: "wiki",
|
|
97
|
+
category: "reference",
|
|
98
|
+
importance: 0.3,
|
|
99
|
+
metadata: {
|
|
100
|
+
signatureLines: Array.from(signature).slice(0, MAX_SIGNATURE_LINES),
|
|
101
|
+
originalChars: text.length,
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
cache.push({ id: memory.id, summary, signature });
|
|
105
|
+
if (cache.length > MAX_CACHED_ENTRIES) cache.shift();
|
|
106
|
+
} catch (err) {
|
|
107
|
+
logger.warn({ err }, "[wiki] Failed to register wiki entry");
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Replace large repeated blocks inside old-history messages with wiki
|
|
115
|
+
* references. Only text and tool_result content is touched.
|
|
116
|
+
*
|
|
117
|
+
* @param {Array} messages - Old (about-to-be-summarized) messages
|
|
118
|
+
* @returns {{messages: Array, stats: {registered: number, dereferenced: number, charsSaved: number}}}
|
|
119
|
+
*/
|
|
120
|
+
function dereferenceMessages(messages) {
|
|
121
|
+
const stats = { registered: 0, dereferenced: 0, charsSaved: 0 };
|
|
122
|
+
if (wikiConfig().enabled === false || !messages?.length) {
|
|
123
|
+
return { messages: messages || [], stats };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const processText = (text) => {
|
|
127
|
+
const before = loadCache().length;
|
|
128
|
+
const result = registerOrDereference(text);
|
|
129
|
+
if (result) {
|
|
130
|
+
stats.dereferenced++;
|
|
131
|
+
stats.charsSaved += result.saved;
|
|
132
|
+
return result.ref;
|
|
133
|
+
}
|
|
134
|
+
if (loadCache().length > before) stats.registered++;
|
|
135
|
+
return text;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
const processed = messages.map(msg => {
|
|
139
|
+
if (typeof msg.content === "string") {
|
|
140
|
+
if (msg.content.length < minChars()) return msg;
|
|
141
|
+
return { ...msg, content: processText(msg.content) };
|
|
142
|
+
}
|
|
143
|
+
if (!Array.isArray(msg.content)) return msg;
|
|
144
|
+
|
|
145
|
+
const newContent = msg.content.map(block => {
|
|
146
|
+
if (block.type === "text" && block.text?.length >= minChars()) {
|
|
147
|
+
return { ...block, text: processText(block.text) };
|
|
148
|
+
}
|
|
149
|
+
if (block.type === "tool_result" && typeof block.content === "string" &&
|
|
150
|
+
block.content.length >= minChars()) {
|
|
151
|
+
return { ...block, content: processText(block.content) };
|
|
152
|
+
}
|
|
153
|
+
return block;
|
|
154
|
+
});
|
|
155
|
+
return { ...msg, content: newContent };
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
return { messages: processed, stats };
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Reset the in-memory cache (test support). */
|
|
162
|
+
function resetCache() {
|
|
163
|
+
entryCache = null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
module.exports = {
|
|
167
|
+
registerOrDereference,
|
|
168
|
+
dereferenceMessages,
|
|
169
|
+
makeSummary,
|
|
170
|
+
resetCache,
|
|
171
|
+
};
|
|
@@ -21,6 +21,7 @@ const crypto = require("crypto");
|
|
|
21
21
|
const { getSemanticCache, isSemanticCacheEnabled } = require("../cache/semantic");
|
|
22
22
|
const { areSimilarToolCalls } = require("../clients/gpt-utils");
|
|
23
23
|
const { getModelRegistrySync } = require("../routing/model-registry");
|
|
24
|
+
const sessionAffinity = require("../routing/session-affinity");
|
|
24
25
|
|
|
25
26
|
/**
|
|
26
27
|
* Get destination URL for audit logging based on provider type
|
|
@@ -433,7 +434,7 @@ function injectToolLoopStopInstruction(messages, threshold = 5) {
|
|
|
433
434
|
// Inject instruction to stop tool calls and provide a final answer
|
|
434
435
|
const stopInstruction = {
|
|
435
436
|
role: "user",
|
|
436
|
-
content: `⚠️ IMPORTANT: You have already executed ${toolResultCount} tool calls in this conversation. This is likely an infinite loop. STOP calling tools immediately and provide a direct text response to the user based on the information you have gathered. If you cannot complete the task, explain why. DO NOT call any more tools.`,
|
|
437
|
+
content: `⚠️ IMPORTANT: You have already executed ${toolResultCount} tool calls in this conversation. This is likely an infinite loop. STOP calling tools immediately and provide a direct text response to the user based on the information you have gathered. If you cannot complete the task, explain why. DO NOT call any more tools. Your response must contain your actual findings — NOT an acknowledgment or restatement of this warning.`,
|
|
437
438
|
};
|
|
438
439
|
|
|
439
440
|
return [...messages, stopInstruction];
|
|
@@ -452,6 +453,23 @@ const DEDUP_MAX_SIGNATURES = 50;
|
|
|
452
453
|
const DEDUP_WARN_THRESHOLD = 5;
|
|
453
454
|
const DEDUP_TERMINATE_THRESHOLD = 8;
|
|
454
455
|
|
|
456
|
+
/**
|
|
457
|
+
* State-management tools are called repeatedly with similar args BY DESIGN —
|
|
458
|
+
* that's their job, not a loop. Opencode calls `todowrite` after every step;
|
|
459
|
+
* Claude Code uses `TodoWrite`; OpenWorker uses `todo_write`/`propose_plan`.
|
|
460
|
+
* Counting them in the dedup tracker produced false-positive loop warnings
|
|
461
|
+
* and force-terminations on healthy agent sessions (live incident: an
|
|
462
|
+
* opencode analysis run was force-terminated into a raw tool-result dump
|
|
463
|
+
* because its todo updates hit the terminate threshold).
|
|
464
|
+
* @param {Object} toolCall - tool_use block
|
|
465
|
+
* @returns {boolean}
|
|
466
|
+
*/
|
|
467
|
+
function isDedupExemptTool(toolCall) {
|
|
468
|
+
const raw = toolCall?.function?.name ?? toolCall?.name ?? '';
|
|
469
|
+
const name = String(raw).toLowerCase().replace(/[_-]/g, '');
|
|
470
|
+
return name.includes('todo') || name === 'updateplan' || name === 'proposeplan';
|
|
471
|
+
}
|
|
472
|
+
|
|
455
473
|
/**
|
|
456
474
|
* Initialise session.metadata.toolCallDedup if missing.
|
|
457
475
|
* @param {Object} session
|
|
@@ -462,6 +480,7 @@ function ensureDedupStructure(session) {
|
|
|
462
480
|
session.metadata.toolCallDedup = {
|
|
463
481
|
signatures: {},
|
|
464
482
|
similarGroups: {},
|
|
483
|
+
seenIds: {},
|
|
465
484
|
lastResetAt: Date.now(),
|
|
466
485
|
warningInjected: false,
|
|
467
486
|
};
|
|
@@ -476,9 +495,26 @@ function ensureDedupStructure(session) {
|
|
|
476
495
|
*/
|
|
477
496
|
function recordCrossRequestToolCall(session, toolCall) {
|
|
478
497
|
if (!session?.metadata) return;
|
|
498
|
+
if (isDedupExemptTool(toolCall)) return; // repeat-by-design tools never count toward loop detection
|
|
479
499
|
ensureDedupStructure(session);
|
|
480
500
|
|
|
481
501
|
const dedup = session.metadata.toolCallDedup;
|
|
502
|
+
|
|
503
|
+
// Stateless HTTP agents (opencode, Cursor, OpenWorker) replay the FULL
|
|
504
|
+
// conversation history on every agent-loop step, so the same tool_use block
|
|
505
|
+
// arrives again on step 2, 3, 4… Counting each replay made counts grow
|
|
506
|
+
// quadratically with turn length — any exploration past ~5 steps hit the
|
|
507
|
+
// terminate threshold regardless of what it was doing (live incident:
|
|
508
|
+
// a 7-step opencode code-trace was force-terminated as a "loop").
|
|
509
|
+
// Each tool_use block carries a unique id: count each id exactly once.
|
|
510
|
+
const blockId = toolCall?.id;
|
|
511
|
+
if (blockId) {
|
|
512
|
+
if (!dedup.seenIds) dedup.seenIds = {};
|
|
513
|
+
if (dedup.seenIds[blockId]) return; // already counted this exact call
|
|
514
|
+
dedup.seenIds[blockId] = 1;
|
|
515
|
+
// Bounded: reset wipes this on each new user question; cap as a backstop.
|
|
516
|
+
if (Object.keys(dedup.seenIds).length > 500) dedup.seenIds = { [blockId]: 1 };
|
|
517
|
+
}
|
|
482
518
|
const signature = getToolCallSignature(toolCall);
|
|
483
519
|
const toolName = toolCall.function?.name ?? toolCall.name ?? 'unknown';
|
|
484
520
|
const args = toolCall.function?.arguments ?? toolCall.input;
|
|
@@ -620,6 +656,7 @@ function resetDedupTracking(session) {
|
|
|
620
656
|
session.metadata.toolCallDedup = {
|
|
621
657
|
signatures: {},
|
|
622
658
|
similarGroups: {},
|
|
659
|
+
seenIds: {},
|
|
623
660
|
lastResetAt: Date.now(),
|
|
624
661
|
warningInjected: false,
|
|
625
662
|
};
|
|
@@ -1421,6 +1458,30 @@ async function runAgentLoop({
|
|
|
1421
1458
|
|
|
1422
1459
|
|
|
1423
1460
|
if (steps === 1 && agentTimer) agentTimer.mark("preCompression");
|
|
1461
|
+
|
|
1462
|
+
// === CONVERSATION DISTILLATION (TencentDB-inspired L0-L3 pipeline) ===
|
|
1463
|
+
// Long conversations collapse older turns into one distilled block
|
|
1464
|
+
// (persona + scenario summary) before history compression runs.
|
|
1465
|
+
if (steps === 1 && config.memory?.enabled !== false && config.memory?.distillation?.enabled !== false) {
|
|
1466
|
+
try {
|
|
1467
|
+
const distiller = require('../memory/distiller');
|
|
1468
|
+
if (distiller.needsDistillation(cleanPayload.messages)) {
|
|
1469
|
+
const result = distiller.distillMessages(cleanPayload.messages, {
|
|
1470
|
+
sessionId: session?.id,
|
|
1471
|
+
});
|
|
1472
|
+
if (result.applied) {
|
|
1473
|
+
cleanPayload.messages = result.messages;
|
|
1474
|
+
logger.debug({
|
|
1475
|
+
sessionId: session?.id ?? null,
|
|
1476
|
+
...result.stats,
|
|
1477
|
+
}, '[distiller] Conversation distillation applied');
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
} catch (err) {
|
|
1481
|
+
logger.warn({ err, sessionId: session?.id }, 'Distillation failed, continuing with full history');
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1424
1485
|
if (steps === 1 && config.historyCompression?.enabled !== false) {
|
|
1425
1486
|
try {
|
|
1426
1487
|
if (historyCompression.needsCompression(cleanPayload.messages)) {
|
|
@@ -1883,6 +1944,28 @@ IMPORTANT TOOL USAGE RULES:
|
|
|
1883
1944
|
}
|
|
1884
1945
|
}
|
|
1885
1946
|
|
|
1947
|
+
// Cache-aware routing (Phase 1): persist the session's warm-prefix state
|
|
1948
|
+
// from the response's cache counters so the router can price a mid-session
|
|
1949
|
+
// model switch against the live cache clock. Best-effort — never blocks
|
|
1950
|
+
// the response path.
|
|
1951
|
+
if (session?.id && actualUsage) {
|
|
1952
|
+
try {
|
|
1953
|
+
// Prefer the ROUTED provider/model over the request-level default:
|
|
1954
|
+
// the tier router inside invokeModel may have diverged from
|
|
1955
|
+
// providerType, and the cache lives with whoever actually served
|
|
1956
|
+
// (verified live: databricks default label on ollama-served turns).
|
|
1957
|
+
const served = databricksResponse.routingDecision || {};
|
|
1958
|
+
sessionAffinity.recordCacheUsage(session.id, {
|
|
1959
|
+
provider: served.provider || providerType,
|
|
1960
|
+
model: served.model || cleanPayload.model,
|
|
1961
|
+
cacheReadTokens: actualUsage.cacheReadTokens,
|
|
1962
|
+
cacheCreationTokens: actualUsage.cacheCreationTokens,
|
|
1963
|
+
});
|
|
1964
|
+
} catch (err) {
|
|
1965
|
+
logger.debug({ err: err.message }, "[Orchestrator] cache-state update failed");
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1886
1969
|
if (auditLogger.enabled) {
|
|
1887
1970
|
const latencyMs = Date.now() - start;
|
|
1888
1971
|
|
|
@@ -2879,7 +2962,7 @@ async function processMessage({ payload, headers, session, cwd, options = {} })
|
|
|
2879
2962
|
if (Array.isArray(payload?.messages)) {
|
|
2880
2963
|
payload.messages.push({
|
|
2881
2964
|
role: "user",
|
|
2882
|
-
content: `⚠️ CRITICAL SYSTEM WARNING: You have called the "${dedupToolName}" tool ${maxCount} times with identical or similar parameters across multiple requests. This IS an infinite loop. STOP calling this tool immediately. You MUST now provide a direct text response based on the results you have received. If the tool returned "no results" or empty output, that IS the final answer - do not retry.
|
|
2965
|
+
content: `⚠️ CRITICAL SYSTEM WARNING: You have called the "${dedupToolName}" tool ${maxCount} times with identical or similar parameters across multiple requests. This IS an infinite loop. STOP calling this tool immediately. You MUST now provide a direct text response based on the results you have received. If the tool returned "no results" or empty output, that IS the final answer - do not retry. Your response must contain your actual findings from the tool results gathered so far — NOT an acknowledgment or restatement of this warning. Do not mention this warning in your response.`,
|
|
2883
2966
|
});
|
|
2884
2967
|
}
|
|
2885
2968
|
}
|
|
@@ -20,9 +20,14 @@
|
|
|
20
20
|
*/
|
|
21
21
|
const logger = require("../logger");
|
|
22
22
|
|
|
23
|
-
// Upstreams whose streaming wire format is OpenAI Chat Completions SSE.
|
|
24
|
-
// ollama is excluded (own format + buffering quirks)
|
|
25
|
-
//
|
|
23
|
+
// Upstreams whose streaming streaming wire format is OpenAI Chat Completions SSE.
|
|
24
|
+
// ollama is excluded (own format + buffering quirks); zai is excluded (its
|
|
25
|
+
// invoke fn converts buffered responses to Anthropic, and its Anthropic-format
|
|
26
|
+
// endpoint has the native passthrough path instead). moonshot joined the list
|
|
27
|
+
// once invokeMoonshot learned to return the raw stream (its old forced
|
|
28
|
+
// stream:false predated this transformer). Caveat: reasoning_content deltas
|
|
29
|
+
// (kimi thinking) are not reshaped — thinking text is dropped from streamed
|
|
30
|
+
// responses; the buffered path still lifts it into thinking blocks.
|
|
26
31
|
const DEFAULT_OPENAI_SSE_PROVIDERS = [
|
|
27
32
|
"openai",
|
|
28
33
|
"azure-openai",
|
|
@@ -30,6 +35,7 @@ const DEFAULT_OPENAI_SSE_PROVIDERS = [
|
|
|
30
35
|
"databricks",
|
|
31
36
|
"lmstudio",
|
|
32
37
|
"llamacpp",
|
|
38
|
+
"moonshot",
|
|
33
39
|
];
|
|
34
40
|
|
|
35
41
|
function _transformProviders() {
|
package/src/prompts/system.js
CHANGED
|
@@ -70,13 +70,27 @@ function compressToolDescriptions(tools, mode = null) {
|
|
|
70
70
|
return tools; // Return unmodified if not in minimal mode
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
// Normalize to Anthropic shape: OpenAI-format callers (OpenWorker, aisuite,
|
|
74
|
+
// any OpenAI SDK) send {type:"function", function:{name, description, parameters}}.
|
|
75
|
+
// Unwrap once here so the compression logic below can assume input_schema exists.
|
|
76
|
+
tools = tools.map(tool =>
|
|
77
|
+
tool?.type === "function" && tool.function
|
|
78
|
+
? {
|
|
79
|
+
name: tool.function.name,
|
|
80
|
+
description: tool.function.description,
|
|
81
|
+
input_schema: tool.function.parameters || { type: "object", properties: {} },
|
|
82
|
+
}
|
|
83
|
+
: tool
|
|
84
|
+
);
|
|
85
|
+
|
|
73
86
|
return tools.map(tool => {
|
|
87
|
+
const input_schema = tool.input_schema || { type: "object", properties: {} };
|
|
74
88
|
const compressed = {
|
|
75
89
|
name: tool.name,
|
|
76
90
|
input_schema: {
|
|
77
|
-
type:
|
|
91
|
+
type: input_schema.type || "object",
|
|
78
92
|
properties: {},
|
|
79
|
-
required:
|
|
93
|
+
required: input_schema.required || [],
|
|
80
94
|
}
|
|
81
95
|
};
|
|
82
96
|
|
|
@@ -86,8 +100,8 @@ function compressToolDescriptions(tools, mode = null) {
|
|
|
86
100
|
}
|
|
87
101
|
|
|
88
102
|
// Compress property descriptions
|
|
89
|
-
if (
|
|
90
|
-
for (const [key, value] of Object.entries(
|
|
103
|
+
if (input_schema.properties) {
|
|
104
|
+
for (const [key, value] of Object.entries(input_schema.properties)) {
|
|
91
105
|
compressed.input_schema.properties[key] = {
|
|
92
106
|
type: value.type,
|
|
93
107
|
};
|
|
@@ -108,8 +122,8 @@ function compressToolDescriptions(tools, mode = null) {
|
|
|
108
122
|
}
|
|
109
123
|
|
|
110
124
|
// Preserve additionalProperties if set
|
|
111
|
-
if (
|
|
112
|
-
compressed.input_schema.additionalProperties =
|
|
125
|
+
if (input_schema.additionalProperties !== undefined) {
|
|
126
|
+
compressed.input_schema.additionalProperties = input_schema.additionalProperties;
|
|
113
127
|
}
|
|
114
128
|
|
|
115
129
|
return compressed;
|
|
@@ -56,6 +56,12 @@ function _db() {
|
|
|
56
56
|
if (!cols.has("has_tool_history")) {
|
|
57
57
|
db.exec("ALTER TABLE session_pins ADD COLUMN has_tool_history INTEGER DEFAULT 0");
|
|
58
58
|
}
|
|
59
|
+
// Additive migration for cache-aware routing (Phase 1): JSON blob
|
|
60
|
+
// holding {warmPrefixTokens, provider, model, lastRequestAt, ttlMs},
|
|
61
|
+
// updated after every upstream response that reports cache usage.
|
|
62
|
+
if (!cols.has("cache_state")) {
|
|
63
|
+
db.exec("ALTER TABLE session_pins ADD COLUMN cache_state TEXT");
|
|
64
|
+
}
|
|
59
65
|
schemaEnsured = true;
|
|
60
66
|
} catch (err) {
|
|
61
67
|
degradation.record("feedback", err);
|
|
@@ -160,6 +166,67 @@ function save(sessionId, pin) {
|
|
|
160
166
|
}
|
|
161
167
|
}
|
|
162
168
|
|
|
169
|
+
/**
|
|
170
|
+
* Persist per-session prompt-cache state (Phase 1, cache-aware routing).
|
|
171
|
+
* Piggybacks on the session_pins row; creates a minimal row when the session
|
|
172
|
+
* has no pin yet (possible when sticky sessions are disabled but tracking
|
|
173
|
+
* is on). Best-effort like everything else in this module.
|
|
174
|
+
*
|
|
175
|
+
* @param {string} sessionId
|
|
176
|
+
* @param {{warmPrefixTokens:number, provider:string, model:string|null, lastRequestAt:number, ttlMs:number}} state
|
|
177
|
+
*/
|
|
178
|
+
function saveCacheState(sessionId, state) {
|
|
179
|
+
if (!sessionId || !state?.provider) return;
|
|
180
|
+
const db = _db();
|
|
181
|
+
if (!db) return;
|
|
182
|
+
try {
|
|
183
|
+
const json = JSON.stringify(state);
|
|
184
|
+
const res = _stmt(
|
|
185
|
+
db,
|
|
186
|
+
"cache_state_update",
|
|
187
|
+
"UPDATE session_pins SET cache_state = ? WHERE session_id = ?"
|
|
188
|
+
).run(json, sessionId);
|
|
189
|
+
if (res.changes === 0) {
|
|
190
|
+
_stmt(
|
|
191
|
+
db,
|
|
192
|
+
"cache_state_insert",
|
|
193
|
+
`INSERT INTO session_pins (session_id, provider, model, ts, cache_state)
|
|
194
|
+
VALUES (?, ?, ?, ?, ?)
|
|
195
|
+
ON CONFLICT(session_id) DO UPDATE SET cache_state = excluded.cache_state`
|
|
196
|
+
).run(sessionId, state.provider, state.model ?? null, Date.now(), json);
|
|
197
|
+
}
|
|
198
|
+
} catch (err) {
|
|
199
|
+
degradation.record("feedback", err);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Load per-session cache state. Returns null when absent, unparsable, or the
|
|
205
|
+
* DB is unavailable — callers treat null as "no cache signal for this
|
|
206
|
+
* session" (provider doesn't report cache usage, or no response seen yet).
|
|
207
|
+
*
|
|
208
|
+
* @param {string} sessionId
|
|
209
|
+
* @returns {{warmPrefixTokens:number, provider:string, model:string|null, lastRequestAt:number, ttlMs:number}|null}
|
|
210
|
+
*/
|
|
211
|
+
function loadCacheState(sessionId) {
|
|
212
|
+
if (!sessionId) return null;
|
|
213
|
+
const db = _db();
|
|
214
|
+
if (!db) return null;
|
|
215
|
+
try {
|
|
216
|
+
const row = _stmt(
|
|
217
|
+
db,
|
|
218
|
+
"cache_state_load",
|
|
219
|
+
"SELECT cache_state FROM session_pins WHERE session_id = ?"
|
|
220
|
+
).get(sessionId);
|
|
221
|
+
if (!row?.cache_state) return null;
|
|
222
|
+
const parsed = JSON.parse(row.cache_state);
|
|
223
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
224
|
+
} catch (err) {
|
|
225
|
+
degradation.record("feedback", err);
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
163
230
|
/**
|
|
164
231
|
* Remove a pin.
|
|
165
232
|
* @param {string} sessionId
|
|
@@ -205,4 +272,4 @@ function _clear() {
|
|
|
205
272
|
} catch { /* best-effort */ }
|
|
206
273
|
}
|
|
207
274
|
|
|
208
|
-
module.exports = { load, save, remove, cleanup, _clear };
|
|
275
|
+
module.exports = { load, save, remove, cleanup, saveCacheState, loadCacheState, _clear };
|
|
@@ -224,13 +224,21 @@ class AgenticDetector {
|
|
|
224
224
|
const agentType = this._classifyAgentType(score, signals);
|
|
225
225
|
const isAgentic = score >= 25;
|
|
226
226
|
|
|
227
|
+
// AGENTIC_DISABLE_FLOOR=1 bypasses the per-agent-type minTier — the raw
|
|
228
|
+
// complexity score decides the tier without the tools-based bump. Useful
|
|
229
|
+
// for demos/tests where tool-heavy clients (OpenWorker, aisuite) send 25+
|
|
230
|
+
// tools on every request and otherwise floor everything to MEDIUM+.
|
|
231
|
+
const flooredMinTier = process.env.AGENTIC_DISABLE_FLOOR === '1'
|
|
232
|
+
? 'SIMPLE'
|
|
233
|
+
: AGENT_TYPES[agentType].minTier;
|
|
234
|
+
|
|
227
235
|
const result = {
|
|
228
236
|
isAgentic,
|
|
229
237
|
agentType,
|
|
230
238
|
confidence: Math.min(score / 100, 1),
|
|
231
239
|
score,
|
|
232
240
|
signals,
|
|
233
|
-
minTier:
|
|
241
|
+
minTier: flooredMinTier,
|
|
234
242
|
scoreBoost: AGENT_TYPES[agentType].scoreBoost,
|
|
235
243
|
description: AGENT_TYPES[agentType].description,
|
|
236
244
|
clientProfile: profile?.name ?? null,
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider prompt-cache economics.
|
|
3
|
+
*
|
|
4
|
+
* Single resolution point for "what does the prompt cache cost on this
|
|
5
|
+
* provider/model": read price, write price, TTL, and mechanism. The
|
|
6
|
+
* switch-cost math in `cache-switch-cost.js` and the cache-state tracker in
|
|
7
|
+
* `session-affinity.js` read from here — decision code never hardcodes
|
|
8
|
+
* per-model numbers.
|
|
9
|
+
*
|
|
10
|
+
* Resolution order per field:
|
|
11
|
+
* 1. model-registry entry (models.dev carries absolute cacheRead/cacheWrite
|
|
12
|
+
* $/1M for many models)
|
|
13
|
+
* 2. provider-type fallback table below (multipliers of the model's input
|
|
14
|
+
* price), researched Aug 2026.
|
|
15
|
+
*
|
|
16
|
+
* Mechanisms:
|
|
17
|
+
* - 'explicit' — cache_control breakpoints, paid writes (Anthropic-style).
|
|
18
|
+
* TTL refreshes on every read, so lastRequestAt + ttlMs is a live clock.
|
|
19
|
+
* - 'automatic' — provider caches transparently, writes are free
|
|
20
|
+
* (OpenAI/DeepSeek/Gemini-style). TTL is best-effort.
|
|
21
|
+
* - 'local' — no dollar cost at all; the "cost" of a cold prefix is
|
|
22
|
+
* prefill latency, handled separately in cache-switch-cost.js.
|
|
23
|
+
*
|
|
24
|
+
* @module routing/cache-economics
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const logger = require('../logger');
|
|
28
|
+
|
|
29
|
+
// Fallback economics keyed by Lynkr provider type. `readMult`/`writeMult`
|
|
30
|
+
// are multipliers of the model's per-1M input price.
|
|
31
|
+
const PROVIDER_CACHE_DEFAULTS = {
|
|
32
|
+
// Anthropic-hosted (explicit cache_control, 1.25x write for the 5-min TTL,
|
|
33
|
+
// TTL refreshed on every read).
|
|
34
|
+
'azure-anthropic': { readMult: 0.1, writeMult: 1.25, ttlMs: 5 * 60 * 1000, mechanism: 'explicit' },
|
|
35
|
+
bedrock: { readMult: 0.1, writeMult: 1.25, ttlMs: 5 * 60 * 1000, mechanism: 'explicit' },
|
|
36
|
+
databricks: { readMult: 0.1, writeMult: 1.25, ttlMs: 5 * 60 * 1000, mechanism: 'explicit' },
|
|
37
|
+
|
|
38
|
+
// OpenAI automatic prefix caching: writes free, ~5-10 min idle eviction.
|
|
39
|
+
openai: { readMult: 0.1, writeMult: 0, ttlMs: 10 * 60 * 1000, mechanism: 'automatic' },
|
|
40
|
+
'azure-openai': { readMult: 0.1, writeMult: 0, ttlMs: 10 * 60 * 1000, mechanism: 'automatic' },
|
|
41
|
+
|
|
42
|
+
// Gemini implicit caching / GLM / Kimi / aggregators: free automatic
|
|
43
|
+
// caching, best-effort TTL. Aggregators (openrouter/edenai) depend on the
|
|
44
|
+
// underlying model; models.dev per-model data wins when present.
|
|
45
|
+
vertex: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
46
|
+
zai: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
47
|
+
moonshot: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
48
|
+
openrouter: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
49
|
+
edenai: { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
50
|
+
// DeepSeek direct (via aggregators today, kept for model-level matches):
|
|
51
|
+
deepseek: { readMult: 0.02, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' },
|
|
52
|
+
|
|
53
|
+
// Local runtimes: no dollars; KV cache lives as long as the model stays
|
|
54
|
+
// loaded (ollama keep_alive default ~5 min).
|
|
55
|
+
ollama: { readMult: 0, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'local' },
|
|
56
|
+
llamacpp: { readMult: 0, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'local' },
|
|
57
|
+
lmstudio: { readMult: 0, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'local' },
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const GENERIC_DEFAULT = { readMult: 0.1, writeMult: 0, ttlMs: 5 * 60 * 1000, mechanism: 'automatic' };
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Fallback cache economics for a provider type (no model lookup).
|
|
64
|
+
* @param {string|null} providerType
|
|
65
|
+
* @returns {{readMult:number, writeMult:number, ttlMs:number, mechanism:string}}
|
|
66
|
+
*/
|
|
67
|
+
function getProviderCacheDefaults(providerType) {
|
|
68
|
+
return PROVIDER_CACHE_DEFAULTS[providerType] || GENERIC_DEFAULT;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Resolve full cache economics for a provider/model pair.
|
|
73
|
+
*
|
|
74
|
+
* @param {string|null} providerType - Lynkr provider type (e.g. 'databricks')
|
|
75
|
+
* @param {string|null} model - model name for registry lookup
|
|
76
|
+
* @returns {{
|
|
77
|
+
* inputPerM:number, outputPerM:number,
|
|
78
|
+
* cacheReadPerM:number, cacheWritePerM:number,
|
|
79
|
+
* ttlMs:number, mechanism:string, unknownPricing:boolean
|
|
80
|
+
* }} All prices are USD per 1M tokens.
|
|
81
|
+
*/
|
|
82
|
+
function resolveCacheEconomics(providerType, model) {
|
|
83
|
+
const defaults = getProviderCacheDefaults(providerType);
|
|
84
|
+
|
|
85
|
+
let cost = null;
|
|
86
|
+
try {
|
|
87
|
+
const { getModelRegistrySync } = require('./model-registry');
|
|
88
|
+
cost = model ? getModelRegistrySync().getCost(model) : null;
|
|
89
|
+
} catch (err) {
|
|
90
|
+
logger.debug({ err: err.message }, '[CacheEconomics] registry lookup failed');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const inputPerM = typeof cost?.input === 'number' ? cost.input : 0;
|
|
94
|
+
const outputPerM = typeof cost?.output === 'number' ? cost.output : 0;
|
|
95
|
+
|
|
96
|
+
// Registry cacheTtlMs/cacheMechanism (Phase 4 entries) win over the
|
|
97
|
+
// provider fallback; models.dev absolute cache prices win over multipliers.
|
|
98
|
+
const mechanism = cost?.cacheMechanism || defaults.mechanism;
|
|
99
|
+
const ttlMs = typeof cost?.cacheTtlMs === 'number' ? cost.cacheTtlMs : defaults.ttlMs;
|
|
100
|
+
|
|
101
|
+
const cacheReadPerM = mechanism === 'local'
|
|
102
|
+
? 0
|
|
103
|
+
: (typeof cost?.cacheRead === 'number' ? cost.cacheRead : inputPerM * defaults.readMult);
|
|
104
|
+
const cacheWritePerM = mechanism === 'local'
|
|
105
|
+
? 0
|
|
106
|
+
: (typeof cost?.cacheWrite === 'number' ? cost.cacheWrite : inputPerM * defaults.writeMult);
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
inputPerM,
|
|
110
|
+
outputPerM,
|
|
111
|
+
cacheReadPerM,
|
|
112
|
+
cacheWritePerM,
|
|
113
|
+
ttlMs,
|
|
114
|
+
mechanism,
|
|
115
|
+
unknownPricing: !!cost?.unknown,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
module.exports = {
|
|
120
|
+
PROVIDER_CACHE_DEFAULTS,
|
|
121
|
+
getProviderCacheDefaults,
|
|
122
|
+
resolveCacheEconomics,
|
|
123
|
+
};
|