lynkr 9.10.0 → 9.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -11
- package/bin/lynkr-init.js +7 -0
- package/bin/statusline.js +80 -0
- package/bin/wrap.js +10 -6
- package/config/model-tiers.json +5 -5
- package/package.json +9 -4
- package/public/dashboard.html +354 -1
- package/src/api/openai-router.js +120 -2
- package/src/api/router.js +34 -0
- package/src/clients/databricks.js +148 -66
- package/src/clients/gpt-utils.js +8 -1
- package/src/clients/ollama-utils.js +19 -10
- package/src/clients/prompt-cache-injection.js +93 -30
- package/src/config/index.js +79 -0
- package/src/context/compression.js +27 -8
- package/src/context/distill.js +2 -1
- package/src/dashboard/api.js +82 -2
- package/src/dashboard/recommendations/cache-weakspots.js +88 -0
- package/src/dashboard/recommendations/deadweight.js +82 -0
- package/src/dashboard/recommendations/downsize.js +109 -0
- package/src/dashboard/recommendations/index.js +106 -0
- package/src/dashboard/recommendations/side-requests.js +68 -0
- package/src/dashboard/recommendations/wilson.js +24 -0
- package/src/dashboard/router.js +5 -0
- package/src/memory/distiller.js +350 -0
- package/src/memory/extractor.js +0 -8
- package/src/memory/format.js +17 -2
- package/src/memory/index.js +10 -0
- package/src/memory/retriever.js +7 -2
- package/src/memory/skills-cache.js +180 -0
- package/src/memory/tencentdb-launcher.js +459 -0
- package/src/memory/wiki.js +171 -0
- package/src/orchestrator/index.js +85 -2
- package/src/orchestrator/sse-transformer.js +9 -3
- package/src/prompts/system.js +20 -6
- package/src/routing/affinity-store.js +68 -1
- package/src/routing/agentic-detector.js +9 -1
- package/src/routing/cache-economics.js +123 -0
- package/src/routing/cache-switch-cost.js +194 -0
- package/src/routing/calibration.js +8 -1
- package/src/routing/complexity-analyzer.js +37 -10
- package/src/routing/context-compressor.js +170 -0
- package/src/routing/difficulty-classifier.js +115 -13
- package/src/routing/index.js +161 -53
- package/src/routing/model-registry.js +35 -10
- package/src/routing/session-affinity.js +60 -0
- package/src/routing/telemetry.js +343 -2
- package/src/routing/verifier.js +41 -2
- package/src/server.js +36 -3
- package/src/tools/code-graph.js +6 -4
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Switch-cost break-even model (Phase 3, cache-aware routing).
|
|
3
|
+
*
|
|
4
|
+
* Answers one question for a session that holds a warm prompt-cache prefix
|
|
5
|
+
* on its pinned model: does switching to a cheaper target model pay for the
|
|
6
|
+
* cache it breaks, within the turns this session is expected to have left?
|
|
7
|
+
*
|
|
8
|
+
* stayPerTurn = warm*cacheRead(cur) + new*input(cur) + out*output(cur)
|
|
9
|
+
* switchOnce = (warm + new) * cacheWrite(target) // one-time
|
|
10
|
+
* switchPerTurn = warm*cacheRead(tgt) + new*input(tgt) + out*output(tgt)
|
|
11
|
+
* breakEvenTurns = switchOnce / (stayPerTurn - switchPerTurn)
|
|
12
|
+
*
|
|
13
|
+
* Switch iff breakEvenTurns <= expectedRemainingTurns.
|
|
14
|
+
*
|
|
15
|
+
* The output-token term is deliberately included: for de-escalation the
|
|
16
|
+
* dominant per-turn saving is often the output-price spread (e.g. Opus $25/M
|
|
17
|
+
* vs Haiku $5/M out), and omitting it makes the router hold expensive pins
|
|
18
|
+
* long past the point the economics justify.
|
|
19
|
+
*
|
|
20
|
+
* Scope: this module prices COST-MOTIVATED switches (de-escalator, bandit
|
|
21
|
+
* exploration, economic downgrades). Hard triggers — risk keywords, force
|
|
22
|
+
* phrases, context overflow, vision, upward drift — are never gated here;
|
|
23
|
+
* correctness beats cost and the math can never favor a pricier model anyway
|
|
24
|
+
* (savings would be negative).
|
|
25
|
+
*
|
|
26
|
+
* Local models have no dollar cost; the price of a cold prefix is prefill
|
|
27
|
+
* latency, so a warm local prefix above a size threshold holds the pin.
|
|
28
|
+
*
|
|
29
|
+
* All pricing comes from cache-economics.js (registry-first, provider
|
|
30
|
+
* fallback) — no per-model numbers live in this file.
|
|
31
|
+
*
|
|
32
|
+
* @module routing/cache-switch-cost
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
const config = require('../config');
|
|
36
|
+
const logger = require('../logger');
|
|
37
|
+
|
|
38
|
+
const PER_TOKEN = 1 / 1_000_000; // registry prices are USD per 1M tokens
|
|
39
|
+
|
|
40
|
+
function _cfg() {
|
|
41
|
+
const c = config.routing?.cacheAware || {};
|
|
42
|
+
return {
|
|
43
|
+
enabled: c.enabled !== false,
|
|
44
|
+
defaultRemainingTurns: c.defaultRemainingTurns ?? 10,
|
|
45
|
+
newTokensPerTurn: c.newTokensPerTurn ?? 2000,
|
|
46
|
+
outputTokensPerTurn: c.outputTokensPerTurn ?? 800,
|
|
47
|
+
localMaxSwitchPrefixTokens: c.localMaxSwitchPrefixTokens ?? 16000,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Evaluate whether a cost-motivated switch away from the session's warm
|
|
53
|
+
* prefix should be allowed.
|
|
54
|
+
*
|
|
55
|
+
* @param {Object} args
|
|
56
|
+
* @param {Object|null} args.cacheState - from sessionAffinity.getCacheState:
|
|
57
|
+
* {warmPrefixTokens, provider, model, lastRequestAt, ttlMs, cold} or null.
|
|
58
|
+
* @param {{provider:string, model:string|null}} args.current - pinned target.
|
|
59
|
+
* @param {{provider:string, model:string|null}} args.target - proposed target.
|
|
60
|
+
* @param {number} [args.newTokensPerTurn] - est. fresh input tokens per turn.
|
|
61
|
+
* @param {number} [args.outputTokensPerTurn] - est. output tokens per turn.
|
|
62
|
+
* @param {number|null} [args.expectedRemainingTurns] - median remaining
|
|
63
|
+
* turns; falls back to the conservative config default when null.
|
|
64
|
+
* @param {Object} [args.deps] - test injection: {resolveCacheEconomics}.
|
|
65
|
+
* @returns {{
|
|
66
|
+
* switchAllowed: boolean,
|
|
67
|
+
* reason: string,
|
|
68
|
+
* breakEvenTurns: number|null,
|
|
69
|
+
* expectedRemainingTurns: number,
|
|
70
|
+
* warmPrefixTokens: number,
|
|
71
|
+
* switchOnceUsd: number|null,
|
|
72
|
+
* stayPerTurnUsd: number|null,
|
|
73
|
+
* switchPerTurnUsd: number|null,
|
|
74
|
+
* projectedStaySavingsUsd: number|null,
|
|
75
|
+
* }}
|
|
76
|
+
*/
|
|
77
|
+
function evaluateSwitch({
|
|
78
|
+
cacheState,
|
|
79
|
+
current,
|
|
80
|
+
target,
|
|
81
|
+
newTokensPerTurn,
|
|
82
|
+
outputTokensPerTurn,
|
|
83
|
+
expectedRemainingTurns,
|
|
84
|
+
deps = {},
|
|
85
|
+
} = {}) {
|
|
86
|
+
const cfg = _cfg();
|
|
87
|
+
const remaining = Number.isFinite(expectedRemainingTurns) && expectedRemainingTurns > 0
|
|
88
|
+
? expectedRemainingTurns
|
|
89
|
+
: cfg.defaultRemainingTurns;
|
|
90
|
+
|
|
91
|
+
const base = {
|
|
92
|
+
switchAllowed: true,
|
|
93
|
+
reason: 'no_cache_state',
|
|
94
|
+
breakEvenTurns: null,
|
|
95
|
+
expectedRemainingTurns: remaining,
|
|
96
|
+
warmPrefixTokens: cacheState?.warmPrefixTokens ?? 0,
|
|
97
|
+
switchOnceUsd: null,
|
|
98
|
+
stayPerTurnUsd: null,
|
|
99
|
+
switchPerTurnUsd: null,
|
|
100
|
+
projectedStaySavingsUsd: null,
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
if (!cfg.enabled) return { ...base, reason: 'feature_disabled' };
|
|
104
|
+
if (!current?.provider || !target?.provider) return base;
|
|
105
|
+
if (current.provider === target.provider && current.model === target.model) {
|
|
106
|
+
return { ...base, reason: 'same_model' };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// No tracked state → nothing warm to protect (provider reports no cache
|
|
110
|
+
// usage, or no response seen yet). Switching is cache-free.
|
|
111
|
+
if (!cacheState || !(cacheState.warmPrefixTokens > 0)) return base;
|
|
112
|
+
|
|
113
|
+
// State recorded for a different model than the pin we're defending —
|
|
114
|
+
// stale after a prior switch; treat as no protection.
|
|
115
|
+
if (cacheState.model && current.model && cacheState.model !== current.model) {
|
|
116
|
+
return { ...base, reason: 'stale_cache_state' };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Phase 2 — TTL clock. Anthropic-style caches refresh on every read, so
|
|
120
|
+
// lastRequestAt + ttlMs going stale means the prefix is already cold and
|
|
121
|
+
// the switch costs nothing extra.
|
|
122
|
+
if (cacheState.cold) return { ...base, reason: 'cache_cold' };
|
|
123
|
+
|
|
124
|
+
let resolve = deps.resolveCacheEconomics;
|
|
125
|
+
if (typeof resolve !== 'function') {
|
|
126
|
+
({ resolveCacheEconomics: resolve } = require('./cache-economics'));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const econCur = resolve(current.provider, current.model);
|
|
130
|
+
const econTgt = resolve(target.provider, target.model);
|
|
131
|
+
|
|
132
|
+
const W = cacheState.warmPrefixTokens;
|
|
133
|
+
const N = Number.isFinite(newTokensPerTurn) && newTokensPerTurn >= 0
|
|
134
|
+
? newTokensPerTurn : cfg.newTokensPerTurn;
|
|
135
|
+
const O = Number.isFinite(outputTokensPerTurn) && outputTokensPerTurn >= 0
|
|
136
|
+
? outputTokensPerTurn : cfg.outputTokensPerTurn;
|
|
137
|
+
|
|
138
|
+
// Local pin: no dollars in either column — the cost of abandoning the
|
|
139
|
+
// warm prefix is prefill latency on whatever serves the next turn.
|
|
140
|
+
// Hold while the prefix is large; small prefixes re-fill fast enough.
|
|
141
|
+
if (econCur.mechanism === 'local' && econTgt.mechanism === 'local') {
|
|
142
|
+
const allowed = W <= cfg.localMaxSwitchPrefixTokens;
|
|
143
|
+
return {
|
|
144
|
+
...base,
|
|
145
|
+
switchAllowed: allowed,
|
|
146
|
+
reason: allowed ? 'local_prefix_small' : 'local_prefill_hold',
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const stayPerTurn = (W * econCur.cacheReadPerM + N * econCur.inputPerM + O * econCur.outputPerM) * PER_TOKEN;
|
|
151
|
+
const switchOnce = (W + N) * econTgt.cacheWritePerM * PER_TOKEN;
|
|
152
|
+
const switchPerTurn = (W * econTgt.cacheReadPerM + N * econTgt.inputPerM + O * econTgt.outputPerM) * PER_TOKEN;
|
|
153
|
+
const savingsPerTurn = stayPerTurn - switchPerTurn;
|
|
154
|
+
|
|
155
|
+
const priced = {
|
|
156
|
+
...base,
|
|
157
|
+
switchOnceUsd: switchOnce,
|
|
158
|
+
stayPerTurnUsd: stayPerTurn,
|
|
159
|
+
switchPerTurnUsd: switchPerTurn,
|
|
160
|
+
projectedStaySavingsUsd: savingsPerTurn,
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// Switching to a same-or-more-expensive target never pays on economics.
|
|
164
|
+
// (Escalations don't route through this gate, so blocking is safe.)
|
|
165
|
+
if (savingsPerTurn <= 0) {
|
|
166
|
+
return {
|
|
167
|
+
...priced,
|
|
168
|
+
switchAllowed: false,
|
|
169
|
+
reason: 'never_profitable',
|
|
170
|
+
breakEvenTurns: Infinity,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const breakEvenTurns = switchOnce / savingsPerTurn;
|
|
175
|
+
const allowed = breakEvenTurns <= remaining;
|
|
176
|
+
|
|
177
|
+
logger.debug({
|
|
178
|
+
current: `${current.provider}:${current.model}`,
|
|
179
|
+
target: `${target.provider}:${target.model}`,
|
|
180
|
+
warmPrefixTokens: W,
|
|
181
|
+
breakEvenTurns: Number(breakEvenTurns.toFixed(2)),
|
|
182
|
+
expectedRemainingTurns: remaining,
|
|
183
|
+
allowed,
|
|
184
|
+
}, '[CacheSwitchCost] break-even evaluated');
|
|
185
|
+
|
|
186
|
+
return {
|
|
187
|
+
...priced,
|
|
188
|
+
switchAllowed: allowed,
|
|
189
|
+
reason: allowed ? 'break_even_cleared' : 'break_even_blocked',
|
|
190
|
+
breakEvenTurns,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
module.exports = { evaluateSwitch };
|
|
@@ -146,7 +146,8 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
|
|
|
146
146
|
continue;
|
|
147
147
|
}
|
|
148
148
|
const ordered = Array.from(tierBuckets.entries()).sort((a, b) => a[0] - b[0]);
|
|
149
|
-
|
|
149
|
+
const [defaultLower, defaultUpper] = DEFAULT_RANGES[tier];
|
|
150
|
+
let suggestedUpper = defaultUpper;
|
|
150
151
|
const bucketsSummary = [];
|
|
151
152
|
for (const [lo, vals] of ordered) {
|
|
152
153
|
if (vals.length < 5) {
|
|
@@ -155,6 +156,12 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
|
|
|
155
156
|
}
|
|
156
157
|
const med = _median(vals);
|
|
157
158
|
bucketsSummary.push({ bucket: lo, samples: vals.length, median: med });
|
|
159
|
+
// Buckets outside the tier's own default range come from fallback-served
|
|
160
|
+
// requests (another tier's scores recorded under the serving tier). They
|
|
161
|
+
// say nothing about this tier's real range, and letting them shrink the
|
|
162
|
+
// upper bound below the lower bound collapses the tier to a single point
|
|
163
|
+
// in re-stitching (observed 2026-07-26: MEDIUM → [20,20]).
|
|
164
|
+
if (lo + 4 < defaultLower || lo > defaultUpper) continue;
|
|
158
165
|
if (med < floor && lo + 4 < suggestedUpper) {
|
|
159
166
|
// shrink tier upper bound just below the failing bucket
|
|
160
167
|
suggestedUpper = lo + 4;
|
|
@@ -16,6 +16,7 @@ const logger = require('../logger');
|
|
|
16
16
|
const config = require('../config');
|
|
17
17
|
const codeGraph = require('../tools/code-graph');
|
|
18
18
|
const clientProfiles = require('./client-profiles');
|
|
19
|
+
const { compressMessages, shouldCompress } = require('./context-compressor');
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
22
|
* Return the tools the user actually added beyond the client harness's
|
|
@@ -769,13 +770,31 @@ function getThreshold() {
|
|
|
769
770
|
* @returns {Object} Complexity analysis result
|
|
770
771
|
*/
|
|
771
772
|
async function analyzeComplexity(payload, options = {}) {
|
|
772
|
-
|
|
773
|
-
|
|
773
|
+
// Phase 0: Context Compression (TencentDB-inspired)
|
|
774
|
+
// Compress conversation before scoring to account for token reduction
|
|
775
|
+
let workingPayload = payload;
|
|
776
|
+
let compressionStats = null;
|
|
777
|
+
|
|
778
|
+
const useCompression = options.compression ?? config.routing?.compression ?? true;
|
|
779
|
+
if (useCompression && shouldCompress(payload)) {
|
|
780
|
+
const { compressed, stats } = compressMessages(payload.messages);
|
|
781
|
+
compressionStats = stats;
|
|
782
|
+
workingPayload = { ...payload, messages: compressed };
|
|
783
|
+
|
|
784
|
+
logger.debug({
|
|
785
|
+
reduction: stats.reduction,
|
|
786
|
+
originalTokens: stats.original,
|
|
787
|
+
compressedTokens: stats.compressed,
|
|
788
|
+
}, '[complexity] Context compressed');
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
const content = extractContent(workingPayload);
|
|
792
|
+
const messageCount = workingPayload?.messages?.length ?? 0;
|
|
774
793
|
const useWeighted = options.weighted ?? config.routing?.weightedScoring ?? false;
|
|
775
794
|
|
|
776
795
|
// Use weighted scoring if enabled
|
|
777
796
|
if (useWeighted) {
|
|
778
|
-
const weighted = calculateWeightedScore(
|
|
797
|
+
const weighted = calculateWeightedScore(workingPayload, content);
|
|
779
798
|
const threshold = getThreshold();
|
|
780
799
|
const mode = config.smartToolSelection?.mode ?? 'heuristic';
|
|
781
800
|
|
|
@@ -795,12 +814,19 @@ async function analyzeComplexity(payload, options = {}) {
|
|
|
795
814
|
threshold,
|
|
796
815
|
mode: 'weighted',
|
|
797
816
|
recommendation,
|
|
798
|
-
|
|
817
|
+
// taskType rides alongside the numeric dimensions. The legacy path
|
|
818
|
+
// has always carried breakdown.taskType, and both the deescalator
|
|
819
|
+
// (tier + request_type evidence key) and the telemetry request_type
|
|
820
|
+
// column derive from it — without it here, weighted-mode requests
|
|
821
|
+
// (the default) recorded request_type NULL and could never
|
|
822
|
+
// accumulate demotion evidence.
|
|
823
|
+
breakdown: { ...weighted.dimensions, taskType: taskTypeResult },
|
|
799
824
|
weights: weighted.weights,
|
|
800
825
|
meta: weighted.meta,
|
|
801
826
|
forceReason: taskTypeResult.reason?.startsWith('force_') ? taskTypeResult.reason : null,
|
|
802
827
|
content: content.slice(0, 100) + (content.length > 100 ? '...' : ''),
|
|
803
828
|
graphSignals: null,
|
|
829
|
+
compression: compressionStats, // Add compression stats
|
|
804
830
|
};
|
|
805
831
|
|
|
806
832
|
// Phase 5: Structural Analysis (code-review-graph, optional)
|
|
@@ -835,9 +861,9 @@ async function analyzeComplexity(payload, options = {}) {
|
|
|
835
861
|
return result;
|
|
836
862
|
}
|
|
837
863
|
|
|
838
|
-
// Standard scoring (original logic)
|
|
839
|
-
const tokenScore = scoreTokens(
|
|
840
|
-
const toolScore = scoreTools(
|
|
864
|
+
// Standard scoring (original logic) - use compressed payload
|
|
865
|
+
const tokenScore = scoreTokens(workingPayload);
|
|
866
|
+
const toolScore = scoreTools(workingPayload);
|
|
841
867
|
const taskTypeResult = scoreTaskType(content);
|
|
842
868
|
const codeComplexityResult = scoreCodeComplexity(content);
|
|
843
869
|
const reasoningResult = scoreReasoning(content);
|
|
@@ -875,8 +901,8 @@ async function analyzeComplexity(payload, options = {}) {
|
|
|
875
901
|
mode,
|
|
876
902
|
recommendation,
|
|
877
903
|
breakdown: {
|
|
878
|
-
tokens: { score: tokenScore, estimated: estimateTokens(
|
|
879
|
-
tools: { score: toolScore, count: _effectiveTools(
|
|
904
|
+
tokens: { score: tokenScore, estimated: estimateTokens(workingPayload) },
|
|
905
|
+
tools: { score: toolScore, count: _effectiveTools(workingPayload).length },
|
|
880
906
|
taskType: taskTypeResult,
|
|
881
907
|
codeComplexity: codeComplexityResult,
|
|
882
908
|
reasoning: reasoningResult,
|
|
@@ -884,11 +910,12 @@ async function analyzeComplexity(payload, options = {}) {
|
|
|
884
910
|
},
|
|
885
911
|
content: content.slice(0, 100) + (content.length > 100 ? '...' : ''),
|
|
886
912
|
graphSignals: null,
|
|
913
|
+
compression: compressionStats, // Add compression stats
|
|
887
914
|
};
|
|
888
915
|
|
|
889
916
|
// Phase 5: Structural Analysis (code-review-graph, optional)
|
|
890
917
|
try {
|
|
891
|
-
const filePaths = extractFilePaths(
|
|
918
|
+
const filePaths = extractFilePaths(workingPayload);
|
|
892
919
|
const graphOpts = { filePaths, workspace: options?.workspace };
|
|
893
920
|
const graphAvailable = await codeGraph.isAvailable(graphOpts);
|
|
894
921
|
if (graphAvailable && filePaths.length > 0) {
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context Compression Module
|
|
3
|
+
*
|
|
4
|
+
* Inspired by TencentDB Agent Memory's short-term memory compression.
|
|
5
|
+
* Reduces conversation context size before complexity scoring to enable
|
|
6
|
+
* more accurate tier routing.
|
|
7
|
+
*
|
|
8
|
+
* Key insight: Long conversations bloat context → force expensive routing
|
|
9
|
+
* even for simple follow-ups. By compressing tool outputs and previous
|
|
10
|
+
* turns, we keep more requests in SIMPLE/MEDIUM tiers.
|
|
11
|
+
*
|
|
12
|
+
* @module routing/context-compressor
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const logger = require('../logger');
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Compress conversation messages by:
|
|
19
|
+
* 1. Offloading verbose tool_result content
|
|
20
|
+
* 2. Summarizing repetitive patterns
|
|
21
|
+
* 3. Keeping only essential context for scoring
|
|
22
|
+
*
|
|
23
|
+
* Does NOT modify the actual request sent to LLM — only used for complexity scoring.
|
|
24
|
+
*
|
|
25
|
+
* @param {Array} messages - Conversation messages
|
|
26
|
+
* @returns {{ compressed: Array, stats: Object }} - Compressed messages + stats
|
|
27
|
+
*/
|
|
28
|
+
function compressMessages(messages) {
|
|
29
|
+
if (!Array.isArray(messages) || messages.length === 0) {
|
|
30
|
+
return { compressed: messages, stats: { original: 0, compressed: 0, ratio: 1.0 } };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const compressed = [];
|
|
34
|
+
let originalSize = 0;
|
|
35
|
+
let compressedSize = 0;
|
|
36
|
+
let toolResultsOffloaded = 0;
|
|
37
|
+
let messagesKept = 0;
|
|
38
|
+
|
|
39
|
+
// Keep system message always
|
|
40
|
+
const systemMsg = messages.find(m => m.role === 'system');
|
|
41
|
+
if (systemMsg) {
|
|
42
|
+
compressed.push(systemMsg);
|
|
43
|
+
const size = estimateMessageSize(systemMsg);
|
|
44
|
+
originalSize += size;
|
|
45
|
+
compressedSize += size;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Keep last N user/assistant turns (sliding window)
|
|
49
|
+
const WINDOW_SIZE = 5;
|
|
50
|
+
const recentTurns = messages
|
|
51
|
+
.filter(m => m.role === 'user' || m.role === 'assistant')
|
|
52
|
+
.slice(-WINDOW_SIZE);
|
|
53
|
+
|
|
54
|
+
for (const msg of recentTurns) {
|
|
55
|
+
const original = estimateMessageSize(msg);
|
|
56
|
+
originalSize += original;
|
|
57
|
+
|
|
58
|
+
// Compress tool_result blocks (biggest token consumers)
|
|
59
|
+
if (Array.isArray(msg.content)) {
|
|
60
|
+
const compressedContent = msg.content.map(block => {
|
|
61
|
+
if (block?.type === 'tool_result') {
|
|
62
|
+
toolResultsOffloaded++;
|
|
63
|
+
// Keep only metadata, offload actual content
|
|
64
|
+
return {
|
|
65
|
+
type: 'tool_result',
|
|
66
|
+
tool_use_id: block.tool_use_id,
|
|
67
|
+
content: '[offloaded]', // Replaced with placeholder
|
|
68
|
+
_compressed: true,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
return block;
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const compressedMsg = { ...msg, content: compressedContent };
|
|
75
|
+
compressed.push(compressedMsg);
|
|
76
|
+
compressedSize += estimateMessageSize(compressedMsg);
|
|
77
|
+
messagesKept++;
|
|
78
|
+
} else {
|
|
79
|
+
// No compression needed for string content
|
|
80
|
+
compressed.push(msg);
|
|
81
|
+
compressedSize += original;
|
|
82
|
+
messagesKept++;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const ratio = originalSize > 0 ? compressedSize / originalSize : 1.0;
|
|
87
|
+
|
|
88
|
+
const stats = {
|
|
89
|
+
original: originalSize,
|
|
90
|
+
compressed: compressedSize,
|
|
91
|
+
ratio,
|
|
92
|
+
reduction: Math.round((1 - ratio) * 100),
|
|
93
|
+
toolResultsOffloaded,
|
|
94
|
+
messagesKept,
|
|
95
|
+
messagesDropped: messages.length - messagesKept - (systemMsg ? 1 : 0),
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
logger.debug({
|
|
99
|
+
...stats,
|
|
100
|
+
originalMsgs: messages.length,
|
|
101
|
+
compressedMsgs: compressed.length,
|
|
102
|
+
}, '[context-compressor] Compression complete');
|
|
103
|
+
|
|
104
|
+
return { compressed, stats };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Estimate message size in tokens (rough heuristic: chars / 4)
|
|
109
|
+
*/
|
|
110
|
+
function estimateMessageSize(msg) {
|
|
111
|
+
if (!msg) return 0;
|
|
112
|
+
|
|
113
|
+
let size = 0;
|
|
114
|
+
|
|
115
|
+
// Role + metadata overhead
|
|
116
|
+
size += 10;
|
|
117
|
+
|
|
118
|
+
// Content
|
|
119
|
+
if (typeof msg.content === 'string') {
|
|
120
|
+
size += Math.ceil(msg.content.length / 4);
|
|
121
|
+
} else if (Array.isArray(msg.content)) {
|
|
122
|
+
for (const block of msg.content) {
|
|
123
|
+
if (block?.type === 'text' && block.text) {
|
|
124
|
+
size += Math.ceil(block.text.length / 4);
|
|
125
|
+
} else if (block?.type === 'tool_result') {
|
|
126
|
+
// Tool results are verbose (code, logs, errors)
|
|
127
|
+
const content = Array.isArray(block.content)
|
|
128
|
+
? block.content.map(c => c?.text || '').join('')
|
|
129
|
+
: (block.content || '');
|
|
130
|
+
size += Math.ceil(content.length / 4);
|
|
131
|
+
} else if (block?.type === 'tool_use') {
|
|
132
|
+
// Tool use is compact (function name + args)
|
|
133
|
+
size += 50;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return size;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Check if compression is beneficial for this payload.
|
|
143
|
+
* Only compress if conversation is long enough to matter.
|
|
144
|
+
*/
|
|
145
|
+
function shouldCompress(payload) {
|
|
146
|
+
if (!payload?.messages || !Array.isArray(payload.messages)) {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const msgCount = payload.messages.length;
|
|
151
|
+
|
|
152
|
+
// Count total tool_result blocks (not just messages with tool results)
|
|
153
|
+
let toolResultCount = 0;
|
|
154
|
+
for (const msg of payload.messages) {
|
|
155
|
+
if (Array.isArray(msg.content)) {
|
|
156
|
+
toolResultCount += msg.content.filter(c => c?.type === 'tool_result').length;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Compress if:
|
|
161
|
+
// 1. More than 10 messages (long conversation), OR
|
|
162
|
+
// 2. More than 3 tool_result blocks (verbose outputs)
|
|
163
|
+
return msgCount > 10 || toolResultCount > 3;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
module.exports = {
|
|
167
|
+
compressMessages,
|
|
168
|
+
shouldCompress,
|
|
169
|
+
estimateMessageSize,
|
|
170
|
+
};
|
|
@@ -7,14 +7,16 @@
|
|
|
7
7
|
* hard problems. An LLM reading the actual sentence knows better.
|
|
8
8
|
*
|
|
9
9
|
* DESIGN:
|
|
10
|
-
* -
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* - Dedicated classifier model (CLASSIFIER_MODEL below), deliberately
|
|
11
|
+
* decoupled from tier serving — swapping TIER_* env vars does not change
|
|
12
|
+
* what classifies prompts. The tier DEPLOYMENT (which models the tiers
|
|
13
|
+
* route to) is surfaced to the classifier as tie-breaker context; see
|
|
14
|
+
* _buildTierContext.
|
|
13
15
|
* - Structured JSON output; parse failure → null → caller falls back.
|
|
14
|
-
* - Hard
|
|
16
|
+
* - Hard timeout (TIMEOUT_MS); on timeout → null → caller falls back.
|
|
15
17
|
* - LRU cache keyed by sha256(text.trim().toLowerCase()); capacity 500.
|
|
16
18
|
* - Skip conditions surface via classifyDifficulty returning null with
|
|
17
|
-
* reason=skipped: text
|
|
19
|
+
* reason=skipped: empty text, force-pattern matched, risk=high,
|
|
18
20
|
* cache hit is transparent (returns cached).
|
|
19
21
|
* - Hardcoded kill-switch CLASSIFIER_ENABLED — no env var per user policy.
|
|
20
22
|
*
|
|
@@ -31,7 +33,6 @@ const CLASSIFIER_ENABLED = true;
|
|
|
31
33
|
// LRU cache to keep amortized latency low.
|
|
32
34
|
const TIMEOUT_MS = 10000;
|
|
33
35
|
const CACHE_CAPACITY = 500;
|
|
34
|
-
const MIN_TEXT_LENGTH = 15;
|
|
35
36
|
|
|
36
37
|
// Classifier model — decoupled from tier serving so SIMPLE tier can run a
|
|
37
38
|
// more capable model for real traffic while the classifier stays fast and
|
|
@@ -44,7 +45,7 @@ const CLASSIFIER_MODEL = 'qwen2.5:3b';
|
|
|
44
45
|
|
|
45
46
|
const VALID_TIERS = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
|
|
46
47
|
|
|
47
|
-
// One-shot classification prompt (
|
|
48
|
+
// One-shot classification prompt (v3). Kept in a const so drift is diffable.
|
|
48
49
|
// Difficulty framing (not intent) — matches config B routing goals.
|
|
49
50
|
//
|
|
50
51
|
// v2 (2026-07-21): added the follow-up rule, negative examples under
|
|
@@ -54,11 +55,16 @@ const VALID_TIERS = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
|
|
|
54
55
|
// flavored and nothing said surface vocabulary isn't the signal. Baseline
|
|
55
56
|
// on data/difficulty-eval-followups.jsonl: 60% overall, 33% on SIMPLE,
|
|
56
57
|
// 3 SIMPLE→REASONING criticals.
|
|
58
|
+
//
|
|
59
|
+
// v3 (2026-08-09): added trivial-arithmetic SIMPLE examples after a live
|
|
60
|
+
// over-route: "12+21" scored anchor 25 (MEDIUM band) and qwen ALSO said
|
|
61
|
+
// MEDIUM conf 1.0 — bare arithmetic read as "a specific mechanical task".
|
|
62
|
+
// Any tiny model adds two numbers; it belongs in SIMPLE.
|
|
57
63
|
const CLASSIFY_PROMPT = `You are a classifier for an LLM routing proxy. Classify the difficulty of the CURRENT user prompt into exactly one of four tiers. Reply with ONLY valid JSON on a single line, no other text.
|
|
58
64
|
|
|
59
65
|
Tiers:
|
|
60
66
|
- SIMPLE: casual acknowledgments, greetings, one-word answers, trivial factual lookups, and short conversational follow-up questions about people, stories, events, or everyday facts. Any tiny model handles.
|
|
61
|
-
examples: "hi", "ok thanks", "yes continue", "what time is it", "who is doctor doom?", "who kills him?", "why did he do that?", "and then what happened?", "does bleach kill mold?"
|
|
67
|
+
examples: "hi", "ok thanks", "yes continue", "what time is it", "who is doctor doom?", "who kills him?", "why did he do that?", "and then what happened?", "does bleach kill mold?", "12+21", "what is 15% of 80?", "convert 3km to miles"
|
|
62
68
|
- MEDIUM: one specific mechanical task or a focused explanation. Mid-size local model suffices.
|
|
63
69
|
examples: "list the exports from this file", "run the unit tests", "fix the linter warnings", "explain this regex", "add error handling to this block", "verify the file exists before reading it"
|
|
64
70
|
- COMPLEX: multi-file design, systemic refactor, architecture review, debugging that requires broad code understanding. Needs a strong general model.
|
|
@@ -81,15 +87,101 @@ Reply format (strict): {"tier":"SIMPLE|MEDIUM|COMPLEX|REASONING","confidence":0.
|
|
|
81
87
|
const CONTEXT_MAX_TEXT_LENGTH = 40;
|
|
82
88
|
const CONTEXT_MAX_CHARS = 300;
|
|
83
89
|
|
|
90
|
+
// --- Tier deployment context -------------------------------------------------
|
|
91
|
+
//
|
|
92
|
+
// Tells the classifier WHAT each tier routes to (model + coarse traits from
|
|
93
|
+
// the registry), so borderline verdicts can weigh the actual fleet — e.g.
|
|
94
|
+
// prefer MEDIUM when its configured model is a large cloud model, or lean
|
|
95
|
+
// away from a tier whose model lacks the needed capability.
|
|
96
|
+
//
|
|
97
|
+
// Constraints that shape the design:
|
|
98
|
+
// - TIE-BREAKER ONLY. The difficulty rubric in CLASSIFY_PROMPT governs;
|
|
99
|
+
// the deployment block must not redefine what the tiers mean, or eval
|
|
100
|
+
// labels (data/difficulty-eval*.jsonl) stop being comparable.
|
|
101
|
+
// - BYTE-STABLE per process. Tier config is env-driven and fixed for the
|
|
102
|
+
// process lifetime, so the block is built once and memoized. It also
|
|
103
|
+
// participates in the LRU cache key (fingerprint) so a reconfigured
|
|
104
|
+
// deployment never serves verdicts cached under the old one.
|
|
105
|
+
// - FAIL-SAFE. Registry misses or config errors degrade to provider-type
|
|
106
|
+
// traits or to no block at all — classification proceeds regardless.
|
|
107
|
+
// MEASURED OFF (2026-08-08): A/B on data/difficulty-eval-followups.jsonl —
|
|
108
|
+
// baseline 84.0% overall / 0 SIMPLE→REASONING criticals; with the block
|
|
109
|
+
// 76.0% / 1 critical ('prove me wrong lol' → REASONING conf 1.0, the exact
|
|
110
|
+
// failure class prompt v2 was built to kill). qwen2.5:3b can't use fleet
|
|
111
|
+
// info — it dilutes the rubric. Machinery kept dark for a future stronger
|
|
112
|
+
// classifier: flip this flag, then re-run
|
|
113
|
+
// scripts/validate-difficulty-classifier.js on BOTH eval files and hold
|
|
114
|
+
// the ship bar (>=85% overall, zero MEDIUM→REASONING false positives).
|
|
115
|
+
const TIER_CONTEXT_ENABLED = false;
|
|
116
|
+
const LOCAL_PROVIDERS = new Set(['ollama', 'llamacpp', 'lmstudio']);
|
|
117
|
+
|
|
118
|
+
/** Pure builder — exported for tests. @param {Object} tiers {TIER: 'provider:model'} */
|
|
119
|
+
function _buildTierContext(tiers) {
|
|
120
|
+
const lines = [];
|
|
121
|
+
for (const tier of VALID_TIERS) {
|
|
122
|
+
const spec = tiers?.[tier];
|
|
123
|
+
if (typeof spec !== 'string' || !spec.includes(':')) continue;
|
|
124
|
+
const sep = spec.indexOf(':');
|
|
125
|
+
const provider = spec.slice(0, sep);
|
|
126
|
+
const model = spec.slice(sep + 1);
|
|
127
|
+
const traits = [LOCAL_PROVIDERS.has(provider) ? 'local' : 'cloud'];
|
|
128
|
+
try {
|
|
129
|
+
const { getModelRegistrySync } = require('./model-registry');
|
|
130
|
+
const cost = getModelRegistrySync().getCost(model);
|
|
131
|
+
if (cost && !cost.unknown) {
|
|
132
|
+
if (Number.isFinite(cost.context) && cost.context >= 1000) {
|
|
133
|
+
traits.push(`${Math.round(cost.context / 1000)}k ctx`);
|
|
134
|
+
}
|
|
135
|
+
if (cost.reasoning) traits.push('reasoning');
|
|
136
|
+
if (cost.vision) traits.push('vision');
|
|
137
|
+
}
|
|
138
|
+
} catch { /* registry unavailable — provider trait only */ }
|
|
139
|
+
lines.push(`- ${tier} routes to ${model} (${traits.join(', ')})`);
|
|
140
|
+
}
|
|
141
|
+
if (!lines.length) return null;
|
|
142
|
+
const block = `Deployment (what each tier currently routes to — use only as a tie-breaker between adjacent tiers; the tier definitions above still govern):
|
|
143
|
+
${lines.join('\n')}`;
|
|
144
|
+
return {
|
|
145
|
+
block,
|
|
146
|
+
fingerprint: crypto.createHash('sha256').update(block).digest('hex').slice(0, 16),
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** @type {{block:string, fingerprint:string}|null|undefined} undefined = not built yet */
|
|
151
|
+
let _tierContext;
|
|
152
|
+
|
|
153
|
+
function _tierDeploymentContext() {
|
|
154
|
+
// Test override (set via _setTierContextForTests) wins over the flag so
|
|
155
|
+
// the machinery stays exercised while shipping dark.
|
|
156
|
+
if (_tierContext !== undefined) return _tierContext;
|
|
157
|
+
if (!TIER_CONTEXT_ENABLED) { _tierContext = null; return null; }
|
|
158
|
+
try {
|
|
159
|
+
const config = require('../config');
|
|
160
|
+
_tierContext = _buildTierContext(config.modelTiers);
|
|
161
|
+
} catch {
|
|
162
|
+
_tierContext = null;
|
|
163
|
+
}
|
|
164
|
+
return _tierContext;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** Test helper — override or reset (pass undefined) the memoized block. */
|
|
168
|
+
function _setTierContextForTests(tiers) {
|
|
169
|
+
_tierContext = tiers === undefined ? undefined : _buildTierContext(tiers);
|
|
170
|
+
}
|
|
171
|
+
|
|
84
172
|
function _buildPrompt(text, context) {
|
|
173
|
+
const tierCtx = _tierDeploymentContext();
|
|
174
|
+
const preamble = tierCtx ? `${CLASSIFY_PROMPT}
|
|
175
|
+
${tierCtx.block}
|
|
176
|
+
` : CLASSIFY_PROMPT;
|
|
85
177
|
if (context) {
|
|
86
|
-
return `${
|
|
178
|
+
return `${preamble}
|
|
87
179
|
Conversation so far (context only — classify the CURRENT prompt, inheriting topic difficulty per the rules):
|
|
88
180
|
${context}
|
|
89
181
|
|
|
90
182
|
CURRENT user prompt: """${text}"""`;
|
|
91
183
|
}
|
|
92
|
-
return `${
|
|
184
|
+
return `${preamble}
|
|
93
185
|
User prompt: """${text}"""`;
|
|
94
186
|
}
|
|
95
187
|
|
|
@@ -212,7 +304,12 @@ async function classifyDifficulty(text, opts = {}) {
|
|
|
212
304
|
if (!CLASSIFIER_ENABLED) return null;
|
|
213
305
|
if (typeof text !== 'string') return null;
|
|
214
306
|
const trimmed = text.trim();
|
|
215
|
-
|
|
307
|
+
// No minimum length (removed 2026-08-09): short prompts are exactly what
|
|
308
|
+
// the v2 context feature was built for, and the 15-char gate meant the
|
|
309
|
+
// classifier could never rescue anchor over-reads like '12+21' -> MEDIUM.
|
|
310
|
+
// Greetings still skip upstream via force patterns (opts.forceMatched),
|
|
311
|
+
// and the LRU absorbs repeats, so the added model calls are bounded.
|
|
312
|
+
if (!trimmed) return null;
|
|
216
313
|
if (opts.forceMatched) return null;
|
|
217
314
|
if (opts.riskLevel === 'high') return null;
|
|
218
315
|
|
|
@@ -222,8 +319,11 @@ async function classifyDifficulty(text, opts = {}) {
|
|
|
222
319
|
: null;
|
|
223
320
|
|
|
224
321
|
// Context participates in the cache key: the same follow-up text means
|
|
225
|
-
// different things in different conversations.
|
|
226
|
-
|
|
322
|
+
// different things in different conversations. The tier-deployment
|
|
323
|
+
// fingerprint participates too (NUL separators are collision-proof), so
|
|
324
|
+
// verdicts cached under one deployment never serve a reconfigured one.
|
|
325
|
+
const tierFp = _tierDeploymentContext()?.fingerprint ?? "";
|
|
326
|
+
const key = _cacheKey(`${tierFp}${context ? `${trimmed}${context}` : trimmed}`);
|
|
227
327
|
const cached = _cache.get(key);
|
|
228
328
|
if (cached) return { ...cached, source: 'cache' };
|
|
229
329
|
|
|
@@ -256,6 +356,8 @@ module.exports = {
|
|
|
256
356
|
_parseResult,
|
|
257
357
|
_cacheKey,
|
|
258
358
|
_buildPrompt,
|
|
359
|
+
_buildTierContext,
|
|
360
|
+
_setTierContextForTests,
|
|
259
361
|
_clearCacheForTests,
|
|
260
362
|
_getCacheStats,
|
|
261
363
|
};
|