adaptive-memory-multi-model-router 2.2.5 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -22
- package/README.md.bak +836 -0
- package/dist/analytics/costAnalytics.d.ts +1 -0
- package/dist/cache/cacheKeyGenerator.d.ts +67 -0
- package/dist/cache/cacheKeyGenerator.d.ts.map +1 -0
- package/dist/cache/cacheKeyGenerator.js +211 -0
- package/dist/cache/cacheKeyGenerator.js.map +1 -0
- package/dist/cache/semanticCache.d.ts +41 -0
- package/dist/cache/semanticCache.d.ts.map +1 -1
- package/dist/cache/semanticCache.js +142 -0
- package/dist/cache/semanticCache.js.map +1 -1
- package/dist/cli.js +35 -478
- package/dist/cost/costTracker.js +0 -3
- package/dist/cost/preCallCostEstimator.d.ts +114 -0
- package/dist/cost/preCallCostEstimator.d.ts.map +1 -0
- package/dist/cost/preCallCostEstimator.js +256 -0
- package/dist/cost/preCallCostEstimator.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +264 -64
- package/dist/index.js.map +1 -1
- package/dist/inference/speculativeDecoding.d.ts +133 -0
- package/dist/inference/speculativeDecoding.d.ts.map +1 -0
- package/dist/inference/speculativeDecoding.js +276 -0
- package/dist/inference/speculativeDecoding.js.map +1 -0
- package/dist/integrations/langchainAdapter.d.ts +1 -0
- package/dist/integrations/oauth.d.ts +1 -0
- package/dist/memory/autoFetch.d.ts +1 -0
- package/dist/memory/memoryTree.d.ts +1 -0
- package/dist/memory/obsidianVault.d.ts +1 -0
- package/dist/providers/providerConfig.d.ts +1 -0
- package/dist/providers/providerConfig.js +2 -0
- package/dist/providers/providerHealth.d.ts +117 -0
- package/dist/providers/providerHealth.d.ts.map +1 -0
- package/dist/providers/providerHealth.js +309 -0
- package/dist/providers/providerHealth.js.map +1 -0
- package/dist/providers/registry.js +126 -128
- package/dist/routing/advancedRouter.js +310 -427
- package/dist/routing/difficultyClassifier.d.ts +79 -0
- package/dist/routing/difficultyClassifier.d.ts.map +1 -0
- package/dist/routing/difficultyClassifier.js +329 -0
- package/dist/routing/difficultyClassifier.js.map +1 -0
- package/dist/sdk.d.ts +125 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +109 -100
- package/dist/sdk.js.map +1 -0
- package/dist/security/guardrails.d.ts +1 -0
- package/dist/server/dashboard.d.ts +1 -0
- package/dist/server/modelMapper.d.ts +1 -0
- package/dist/server/proxyServer.d.ts +1 -0
- package/package.json +4 -2
- package/src/cache/cacheKeyGenerator.ts +242 -0
- package/src/cache/semanticCache.ts +148 -0
- package/src/cost/preCallCostEstimator.ts +345 -0
- package/src/inference/speculativeDecoding.ts +373 -0
- package/src/providers/providerHealth.ts +397 -0
- package/src/routing/difficultyClassifier.ts +420 -0
- package/test/provider-test.js +2 -2
- package/test.js +7 -7
- package/test.js.bak +376 -0
- package/tsconfig.json +15 -5
- package/src/index.ts +0 -99
- package/src/skills/__tests__/skill_manager.test.ts +0 -328
|
@@ -1,453 +1,336 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* -
|
|
7
|
-
* - Provider availability (checks API keys)
|
|
8
|
-
* - Cost optimization
|
|
9
|
-
* - Quality vs speed tradeoff
|
|
10
|
-
*
|
|
11
|
-
* All provider references are dynamically loaded from providerConfig.
|
|
12
|
-
* Users can add/remove providers via environment variables or config files.
|
|
3
|
+
* TMLPD Advanced Routing - RouteLLM Style
|
|
4
|
+
*
|
|
5
|
+
* Learned routing based on arXiv:2404.06035 (RouteLLM)
|
|
6
|
+
* Balances cost-quality tradeoff with confidence-based model selection
|
|
13
7
|
*/
|
|
14
|
-
|
|
15
|
-
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.MODEL_PROFILES = void 0;
|
|
10
|
+
exports.extractQueryFeatures = extractQueryFeatures;
|
|
11
|
+
exports.routeQuery = routeQuery;
|
|
12
|
+
exports.routeBatch = routeBatch;
|
|
13
|
+
exports.recommendForTask = recommendForTask;
|
|
14
|
+
exports.updateModelProfile = updateModelProfile;
|
|
16
15
|
const tokenUtils_1 = require("../utils/tokenUtils");
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
const modelKey = model.includes('/') ? model : providerId + '/' + model;
|
|
29
|
-
const costPerKInput = provider.costPerK ? provider.costPerK.input : 0;
|
|
30
|
-
const costPerKOutput = provider.costPerK ? provider.costPerK.output : 0;
|
|
31
|
-
|
|
32
|
-
// Assign strengths based on model characteristics
|
|
33
|
-
const strengths = [];
|
|
34
|
-
if (provider.type === 'cli') {
|
|
35
|
-
strengths.push('free', 'local');
|
|
36
|
-
}
|
|
37
|
-
if (costPerKInput < 0.3) {
|
|
38
|
-
strengths.push('budget', 'fast');
|
|
39
|
-
} else if (costPerKInput > 2) {
|
|
40
|
-
strengths.push('premium', 'reasoning');
|
|
41
|
-
}
|
|
42
|
-
if (provider.name === 'Mistral' || provider.name === 'Groq' || provider.name === 'Cerebras') {
|
|
43
|
-
strengths.push('fast', 'coding');
|
|
44
|
-
}
|
|
45
|
-
if (provider.name === 'CommandCode') {
|
|
46
|
-
strengths.push('code-aware', 'context-rich');
|
|
47
|
-
}
|
|
48
|
-
if (provider.name === 'OpenCode') {
|
|
49
|
-
strengths.push('free', 'multi-model');
|
|
50
|
-
}
|
|
51
|
-
if (provider.name === 'Google') {
|
|
52
|
-
strengths.push('multilingual', 'long-context');
|
|
53
|
-
}
|
|
54
|
-
if (provider.name === 'OpenAI') {
|
|
55
|
-
strengths.push('reasoning', 'coding', 'analysis');
|
|
56
|
-
}
|
|
57
|
-
if (provider.name === 'Anthropic') {
|
|
58
|
-
strengths.push('reasoning', 'creative', 'analysis');
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
profiles[modelKey] = {
|
|
62
|
-
name: modelKey,
|
|
63
|
-
provider: providerId,
|
|
64
|
-
providerName: provider.name,
|
|
65
|
-
cost_per_1k_input: costPerKInput,
|
|
66
|
-
cost_per_1k_output: costPerKOutput,
|
|
67
|
-
latency_ms: provider.type === 'cli' ? 5000 : (provider.priority * 200 + 300),
|
|
68
|
-
quality_score: strengths.includes('premium') ? 0.95 :
|
|
69
|
-
strengths.includes('reasoning') ? 0.90 :
|
|
70
|
-
strengths.includes('fast') ? 0.82 : 0.80,
|
|
71
|
-
strengths,
|
|
72
|
-
context_window: provider.maxTokens || 8192,
|
|
73
|
-
type: provider.type,
|
|
74
|
-
priority: provider.priority,
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return profiles;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
let MODEL_PROFILES = buildModelProfiles();
|
|
83
|
-
|
|
84
|
-
// Refresh profiles when providers change
|
|
85
|
-
function refreshModelProfiles() {
|
|
86
|
-
MODEL_PROFILES = buildModelProfiles();
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
exports.MODEL_PROFILES = MODEL_PROFILES;
|
|
90
|
-
|
|
91
|
-
// ============================================================
|
|
92
|
-
// FEATURE EXTRACTION (v3 — multi-signal complexity scorer)
|
|
93
|
-
// ============================================================
|
|
94
|
-
|
|
95
|
-
function extractQueryFeatures(prompt) {
|
|
96
|
-
const lower = prompt.toLowerCase();
|
|
97
|
-
const words = prompt.split(/\s+/);
|
|
98
|
-
const wordCount = words.length;
|
|
99
|
-
|
|
100
|
-
// === SIGNAL 1: Domain Detection ===
|
|
101
|
-
// Professional domains that indicate expert-level queries
|
|
102
|
-
const domainSignals = {
|
|
103
|
-
legal: {
|
|
104
|
-
keywords: ['legal', 'law', 'contract', 'liability', 'litigation', 'patent', 'copyright',
|
|
105
|
-
'regulation', 'compliance', 'constitutional', 'statute', 'jurisdiction',
|
|
106
|
-
'court', 'ruling', 'precedent', 'attorney', 'amicus', 'sec ', 'fda ',
|
|
107
|
-
'gdpr', 'ccpa', 'cfpr', 'due diligence', 'merger', 'acquisition',
|
|
108
|
-
'10-k', 'sec filing', 'forensic', 'embezzlement', 'infringement'],
|
|
109
|
-
weight: 0.35
|
|
16
|
+
// Pre-configured model profiles
|
|
17
|
+
exports.MODEL_PROFILES = {
|
|
18
|
+
"openai/gpt-4o": {
|
|
19
|
+
name: "openai/gpt-4o",
|
|
20
|
+
provider: "openai",
|
|
21
|
+
cost_per_1k_input: 2.50,
|
|
22
|
+
cost_per_1k_output: 10.00,
|
|
23
|
+
latency_ms: 2000,
|
|
24
|
+
quality_score: 0.95,
|
|
25
|
+
strengths: ["reasoning", "coding", "analysis"],
|
|
26
|
+
context_window: 128000
|
|
110
27
|
},
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
28
|
+
"openai/gpt-4o-mini": {
|
|
29
|
+
name: "openai/gpt-4o-mini",
|
|
30
|
+
provider: "openai",
|
|
31
|
+
cost_per_1k_input: 0.15,
|
|
32
|
+
cost_per_1k_output: 0.60,
|
|
33
|
+
latency_ms: 500,
|
|
34
|
+
quality_score: 0.85,
|
|
35
|
+
strengths: ["fast", "coding"],
|
|
36
|
+
context_window: 128000
|
|
117
37
|
},
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
38
|
+
"anthropic/claude-3.5-sonnet": {
|
|
39
|
+
name: "anthropic/claude-3.5-sonnet",
|
|
40
|
+
provider: "anthropic",
|
|
41
|
+
cost_per_1k_input: 3.00,
|
|
42
|
+
cost_per_1k_output: 15.00,
|
|
43
|
+
latency_ms: 2500,
|
|
44
|
+
quality_score: 0.96,
|
|
45
|
+
strengths: ["reasoning", "creative", "analysis"],
|
|
46
|
+
context_window: 200000
|
|
125
47
|
},
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
48
|
+
"anthropic/claude-3-haiku": {
|
|
49
|
+
name: "anthropic/claude-3-haiku",
|
|
50
|
+
provider: "anthropic",
|
|
51
|
+
cost_per_1k_input: 0.25,
|
|
52
|
+
cost_per_1k_output: 1.25,
|
|
53
|
+
latency_ms: 500,
|
|
54
|
+
quality_score: 0.80,
|
|
55
|
+
strengths: ["fast", "simple"],
|
|
56
|
+
context_window: 200000
|
|
133
57
|
},
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
58
|
+
"google/gemini-2.0-flash": {
|
|
59
|
+
name: "google/gemini-2.0-flash",
|
|
60
|
+
provider: "google",
|
|
61
|
+
cost_per_1k_input: 0.00, // Free
|
|
62
|
+
cost_per_1k_output: 0.00,
|
|
63
|
+
latency_ms: 800,
|
|
64
|
+
quality_score: 0.88,
|
|
65
|
+
strengths: ["fast", "multilingual"],
|
|
66
|
+
context_window: 1000000
|
|
141
67
|
},
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
68
|
+
"google/gemini-1.5-pro": {
|
|
69
|
+
name: "google/gemini-1.5-pro",
|
|
70
|
+
provider: "google",
|
|
71
|
+
cost_per_1k_input: 1.25,
|
|
72
|
+
cost_per_1k_output: 5.00,
|
|
73
|
+
latency_ms: 1500,
|
|
74
|
+
quality_score: 0.92,
|
|
75
|
+
strengths: ["reasoning", "long-context"],
|
|
76
|
+
context_window: 2000000
|
|
77
|
+
},
|
|
78
|
+
"groq/llama-3.3-70b": {
|
|
79
|
+
name: "groq/llama-3.3-70b",
|
|
80
|
+
provider: "groq",
|
|
81
|
+
cost_per_1k_input: 0.59,
|
|
82
|
+
cost_per_1k_output: 0.79,
|
|
83
|
+
latency_ms: 400,
|
|
84
|
+
quality_score: 0.82,
|
|
85
|
+
strengths: ["fast", "coding"],
|
|
86
|
+
context_window: 128000
|
|
87
|
+
},
|
|
88
|
+
"cerebras/llama-3.3-70b": {
|
|
89
|
+
name: "cerebras/llama-3.3-70b",
|
|
90
|
+
provider: "cerebras",
|
|
91
|
+
cost_per_1k_input: 0.60,
|
|
92
|
+
cost_per_1k_output: 0.60,
|
|
93
|
+
latency_ms: 350,
|
|
94
|
+
quality_score: 0.82,
|
|
95
|
+
strengths: ["fast", "budget"],
|
|
96
|
+
context_window: 128000
|
|
97
|
+
},
|
|
98
|
+
"local/llama-3.3-70b": {
|
|
99
|
+
name: "local/llama-3.3-70b",
|
|
100
|
+
provider: "ollama",
|
|
101
|
+
cost_per_1k_input: 0.00,
|
|
102
|
+
cost_per_1k_output: 0.00,
|
|
103
|
+
latency_ms: 100,
|
|
104
|
+
quality_score: 0.75,
|
|
105
|
+
strengths: ["privacy", "free"],
|
|
106
|
+
context_window: 128000
|
|
162
107
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
return {
|
|
230
|
-
complexity,
|
|
231
|
-
length: wordCount,
|
|
232
|
-
has_code,
|
|
233
|
-
has_math,
|
|
234
|
-
is_multilingual,
|
|
235
|
-
is_translation,
|
|
236
|
-
is_creative,
|
|
237
|
-
requires_reasoning,
|
|
238
|
-
is_security: /security|vulnerability|inject|exploit|attack|encryption|auth/i.test(lower),
|
|
239
|
-
is_devops: /ci\/cd|docker|kubernetes|k8s|deploy|pipeline|github action|terraform/i.test(lower),
|
|
240
|
-
is_data: /dataset|pandas|numpy|training|model|neural|transformer|bert|llm/i.test(lower),
|
|
241
|
-
detected_domain: detectedDomain,
|
|
242
|
-
domain_score: domainScore,
|
|
243
|
-
};
|
|
108
|
+
};
|
|
109
|
+
/**
|
|
110
|
+
* Extract features from prompt for routing decision
|
|
111
|
+
*/
|
|
112
|
+
function extractQueryFeatures(prompt) {
|
|
113
|
+
const lower = prompt.toLowerCase();
|
|
114
|
+
// Code patterns
|
|
115
|
+
const code_indicators = [
|
|
116
|
+
"function", "class ", "def ", "import ", "const ", "let ",
|
|
117
|
+
"python", "javascript", "typescript", "java", "cpp", "rust",
|
|
118
|
+
"```", "=>", "->", "async", "await"
|
|
119
|
+
];
|
|
120
|
+
const has_code = code_indicators.some(pattern => lower.includes(pattern));
|
|
121
|
+
// Math patterns (expanded for unicode and common notation)
|
|
122
|
+
const math_indicators = [
|
|
123
|
+
"equation", "formula", "calculate", "sqrt", "^", "log",
|
|
124
|
+
"sin", "cos", "tan", "integral", "derivative", "$", "math",
|
|
125
|
+
"∫", "∂", "∑", "∏", "√", "∞", "π", "θ", "β",
|
|
126
|
+
"dx", "dy", "dz", "=", "solver", "compute"
|
|
127
|
+
];
|
|
128
|
+
const has_math = math_indicators.some(pattern => prompt.includes(pattern));
|
|
129
|
+
// Multilingual
|
|
130
|
+
const lang_patterns = [
|
|
131
|
+
/[\u4e00-\u9fff]/, // Chinese
|
|
132
|
+
/[\u3040-\u309f\u30a0-\u30ff]/, // Japanese
|
|
133
|
+
/[\uac00-\ud7af]/, // Korean
|
|
134
|
+
/[а-яА-Я]/, // Russian
|
|
135
|
+
/[áéíóúñ]/ // Spanish accented
|
|
136
|
+
];
|
|
137
|
+
const is_multilingual = lang_patterns.some(pattern => pattern.test(prompt));
|
|
138
|
+
// Creative writing
|
|
139
|
+
const creative_indicators = [
|
|
140
|
+
"write a", "story", "poem", "creative", "imagine",
|
|
141
|
+
"describe", "explain in", "tell me", "narrative"
|
|
142
|
+
];
|
|
143
|
+
const is_creative = creative_indicators.some(pattern => lower.includes(pattern));
|
|
144
|
+
// Reasoning
|
|
145
|
+
const reasoning_indicators = [
|
|
146
|
+
"explain", "why", "because", "therefore", "thus",
|
|
147
|
+
"analyze", "think", "consider", "reason", "logic"
|
|
148
|
+
];
|
|
149
|
+
const requires_reasoning = reasoning_indicators.some(pattern => lower.includes(pattern));
|
|
150
|
+
// Complexity estimation based on length and patterns
|
|
151
|
+
const tokens = (0, tokenUtils_1.countTokens)(prompt, "gpt-4o");
|
|
152
|
+
let complexity = 0.3;
|
|
153
|
+
if (tokens > 1000)
|
|
154
|
+
complexity += 0.2;
|
|
155
|
+
if (has_code)
|
|
156
|
+
complexity += 0.15;
|
|
157
|
+
if (has_math)
|
|
158
|
+
complexity += 0.2;
|
|
159
|
+
if (requires_reasoning)
|
|
160
|
+
complexity += 0.15;
|
|
161
|
+
if (is_creative)
|
|
162
|
+
complexity += 0.1;
|
|
163
|
+
complexity = Math.min(1.0, complexity);
|
|
164
|
+
return {
|
|
165
|
+
complexity,
|
|
166
|
+
length: tokens,
|
|
167
|
+
has_code,
|
|
168
|
+
has_math,
|
|
169
|
+
is_multilingual,
|
|
170
|
+
is_creative,
|
|
171
|
+
requires_reasoning
|
|
172
|
+
};
|
|
244
173
|
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
// ============================================================
|
|
249
|
-
// SCORING FUNCTIONS
|
|
250
|
-
// ============================================================
|
|
251
|
-
|
|
174
|
+
/**
|
|
175
|
+
* Score model fit for query
|
|
176
|
+
*/
|
|
252
177
|
function scoreModelFit(model, features) {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
178
|
+
let score = model.quality_score * 0.4; // Base quality
|
|
179
|
+
// Strengths matching
|
|
180
|
+
if (features.has_code && model.strengths.includes("coding")) {
|
|
181
|
+
score += 0.2;
|
|
182
|
+
}
|
|
183
|
+
if (features.requires_reasoning && model.strengths.includes("reasoning")) {
|
|
184
|
+
score += 0.2;
|
|
185
|
+
}
|
|
186
|
+
if (features.is_creative && model.strengths.includes("creative")) {
|
|
187
|
+
score += 0.15;
|
|
188
|
+
}
|
|
189
|
+
if (features.is_multilingual && model.strengths.includes("multilingual")) {
|
|
190
|
+
score += 0.15;
|
|
191
|
+
}
|
|
192
|
+
if (features.has_math && model.strengths.includes("analysis")) {
|
|
193
|
+
score += 0.15;
|
|
194
|
+
}
|
|
195
|
+
// Speed bonus for simple tasks
|
|
196
|
+
if (features.complexity < 0.4 && model.latency_ms < 1000) {
|
|
197
|
+
score += 0.1;
|
|
198
|
+
}
|
|
199
|
+
return score;
|
|
271
200
|
}
|
|
272
|
-
|
|
201
|
+
/**
|
|
202
|
+
* Cost efficiency score (inverse of normalized cost)
|
|
203
|
+
*/
|
|
273
204
|
function costEfficiency(model, features) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
205
|
+
const avg_cost = (model.cost_per_1k_input + model.cost_per_1k_output) / 2;
|
|
206
|
+
// For simple tasks, prioritize cost efficiency
|
|
207
|
+
if (features.complexity < 0.5) {
|
|
208
|
+
return (1 - Math.min(avg_cost / 10, 1)) * 0.6;
|
|
209
|
+
}
|
|
210
|
+
// For complex tasks, deprioritize cost
|
|
211
|
+
return (1 - Math.min(avg_cost / 10, 1)) * 0.2;
|
|
279
212
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
// ============================================================
|
|
284
|
-
|
|
213
|
+
/**
|
|
214
|
+
* RouteLLM-style learned routing decision
|
|
215
|
+
*/
|
|
285
216
|
function routeQuery(prompt, available_models, budget_multiplier = 1.0) {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
profile,
|
|
302
|
-
quality_score: quality,
|
|
303
|
-
cost_score: cost,
|
|
304
|
-
total_score: quality + cost
|
|
305
|
-
};
|
|
217
|
+
const features = extractQueryFeatures(prompt);
|
|
218
|
+
const candidate_names = available_models || Object.keys(exports.MODEL_PROFILES);
|
|
219
|
+
const candidates = candidate_names
|
|
220
|
+
.filter(name => exports.MODEL_PROFILES[name])
|
|
221
|
+
.map(name => {
|
|
222
|
+
const profile = exports.MODEL_PROFILES[name];
|
|
223
|
+
const quality = scoreModelFit(profile, features);
|
|
224
|
+
const cost = costEfficiency(profile, features);
|
|
225
|
+
return {
|
|
226
|
+
name,
|
|
227
|
+
profile,
|
|
228
|
+
quality_score: quality,
|
|
229
|
+
cost_score: cost,
|
|
230
|
+
total_score: quality + cost
|
|
231
|
+
};
|
|
306
232
|
});
|
|
307
|
-
|
|
308
|
-
|
|
233
|
+
// Sort by total score (quality vs cost tradeoff based on complexity)
|
|
234
|
+
const complexity_bias = features.complexity > 0.6 ? 0.7 : 0.3; // High complexity = quality bias
|
|
235
|
+
candidates.sort((a, b) => {
|
|
236
|
+
const score_a = a.quality_score * complexity_bias + a.cost_score * (1 - complexity_bias);
|
|
237
|
+
const score_b = b.quality_score * complexity_bias + b.cost_score * (1 - complexity_bias);
|
|
238
|
+
return score_b - score_a;
|
|
239
|
+
});
|
|
240
|
+
const primary = candidates[0];
|
|
241
|
+
const secondary = candidates.slice(1, 3);
|
|
242
|
+
// Calculate confidence based on score gap
|
|
243
|
+
let confidence = 0.5;
|
|
244
|
+
if (candidates.length > 1) {
|
|
245
|
+
const gap = primary.total_score - candidates[1].total_score;
|
|
246
|
+
confidence = Math.min(0.95, 0.5 + gap * 2);
|
|
247
|
+
}
|
|
248
|
+
// Build reasoning
|
|
249
|
+
const reasons = [];
|
|
250
|
+
if (features.has_code)
|
|
251
|
+
reasons.push("code detected");
|
|
252
|
+
if (features.requires_reasoning)
|
|
253
|
+
reasons.push("reasoning needed");
|
|
254
|
+
if (features.complexity > 0.6)
|
|
255
|
+
reasons.push("high complexity");
|
|
256
|
+
if (features.is_multilingual)
|
|
257
|
+
reasons.push("multilingual");
|
|
258
|
+
const estimated_tokens = features.length * 1.5; // rough completion estimate
|
|
259
|
+
const estimated_cost = (0, tokenUtils_1.estimateCost)(features.length, estimated_tokens, primary.name);
|
|
309
260
|
return {
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
261
|
+
primary_model: primary.name,
|
|
262
|
+
fallback_models: secondary.map(c => c.name),
|
|
263
|
+
confidence,
|
|
264
|
+
reasoning: `Selected ${primary.profile.provider}/${primary.name.split("/")[1]} for ${reasons.join(", ") || "general query"}`,
|
|
265
|
+
estimated_cost: estimated_cost * budget_multiplier,
|
|
266
|
+
estimated_latency_ms: primary.profile.latency_ms
|
|
316
267
|
};
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
// Sort by total score (quality vs cost tradeoff based on complexity)
|
|
320
|
-
const complexity_bias = features.complexity > 0.6 ? 0.7 : 0.3;
|
|
321
|
-
candidates.sort((a, b) => {
|
|
322
|
-
const score_a = a.quality_score * complexity_bias + a.cost_score * (1 - complexity_bias);
|
|
323
|
-
const score_b = b.quality_score * complexity_bias + b.cost_score * (1 - complexity_bias);
|
|
324
|
-
return score_b - score_a;
|
|
325
|
-
});
|
|
326
|
-
|
|
327
|
-
const primary = candidates[0];
|
|
328
|
-
const secondary = candidates.slice(1, 3);
|
|
329
|
-
|
|
330
|
-
// Calculate confidence based on score gap
|
|
331
|
-
let confidence = 0.5;
|
|
332
|
-
if (candidates.length > 1) {
|
|
333
|
-
const gap = primary.total_score - candidates[1].total_score;
|
|
334
|
-
confidence = Math.min(0.95, 0.5 + gap * 2);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
// Build reasoning
|
|
338
|
-
const reasons = [];
|
|
339
|
-
if (features.has_code) reasons.push("code detected");
|
|
340
|
-
if (features.requires_reasoning) reasons.push("reasoning needed");
|
|
341
|
-
if (features.complexity > 0.6) reasons.push("high complexity");
|
|
342
|
-
if (features.is_multilingual) reasons.push("multilingual");
|
|
343
|
-
if (features.is_translation) reasons.push("translation");
|
|
344
|
-
if (primary.profile.strengths.includes("free")) reasons.push("free tier");
|
|
345
|
-
|
|
346
|
-
const estimated_tokens = features.length * 1.5;
|
|
347
|
-
const estimated_cost = tokenUtils_1.estimateCost(features.length, estimated_tokens, primary.name);
|
|
348
|
-
|
|
349
|
-
return {
|
|
350
|
-
primary_model: primary.name,
|
|
351
|
-
fallback_models: secondary.map(c => c.name),
|
|
352
|
-
confidence,
|
|
353
|
-
reasoning: `Selected ${primary.profile.providerName || primary.profile.provider}/${primary.name} for ${reasons.join(", ") || "general query"}`,
|
|
354
|
-
estimated_cost: estimated_cost * budget_multiplier,
|
|
355
|
-
estimated_latency_ms: primary.profile.latency_ms,
|
|
356
|
-
features,
|
|
357
|
-
provider_type: primary.profile.type,
|
|
358
|
-
};
|
|
359
268
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
}
|
|
388
|
-
});
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
return decisions;
|
|
269
|
+
/**
|
|
270
|
+
* Batch routing for multiple prompts
|
|
271
|
+
*/
|
|
272
|
+
function routeBatch(prompts, options) {
|
|
273
|
+
const decisions = prompts.map(p => routeQuery(p));
|
|
274
|
+
if (options?.same_model && decisions.length > 0) {
|
|
275
|
+
// Use first decision's model for all (for batch consistency)
|
|
276
|
+
const primary_model = decisions[0].primary_model;
|
|
277
|
+
decisions.forEach(d => {
|
|
278
|
+
d.primary_model = primary_model;
|
|
279
|
+
d.fallback_models = decisions[0].fallback_models;
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
if (options?.max_cost_per_prompt) {
|
|
283
|
+
decisions.forEach(d => {
|
|
284
|
+
if (d.estimated_cost > options.max_cost_per_prompt) {
|
|
285
|
+
// Route to cheaper alternative
|
|
286
|
+
const cheap = Object.entries(exports.MODEL_PROFILES)
|
|
287
|
+
.find(([name, p]) => p.cost_per_1k_input < 0.5);
|
|
288
|
+
if (cheap) {
|
|
289
|
+
d.primary_model = cheap[0];
|
|
290
|
+
d.reasoning = `Budget-limited routing to ${cheap[1].provider}`;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
return decisions;
|
|
392
296
|
}
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
// ============================================================
|
|
397
|
-
// TASK RECOMMENDATIONS
|
|
398
|
-
// ============================================================
|
|
399
|
-
|
|
297
|
+
/**
|
|
298
|
+
* Get model recommendation for task type
|
|
299
|
+
*/
|
|
400
300
|
function recommendForTask(task) {
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
};
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
exports.recommendForTask = recommendForTask;
|
|
413
|
-
|
|
414
|
-
// ============================================================
|
|
415
|
-
// ONLINE LEARNING - Update model profiles from feedback
|
|
416
|
-
// ============================================================
|
|
417
|
-
|
|
418
|
-
function updateModelProfile(model_name, actual_latency_ms, actual_cost, quality_rating) {
|
|
419
|
-
refreshModelProfiles();
|
|
420
|
-
const profile = MODEL_PROFILES[model_name];
|
|
421
|
-
if (!profile) return;
|
|
422
|
-
|
|
423
|
-
const alpha = 0.2; // Learning rate
|
|
424
|
-
profile.latency_ms = profile.latency_ms * (1 - alpha) + actual_latency_ms * alpha;
|
|
425
|
-
profile.quality_score = profile.quality_score * (1 - alpha) + quality_rating * alpha;
|
|
301
|
+
const features = extractQueryFeatures(task);
|
|
302
|
+
const decision = routeQuery(task);
|
|
303
|
+
// Return object format for backward compatibility
|
|
304
|
+
return {
|
|
305
|
+
primary: decision.primary_model,
|
|
306
|
+
fallbacks: decision.fallback_models || [],
|
|
307
|
+
reason: decision.reasoning || 'routed based on task'
|
|
308
|
+
};
|
|
426
309
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
//
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
310
|
+
/**
|
|
311
|
+
* Update model profile from execution feedback (online learning)
|
|
312
|
+
*/
|
|
313
|
+
function updateModelProfile(model_name, actual_latency_ms, actual_cost, quality_rating // 0-1
|
|
314
|
+
) {
|
|
315
|
+
const profile = exports.MODEL_PROFILES[model_name];
|
|
316
|
+
if (!profile)
|
|
317
|
+
return;
|
|
318
|
+
// Exponential moving average update
|
|
319
|
+
const alpha = 0.2; // Learning rate
|
|
320
|
+
profile.latency_ms = profile.latency_ms * (1 - alpha) + actual_latency_ms * alpha;
|
|
321
|
+
profile.quality_score = profile.quality_score * (1 - alpha) + quality_rating * alpha;
|
|
322
|
+
// Adjust cost perception
|
|
323
|
+
const actual_cost_per_1k = actual_cost * 1000;
|
|
324
|
+
const current_avg_cost = (profile.cost_per_1k_input + profile.cost_per_1k_output) / 2;
|
|
325
|
+
// Keep stored costs as reference, but note actual in profile
|
|
326
|
+
console.log(`[RouteLLM] Updated ${model_name}: latency=${profile.latency_ms.toFixed(0)}ms, quality=${profile.quality_score.toFixed(2)}`);
|
|
437
327
|
}
|
|
438
|
-
|
|
439
|
-
exports.getProviderHealth = getProviderHealth;
|
|
440
|
-
|
|
441
|
-
// ============================================================
|
|
442
|
-
// Default export
|
|
443
|
-
// ============================================================
|
|
444
|
-
|
|
445
328
|
exports.default = {
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
MODEL_PROFILES,
|
|
329
|
+
extractQueryFeatures,
|
|
330
|
+
routeQuery,
|
|
331
|
+
routeBatch,
|
|
332
|
+
recommendForTask,
|
|
333
|
+
updateModelProfile,
|
|
334
|
+
MODEL_PROFILES: exports.MODEL_PROFILES
|
|
453
335
|
};
|
|
336
|
+
//# sourceMappingURL=advancedRouter.js.map
|