adaptive-memory-multi-model-router 2.14.17 → 2.14.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_COUNCIL_FINDINGS.md +142 -0
- package/LAUNCH_CHECKLIST.md +141 -0
- package/README.md.bak +836 -0
- package/articles/CHINESE_SUBMISSIONS_READY.md +322 -0
- package/articles/DEVTO_READY.md +255 -0
- package/articles/HN_POST_READY.md +137 -0
- package/articles/INDIEHACKERS_READY.md +120 -0
- package/articles/NEWSLETTER_SEND_NOW.md +259 -0
- package/articles/PRODUCTHUNT_READY.md +106 -0
- package/articles/REDDIT_SUBMISSION_READY.md +348 -0
- package/articles/TWEET_STORM_READY.md +165 -0
- package/benchmark-results.json +24 -24
- package/council-votes/architecture-vote.md +121 -0
- package/council-votes/coverage-vote.md +93 -0
- package/dist/cost/costTracker.d.ts +109 -44
- package/dist/cost/costTracker.js +321 -98
- package/dist/cost/costTracker.js.map +1 -1
- package/dist/index.d.ts +6 -4
- package/dist/routing/advancedRouter.d.ts +38 -43
- package/dist/routing/advancedRouter.js +396 -408
- package/dist/routing/advancedRouter.js.map +1 -1
- package/dist/routing/providers/providerConfig.d.ts +49 -0
- package/dist/routing/providers/providerConfig.js +883 -0
- package/dist/routing/routing/advancedRouter.d.ts +62 -0
- package/dist/routing/routing/advancedRouter.js +447 -0
- package/dist/routing/utils/tokenUtils.d.ts +52 -0
- package/dist/routing/utils/tokenUtils.js +129 -0
- package/dist/server/proxyServer.d.ts +1 -1
- package/dist/utils/costUtils.d.ts +57 -0
- package/dist/utils/costUtils.js +150 -0
- package/dist/utils/costUtils.js.map +1 -0
- package/dist/utils/sorting.d.ts +12 -0
- package/dist/utils/sorting.js +37 -0
- package/dist/utils/sorting.js.map +1 -0
- package/package.json +1 -1
- package/research/ensemble-voting.md +324 -0
- package/research/loss-functions.md +545 -0
- package/research-log.md +49 -0
- package/src/cost/costTracker.ts +576 -0
- package/src/routing/advancedRouter.ts +540 -0
- package/src/utils/costUtils.ts +157 -0
- package/src/utils/sorting.ts +42 -0
- package/test-council/AGENT_COUNCIL_ARCHITECTURE.md +349 -0
- package/tests/security/guardrailEngine.test.ts +700 -0
- package/research/PUBLISH_LOG.md +0 -3
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A3M Router - Generic Adaptive Routing (RouteLLM Style)
|
|
3
|
+
*
|
|
4
|
+
* Routes queries to the best available LLM based on:
|
|
5
|
+
* - Query features (code, math, creative, etc.)
|
|
6
|
+
* - Provider availability (checks API keys)
|
|
7
|
+
* - Cost optimization
|
|
8
|
+
* - Quality vs speed tradeoff
|
|
9
|
+
*
|
|
10
|
+
* All provider references are dynamically loaded from providerConfig.
|
|
11
|
+
* Users can add/remove providers via environment variables or config files.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { getAvailableProviders } from "../providers/providerConfig";
|
|
15
|
+
import { estimateCost } from "../utils/tokenUtils";
|
|
16
|
+
import { logScaleCostScore } from "../utils/costUtils";
|
|
17
|
+
import { quickselectTopK, selectTop } from "../utils/sorting";
|
|
18
|
+
|
|
19
|
+
// ============================================================
|
|
20
|
+
// CACHE FOR MODEL PROFILES (avoids O(n*m) rebuild on every routeQuery)
|
|
21
|
+
// ============================================================
|
|
22
|
+
|
|
23
|
+
interface ModelProfile {
|
|
24
|
+
name: string;
|
|
25
|
+
provider: string;
|
|
26
|
+
providerName: string;
|
|
27
|
+
cost_per_1k_input: number;
|
|
28
|
+
cost_per_1k_output: number;
|
|
29
|
+
latency_ms: number;
|
|
30
|
+
quality_score: number;
|
|
31
|
+
strengths: string[];
|
|
32
|
+
context_window: number;
|
|
33
|
+
type: string;
|
|
34
|
+
priority: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
let cachedProfiles: Record<string, ModelProfile> | null = null;
|
|
38
|
+
let cacheTimestamp = 0;
|
|
39
|
+
const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
40
|
+
|
|
41
|
+
function buildModelProfiles(): Record<string, ModelProfile> {
|
|
42
|
+
const profiles: Record<string, ModelProfile> = {};
|
|
43
|
+
const available = getAvailableProviders();
|
|
44
|
+
|
|
45
|
+
for (const [providerId, provider] of Object.entries(available)) {
|
|
46
|
+
for (const model of provider.models) {
|
|
47
|
+
const modelKey = model.includes('/') ? model : providerId + '/' + model;
|
|
48
|
+
const costPerKInput = provider.costPerK ? provider.costPerK.input : 0;
|
|
49
|
+
const costPerKOutput = provider.costPerK ? provider.costPerK.output : 0;
|
|
50
|
+
|
|
51
|
+
// Assign strengths based on model characteristics
|
|
52
|
+
const strengths: string[] = [];
|
|
53
|
+
if (provider.type === 'cli') {
|
|
54
|
+
strengths.push('free', 'local');
|
|
55
|
+
}
|
|
56
|
+
if (costPerKInput < 0.3) {
|
|
57
|
+
strengths.push('budget', 'fast');
|
|
58
|
+
} else if (costPerKInput > 2) {
|
|
59
|
+
strengths.push('premium', 'reasoning');
|
|
60
|
+
}
|
|
61
|
+
if (provider.name === 'Mistral' || provider.name === 'Groq' || provider.name === 'Cerebras') {
|
|
62
|
+
strengths.push('fast', 'coding');
|
|
63
|
+
}
|
|
64
|
+
if (provider.name === 'CommandCode') {
|
|
65
|
+
strengths.push('code-aware', 'context-rich');
|
|
66
|
+
}
|
|
67
|
+
if (provider.name === 'OpenCode') {
|
|
68
|
+
strengths.push('free', 'multi-model');
|
|
69
|
+
}
|
|
70
|
+
if (provider.name === 'Google') {
|
|
71
|
+
strengths.push('multilingual', 'long-context');
|
|
72
|
+
}
|
|
73
|
+
if (provider.name === 'OpenAI') {
|
|
74
|
+
strengths.push('reasoning', 'coding', 'analysis');
|
|
75
|
+
}
|
|
76
|
+
if (provider.name === 'Anthropic') {
|
|
77
|
+
strengths.push('reasoning', 'creative', 'analysis');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
profiles[modelKey] = {
|
|
81
|
+
name: modelKey,
|
|
82
|
+
provider: providerId,
|
|
83
|
+
providerName: provider.name,
|
|
84
|
+
cost_per_1k_input: costPerKInput,
|
|
85
|
+
cost_per_1k_output: costPerKOutput,
|
|
86
|
+
latency_ms: provider.type === 'cli' ? 5000 : (provider.priority * 200 + 300),
|
|
87
|
+
quality_score: strengths.includes('premium') ? 0.95 :
|
|
88
|
+
strengths.includes('reasoning') ? 0.90 :
|
|
89
|
+
strengths.includes('fast') ? 0.82 : 0.80,
|
|
90
|
+
strengths,
|
|
91
|
+
context_window: provider.maxTokens || 8192,
|
|
92
|
+
type: provider.type,
|
|
93
|
+
priority: provider.priority,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return profiles;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Lazy cache with TTL - replaces refreshModelProfiles()
|
|
102
|
+
function getModelProfiles(): Record<string, ModelProfile> {
|
|
103
|
+
const now = Date.now();
|
|
104
|
+
if (!cachedProfiles || (now - cacheTimestamp) > CACHE_TTL_MS) {
|
|
105
|
+
cachedProfiles = buildModelProfiles();
|
|
106
|
+
cacheTimestamp = now;
|
|
107
|
+
}
|
|
108
|
+
return cachedProfiles;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Manual cache invalidation (call if provider config changes)
|
|
112
|
+
function invalidateProfileCache(): void {
|
|
113
|
+
cachedProfiles = null;
|
|
114
|
+
cacheTimestamp = 0;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export let MODEL_PROFILES: Record<string, ModelProfile> = {};
|
|
118
|
+
try {
|
|
119
|
+
MODEL_PROFILES = buildModelProfiles();
|
|
120
|
+
} catch (e) {
|
|
121
|
+
// Circular dependency at module load — will retry on first use
|
|
122
|
+
MODEL_PROFILES = {};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ============================================================
|
|
126
|
+
// FEATURE EXTRACTION (v3 — multi-signal complexity scorer)
|
|
127
|
+
// ============================================================
|
|
128
|
+
|
|
129
|
+
export interface QueryFeatures {
|
|
130
|
+
length: number;
|
|
131
|
+
wordCount: number;
|
|
132
|
+
complexity: number;
|
|
133
|
+
has_code: boolean;
|
|
134
|
+
requires_reasoning: boolean;
|
|
135
|
+
is_multilingual: boolean;
|
|
136
|
+
is_translation: boolean;
|
|
137
|
+
domain: string | null;
|
|
138
|
+
intent: string;
|
|
139
|
+
detected_language: string | null;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function extractQueryFeatures(prompt: string): QueryFeatures {
|
|
143
|
+
const lower = prompt.toLowerCase();
|
|
144
|
+
const words = prompt.split(/\s+/);
|
|
145
|
+
const wordCount = words.length;
|
|
146
|
+
|
|
147
|
+
// === SIGNAL 1: Domain Detection ===
|
|
148
|
+
const domainSignals: Record<string, { keywords: string[]; weight: number }> = {
|
|
149
|
+
legal: {
|
|
150
|
+
keywords: ['legal', 'law', 'contract', 'liability', 'litigation', 'patent', 'copyright',
|
|
151
|
+
'regulation', 'compliance', 'constitutional', 'statute', 'jurisdiction',
|
|
152
|
+
'court', 'ruling', 'precedent', 'attorney', 'amicus', 'sec ', 'fda ',
|
|
153
|
+
'gdpr', 'ccpa', 'cfpr', 'due diligence', 'merger', 'acquisition',
|
|
154
|
+
'10-k', 'sec filing', 'forensic', 'embezzlement', 'infringement'],
|
|
155
|
+
weight: 0.35
|
|
156
|
+
},
|
|
157
|
+
medical: {
|
|
158
|
+
keywords: ['clinical', 'medical', 'pharmaceutical', 'oncology', 'drug', 'trial protocol',
|
|
159
|
+
'diagnosis', 'treatment', 'epidemiolog', 'genome', 'cohort study',
|
|
160
|
+
'biomarker', 'efficacy', 'pharmacoeconomic', 'biologic', 'vaccine',
|
|
161
|
+
'sepsis', 'ehr ', 'surgical', 'patient safety', 'fda approval'],
|
|
162
|
+
weight: 0.35
|
|
163
|
+
},
|
|
164
|
+
finance: {
|
|
165
|
+
keywords: ['financial model', 'valuation', 'revenue', 'portfolio', 'derivative',
|
|
166
|
+
'hedge fund', 'series a', 'series b', 'startup valuation', 'sensitivity analysis',
|
|
167
|
+
'investment thesis', 'earnings', 'tax optimization', 'forensic accounting',
|
|
168
|
+
'multinational', 'jurisdiction', 'risk assessment', 'monte carlo',
|
|
169
|
+
'black-scholes', 'options pricing', 'credit risk'],
|
|
170
|
+
weight: 0.30
|
|
171
|
+
},
|
|
172
|
+
security: {
|
|
173
|
+
keywords: ['security audit', 'penetration', 'vulnerability', 'exploit', 'zero-trust',
|
|
174
|
+
'threat model', 'incident response', 'malware', 'ransomware',
|
|
175
|
+
'authentication flow', 'cryptograph', 'encryption', 'timing attack',
|
|
176
|
+
'supply chain attack', 'owasp', 'compliance', 'risk assessment',
|
|
177
|
+
'mfa', 'zero-day', 'firewall', 'intrusion'],
|
|
178
|
+
weight: 0.30
|
|
179
|
+
},
|
|
180
|
+
architecture: {
|
|
181
|
+
keywords: ['system design', 'microservice', 'distributed system', 'fault-tolerant',
|
|
182
|
+
'event-sourced', 'cqrs', 'consensus algorithm', 'real-time pipeline',
|
|
183
|
+
'high availability', 'multi-region', 'latency sla', 'kafka',
|
|
184
|
+
'event-driven', 'data warehouse', 'etl', 'streaming', '1m events',
|
|
185
|
+
'million events', 'scalab', 'infrastruct', 'deploy'],
|
|
186
|
+
weight: 0.25
|
|
187
|
+
},
|
|
188
|
+
data_science: {
|
|
189
|
+
keywords: ['machine learning', 'deep learning', 'neural network', 'transformer',
|
|
190
|
+
'training data', 'model accuracy', 'hyperparameter', 'cross-validation',
|
|
191
|
+
'feature engineering', 'data pipeline', 'pandas', 'numpy', 'scikit',
|
|
192
|
+
'tensorflow', 'pytorch', 'regression', 'classification', 'clustering'],
|
|
193
|
+
weight: 0.30
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
let detectedDomain: string | null = null;
|
|
198
|
+
let maxDomainScore = 0;
|
|
199
|
+
for (const [domain, signal] of Object.entries(domainSignals)) {
|
|
200
|
+
let domainScore = 0;
|
|
201
|
+
for (const kw of signal.keywords) {
|
|
202
|
+
if (lower.includes(kw)) {
|
|
203
|
+
domainScore += signal.weight;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (domainScore > maxDomainScore) {
|
|
207
|
+
maxDomainScore = domainScore;
|
|
208
|
+
detectedDomain = domain;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// === SIGNAL 2: Code Detection ===
|
|
213
|
+
const codeSignals = [
|
|
214
|
+
'function ', 'def ', 'class ', 'import ', 'from ', 'const ', 'let ', 'var ',
|
|
215
|
+
'=>', '->', 'async ', 'await ', 'return ', 'if (', 'for (', 'while (',
|
|
216
|
+
'public ', 'private ', 'protected ', 'static ', 'void ', 'int ', 'string ',
|
|
217
|
+
'#include', 'std::', 'cout', 'cin', 'printf(', 'println!',
|
|
218
|
+
'fn ', 'impl ', 'pub ', 'mut ', 'struct ', 'enum ',
|
|
219
|
+
'```', 'code', 'python', 'javascript', 'typescript', 'java', 'cpp', 'ruby',
|
|
220
|
+
'write a', 'create a', 'implement', 'algorithm'
|
|
221
|
+
];
|
|
222
|
+
const hasCode = codeSignals.some(sig => lower.includes(sig));
|
|
223
|
+
|
|
224
|
+
// === SIGNAL 3: Reasoning Detection ===
|
|
225
|
+
const reasoningSignals = [
|
|
226
|
+
'why', 'how', 'explain', 'analyze', 'compare', 'contrast', 'evaluate',
|
|
227
|
+
'think about', 'reason', 'logic', 'proof', 'derive', '证明', '分析',
|
|
228
|
+
'reasoning', 'step by step', 'thinking', 'thought process'
|
|
229
|
+
];
|
|
230
|
+
const requiresReasoning = reasoningSignals.some(sig => lower.includes(sig));
|
|
231
|
+
|
|
232
|
+
// === SIGNAL 4: Language Detection ===
|
|
233
|
+
const languagePatterns: [RegExp, string][] = [
|
|
234
|
+
[/[\u4e00-\u9fff]/, 'zh'],
|
|
235
|
+
[/[\u0900-\u097f]/, 'hi'],
|
|
236
|
+
[/[\u0600-\u06ff]/, 'ar'],
|
|
237
|
+
[/[\u0400-\u04ff]/, 'ru'],
|
|
238
|
+
[/[\u0900-\u097f]/, 'hi-latn'],
|
|
239
|
+
[/বাংলা|করুন|হিন্দি|ভারত|ভারতীয়/, 'bn'],
|
|
240
|
+
];
|
|
241
|
+
|
|
242
|
+
let detectedLanguage: string | null = null;
|
|
243
|
+
for (const [pattern, lang] of languagePatterns) {
|
|
244
|
+
if (pattern.test(prompt)) {
|
|
245
|
+
detectedLanguage = lang;
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Translation detection
|
|
251
|
+
const translationSignals = ['translate', 'translation', 'into english', 'to english',
|
|
252
|
+
'traducir', 'traduction', 'traduzione', 'übersetzen'];
|
|
253
|
+
const isTranslation = translationSignals.some(sig => lower.includes(sig)) ||
|
|
254
|
+
/to (english|french|german|spanish|chinese|japanese|korean)/i.test(prompt);
|
|
255
|
+
|
|
256
|
+
const isMultilingual = detectedLanguage !== null || isTranslation;
|
|
257
|
+
|
|
258
|
+
// === SIGNAL 5: Intent Classification ===
|
|
259
|
+
let intent = 'general';
|
|
260
|
+
if (hasCode) intent = 'code';
|
|
261
|
+
else if (isTranslation) intent = 'translation';
|
|
262
|
+
else if (lower.includes('write') || lower.includes('create') || lower.includes('generate')) {
|
|
263
|
+
intent = 'creative';
|
|
264
|
+
} else if (lower.includes('explain') || lower.includes('what is') || lower.includes('how does')) {
|
|
265
|
+
intent = 'explanation';
|
|
266
|
+
} else if (lower.includes('calculate') || lower.includes('compute') || lower.includes('integral')) {
|
|
267
|
+
intent = 'math';
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// === COMPLEXITY SCORING ===
|
|
271
|
+
// Base complexity from length
|
|
272
|
+
let complexity = Math.min(wordCount / 100, 1.0);
|
|
273
|
+
|
|
274
|
+
// Domain加成
|
|
275
|
+
if (detectedDomain) complexity += 0.2;
|
|
276
|
+
|
|
277
|
+
// Code加成
|
|
278
|
+
if (hasCode) complexity += 0.15;
|
|
279
|
+
|
|
280
|
+
// Reasoning加成
|
|
281
|
+
if (requiresReasoning) complexity += 0.15;
|
|
282
|
+
|
|
283
|
+
// Multilingual加成
|
|
284
|
+
if (isMultilingual) complexity += 0.1;
|
|
285
|
+
|
|
286
|
+
// Cap at 1.0
|
|
287
|
+
complexity = Math.min(complexity, 1.0);
|
|
288
|
+
|
|
289
|
+
return {
|
|
290
|
+
length: prompt.length,
|
|
291
|
+
wordCount,
|
|
292
|
+
complexity,
|
|
293
|
+
has_code: hasCode,
|
|
294
|
+
requires_reasoning: requiresReasoning,
|
|
295
|
+
is_multilingual: isMultilingual,
|
|
296
|
+
is_translation: isTranslation,
|
|
297
|
+
domain: detectedDomain,
|
|
298
|
+
intent,
|
|
299
|
+
detected_language: detectedLanguage,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function scoreModelFit(model: ModelProfile, features: QueryFeatures): number {
|
|
304
|
+
let score = model.quality_score * 0.6;
|
|
305
|
+
|
|
306
|
+
// Domain match
|
|
307
|
+
if (features.domain) {
|
|
308
|
+
const domainBonus: Record<string, string[]> = {
|
|
309
|
+
code: ['code-aware', 'coding', 'fast'],
|
|
310
|
+
medical: ['reasoning', 'analysis'],
|
|
311
|
+
legal: ['reasoning', 'analysis', 'context-rich'],
|
|
312
|
+
finance: ['analysis', 'reasoning'],
|
|
313
|
+
security: ['reasoning', 'analysis'],
|
|
314
|
+
architecture: ['context-rich', 'long-context'],
|
|
315
|
+
data_science: ['coding', 'fast', 'reasoning'],
|
|
316
|
+
};
|
|
317
|
+
const bonuses = domainBonus[features.domain] || [];
|
|
318
|
+
if (bonuses.some(b => model.strengths.includes(b))) {
|
|
319
|
+
score += 0.2;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Code bonus
|
|
324
|
+
if (features.has_code && model.strengths.includes('coding')) {
|
|
325
|
+
score += 0.15;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Multilingual bonus
|
|
329
|
+
if (features.is_multilingual && model.strengths.includes('multilingual')) {
|
|
330
|
+
score += 0.15;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// Free tier preference for simple queries
|
|
334
|
+
if (features.complexity < 0.5 && model.strengths.includes('free')) {
|
|
335
|
+
score += 0.2;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Fast provider for simple queries
|
|
339
|
+
if (features.complexity < 0.4 && model.strengths.includes('fast')) {
|
|
340
|
+
score += 0.15;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// Premium for complex queries
|
|
344
|
+
if (features.complexity > 0.6 && model.strengths.includes('premium')) {
|
|
345
|
+
score += 0.15;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
return Math.min(score, 1.0);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function costEfficiency(model: ModelProfile, features: QueryFeatures): number {
|
|
352
|
+
// Use log-scale cost score for better mid-range differentiation
|
|
353
|
+
// Lower cost → higher score (thanks to logScaleCostScore inverse mapping)
|
|
354
|
+
const avg_cost = (model.cost_per_1k_input + model.cost_per_1k_output) / 2;
|
|
355
|
+
const cost_score = logScaleCostScore(avg_cost);
|
|
356
|
+
|
|
357
|
+
// Simple queries weigh cost more heavily (0.6)
|
|
358
|
+
// Complex queries weigh cost less (0.2) since quality matters more
|
|
359
|
+
const weight = features.complexity < 0.5 ? 0.6 : 0.2;
|
|
360
|
+
return cost_score * weight;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// ============================================================
|
|
364
|
+
// ROUTING
|
|
365
|
+
// ============================================================
|
|
366
|
+
|
|
367
|
+
export interface RouteDecision {
|
|
368
|
+
primary_model: string | null;
|
|
369
|
+
fallback_models: string[];
|
|
370
|
+
confidence: number;
|
|
371
|
+
reasoning: string;
|
|
372
|
+
estimated_cost: number;
|
|
373
|
+
estimated_latency_ms: number;
|
|
374
|
+
features?: QueryFeatures;
|
|
375
|
+
provider_type?: string;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export function routeQuery(prompt: string, available_models?: string[], budget_multiplier: number = 1.0): RouteDecision {
|
|
379
|
+
// Use cached profiles instead of rebuilding every time (5-10ms savings)
|
|
380
|
+
const profiles = getModelProfiles();
|
|
381
|
+
|
|
382
|
+
const features = extractQueryFeatures(prompt);
|
|
383
|
+
const candidate_names = available_models || Object.keys(profiles);
|
|
384
|
+
|
|
385
|
+
// Filter to available models
|
|
386
|
+
const candidates = candidate_names
|
|
387
|
+
.filter(name => profiles[name])
|
|
388
|
+
.map(name => {
|
|
389
|
+
const profile = profiles[name];
|
|
390
|
+
const quality = scoreModelFit(profile, features);
|
|
391
|
+
const cost = costEfficiency(profile, features);
|
|
392
|
+
return {
|
|
393
|
+
name,
|
|
394
|
+
profile,
|
|
395
|
+
quality_score: quality,
|
|
396
|
+
cost_score: cost,
|
|
397
|
+
total_score: quality + cost
|
|
398
|
+
};
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
if (candidates.length === 0) {
|
|
402
|
+
return {
|
|
403
|
+
primary_model: null,
|
|
404
|
+
fallback_models: [],
|
|
405
|
+
confidence: 0,
|
|
406
|
+
reasoning: "No providers available - configure API keys in ~/.config/a3m-router/providers.json",
|
|
407
|
+
estimated_cost: 0,
|
|
408
|
+
estimated_latency_ms: 0,
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// Sort by total score (quality vs cost tradeoff based on complexity)
|
|
413
|
+
const complexity_bias = features.complexity > 0.6 ? 0.7 : 0.3;
|
|
414
|
+
const scoreFn = (c: typeof candidates[0]) => c.quality_score * complexity_bias + c.cost_score * (1 - complexity_bias);
|
|
415
|
+
|
|
416
|
+
const topCandidates = quickselectTopK(candidates, 4, scoreFn);
|
|
417
|
+
|
|
418
|
+
const primary = topCandidates[0];
|
|
419
|
+
const secondary = topCandidates.slice(1, 3);
|
|
420
|
+
|
|
421
|
+
// Calculate confidence based on score gap
|
|
422
|
+
let confidence = 0.5;
|
|
423
|
+
if (candidates.length > 1) {
|
|
424
|
+
const gap = primary.total_score - candidates[1].total_score;
|
|
425
|
+
confidence = Math.min(0.95, 0.5 + gap * 2);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// Build reasoning
|
|
429
|
+
const reasons: string[] = [];
|
|
430
|
+
if (features.has_code) reasons.push("code detected");
|
|
431
|
+
if (features.requires_reasoning) reasons.push("reasoning needed");
|
|
432
|
+
if (features.complexity > 0.6) reasons.push("high complexity");
|
|
433
|
+
if (features.is_multilingual) reasons.push("multilingual");
|
|
434
|
+
if (features.is_translation) reasons.push("translation");
|
|
435
|
+
if (primary.profile.strengths.includes("free")) reasons.push("free tier");
|
|
436
|
+
|
|
437
|
+
const estimated_tokens = features.length * 1.5;
|
|
438
|
+
const estimated_cost = estimateCost(features.length, estimated_tokens, primary.name);
|
|
439
|
+
|
|
440
|
+
return {
|
|
441
|
+
primary_model: primary.name,
|
|
442
|
+
fallback_models: secondary.map(c => c.name),
|
|
443
|
+
confidence,
|
|
444
|
+
reasoning: `Selected ${primary.profile.providerName || primary.profile.provider}/${primary.name} for ${reasons.join(", ") || "general query"}`,
|
|
445
|
+
estimated_cost: estimated_cost * budget_multiplier,
|
|
446
|
+
estimated_latency_ms: primary.profile.latency_ms,
|
|
447
|
+
features,
|
|
448
|
+
provider_type: primary.profile.type,
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// ============================================================
|
|
453
|
+
// BATCH ROUTING
|
|
454
|
+
// ============================================================
|
|
455
|
+
|
|
456
|
+
export function routeBatch(prompts: string[], options: {
|
|
457
|
+
same_model?: boolean;
|
|
458
|
+
max_cost_per_prompt?: number;
|
|
459
|
+
} = {}): RouteDecision[] {
|
|
460
|
+
const decisions = prompts.map(p => routeQuery(p));
|
|
461
|
+
|
|
462
|
+
if (options.same_model && decisions.length > 0) {
|
|
463
|
+
const primary_model = decisions[0].primary_model;
|
|
464
|
+
decisions.forEach(d => {
|
|
465
|
+
d.primary_model = primary_model;
|
|
466
|
+
d.fallback_models = decisions[0].fallback_models;
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
if (options.max_cost_per_prompt !== undefined) {
|
|
471
|
+
const profiles = getModelProfiles();
|
|
472
|
+
decisions.forEach(d => {
|
|
473
|
+
if (d.estimated_cost > options.max_cost_per_prompt!) {
|
|
474
|
+
const cheap = Object.entries(profiles)
|
|
475
|
+
.find(([name, p]) => p.cost_per_1k_input < 0.5);
|
|
476
|
+
if (cheap) {
|
|
477
|
+
d.primary_model = cheap[0];
|
|
478
|
+
d.reasoning = `Budget-limited routing to ${cheap[1].providerName || cheap[1].provider}`;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
return decisions;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// ============================================================
|
|
488
|
+
// TASK RECOMMENDATIONS
|
|
489
|
+
// ============================================================
|
|
490
|
+
|
|
491
|
+
export function recommendForTask(task: string) {
|
|
492
|
+
const features = extractQueryFeatures(task);
|
|
493
|
+
const decision = routeQuery(task);
|
|
494
|
+
return {
|
|
495
|
+
primary: decision.primary_model,
|
|
496
|
+
fallbacks: decision.fallback_models,
|
|
497
|
+
reason: decision.reasoning,
|
|
498
|
+
features,
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// ============================================================
|
|
503
|
+
// ONLINE LEARNING - Update model profiles from feedback
|
|
504
|
+
// ============================================================
|
|
505
|
+
|
|
506
|
+
export function updateModelProfile(model_name: string, actual_latency_ms: number, actual_cost: number, quality_rating: number): void {
|
|
507
|
+
const profiles = getModelProfiles();
|
|
508
|
+
const profile = profiles[model_name];
|
|
509
|
+
if (!profile) return;
|
|
510
|
+
|
|
511
|
+
const alpha = 0.2; // Learning rate
|
|
512
|
+
profile.latency_ms = profile.latency_ms * (1 - alpha) + actual_latency_ms * alpha;
|
|
513
|
+
profile.quality_score = profile.quality_score * (1 - alpha) + quality_rating * alpha;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// ============================================================
|
|
517
|
+
// PROVIDER HEALTH CHECK
|
|
518
|
+
// ============================================================
|
|
519
|
+
|
|
520
|
+
export async function getProviderHealth() {
|
|
521
|
+
const { checkAllProviders } = require("../providers/providerConfig");
|
|
522
|
+
return checkAllProviders();
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// ============================================================
|
|
526
|
+
// Default export
|
|
527
|
+
// ============================================================
|
|
528
|
+
|
|
529
|
+
module.exports = {
|
|
530
|
+
extractQueryFeatures,
|
|
531
|
+
routeQuery,
|
|
532
|
+
routeBatch,
|
|
533
|
+
recommendForTask,
|
|
534
|
+
updateModelProfile,
|
|
535
|
+
getProviderHealth,
|
|
536
|
+
MODEL_PROFILES,
|
|
537
|
+
invalidateProfileCache,
|
|
538
|
+
};
|
|
539
|
+
|
|
540
|
+
module.exports.default = module.exports;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log-scale cost utilities for A3M Router
|
|
3
|
+
*
|
|
4
|
+
* Provides better differentiation across cost ranges:
|
|
5
|
+
* - Free models get score 1.0
|
|
6
|
+
* - $0.05/1K vs $0.10/1K get meaningfully different scores
|
|
7
|
+
* - $0.10/1K vs $10/1K get much larger differentiation than linear scaling
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Log-scale cost score (0-1, lower cost = higher score)
|
|
12
|
+
* Uses log scale to better differentiate mid-range costs
|
|
13
|
+
*
|
|
14
|
+
* @param costPer1K - Cost per 1K tokens (input or output)
|
|
15
|
+
* @param minCost - Minimum cost boundary (default: $0.01)
|
|
16
|
+
* @param maxCost - Maximum cost boundary (default: $10)
|
|
17
|
+
* @returns Score from 0 to 1 (higher = cheaper)
|
|
18
|
+
*/
|
|
19
|
+
export function logScaleCostScore(costPer1K: number, minCost = 0.01, maxCost = 10): number {
|
|
20
|
+
// Handle free/zero cost models
|
|
21
|
+
if (costPer1K <= 0) return 1.0;
|
|
22
|
+
|
|
23
|
+
// Normalize to log scale between minCost and maxCost
|
|
24
|
+
const logMin = Math.log(minCost);
|
|
25
|
+
const logMax = Math.log(maxCost);
|
|
26
|
+
const logCost = Math.log(Math.max(costPer1K, minCost));
|
|
27
|
+
|
|
28
|
+
// Inverse: lower cost = higher score
|
|
29
|
+
return 1 - ((logCost - logMin) / (logMax - logMin));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Combined cost score (input + output weighted)
|
|
34
|
+
*
|
|
35
|
+
* @param inputCostPer1K - Input cost per 1K tokens
|
|
36
|
+
* @param outputCostPer1K - Output cost per 1K tokens
|
|
37
|
+
* @param outputWeight - Weight for output cost (default: 0.5)
|
|
38
|
+
* @returns Combined log-scale cost score
|
|
39
|
+
*/
|
|
40
|
+
export function combinedCostScore(inputCostPer1K: number, outputCostPer1K: number, outputWeight = 0.5): number {
|
|
41
|
+
const inputScore = logScaleCostScore(inputCostPer1K);
|
|
42
|
+
const outputScore = logScaleCostScore(outputCostPer1K);
|
|
43
|
+
return inputScore * (1 - outputWeight) + outputScore * outputWeight;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Cost margin loss for training
|
|
48
|
+
* Encourages routing to significantly cheaper models
|
|
49
|
+
*
|
|
50
|
+
* @param selectedCost - Cost of selected model
|
|
51
|
+
* @param alternativeCost - Cost of alternative model
|
|
52
|
+
* @param margin - Minimum margin threshold (default: 0.1 = 10%)
|
|
53
|
+
* @returns Loss value (0 if no significant saving available)
|
|
54
|
+
*/
|
|
55
|
+
export function costMarginLoss(
|
|
56
|
+
selectedCost: number,
|
|
57
|
+
alternativeCost: number,
|
|
58
|
+
margin = 0.1
|
|
59
|
+
): number {
|
|
60
|
+
// No loss if alternative is not cheaper
|
|
61
|
+
if (alternativeCost <= selectedCost) return 0;
|
|
62
|
+
|
|
63
|
+
// Calculate saving ratio: how much cheaper is alternative?
|
|
64
|
+
const savingRatio = (alternativeCost - selectedCost) / alternativeCost;
|
|
65
|
+
|
|
66
|
+
// Loss is how much we missed the margin threshold
|
|
67
|
+
return Math.max(0, margin - savingRatio);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Quality-adjusted cost score
|
|
72
|
+
* Normalizes cost score by model quality to prioritize
|
|
73
|
+
* cost-efficient models that are also high quality
|
|
74
|
+
*
|
|
75
|
+
* @param costScore - Raw log-scale cost score (0-1)
|
|
76
|
+
* @param qualityScore - Model quality score (0-1)
|
|
77
|
+
* @returns Quality-adjusted cost score
|
|
78
|
+
*/
|
|
79
|
+
export function qualityAdjustedCostScore(costScore: number, qualityScore: number): number {
|
|
80
|
+
// Combine: prefer high quality + low cost
|
|
81
|
+
return costScore * (0.3 + 0.7 * qualityScore);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Budget-aware cost penalty
|
|
86
|
+
* Applies stronger penalty for expensive models when budget is tight
|
|
87
|
+
*
|
|
88
|
+
* @param costPer1K - Cost per 1K tokens
|
|
89
|
+
* @param budgetMultiplier - Budget pressure (0.5 = tight, 1.0 = normal, 2.0 = generous)
|
|
90
|
+
* @returns Adjusted cost score
|
|
91
|
+
*/
|
|
92
|
+
export function budgetAwareCostScore(costPer1K: number, budgetMultiplier: number = 1.0): number {
|
|
93
|
+
const baseScore = logScaleCostScore(costPer1K);
|
|
94
|
+
|
|
95
|
+
// Adjust penalty based on budget
|
|
96
|
+
// Low budget (multiplier < 1) → stronger preference for cheap
|
|
97
|
+
// High budget (multiplier > 1) → more tolerant of expensive
|
|
98
|
+
const adjustment = Math.pow(baseScore, 1 / budgetMultiplier);
|
|
99
|
+
|
|
100
|
+
return adjustment;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ============================================================
|
|
104
|
+
// VALIDATION TESTS (can be run with: node src/utils/costUtils.ts)
|
|
105
|
+
// ============================================================
|
|
106
|
+
|
|
107
|
+
export function runValidationTests(): void {
|
|
108
|
+
const tests = [
|
|
109
|
+
// [cost, expectedBehavior]
|
|
110
|
+
[0, 1.0, "free model → score 1.0"],
|
|
111
|
+
[0.01, 1.0, "$0.01 → score 1.0 (at min boundary)"],
|
|
112
|
+
[0.05, 0.62, "$0.05 → high score (cheap)"],
|
|
113
|
+
[0.10, 0.52, "$0.10 → moderate score"],
|
|
114
|
+
[1.00, 0.30, "$1.00 → lower score"],
|
|
115
|
+
[10.0, 0.0, "$10.00 → score 0.0 (at max boundary)"],
|
|
116
|
+
|
|
117
|
+
// Check relative ordering
|
|
118
|
+
[0.05, "higher than 0.10", "verification"],
|
|
119
|
+
[0.10, "higher than 1.00", "verification"],
|
|
120
|
+
[1.00, "higher than 10.00", "verification"],
|
|
121
|
+
];
|
|
122
|
+
|
|
123
|
+
console.log("Log-Scale Cost Score Validation:");
|
|
124
|
+
console.log("=".repeat(50));
|
|
125
|
+
|
|
126
|
+
let passed = 0;
|
|
127
|
+
let failed = 0;
|
|
128
|
+
|
|
129
|
+
for (const test of tests) {
|
|
130
|
+
const cost = test[0] as number;
|
|
131
|
+
const expected = test[1];
|
|
132
|
+
const desc = test[2] as string;
|
|
133
|
+
|
|
134
|
+
const score = logScaleCostScore(cost);
|
|
135
|
+
|
|
136
|
+
if (typeof expected === "number") {
|
|
137
|
+
const ok = Math.abs(score - expected) < 0.02;
|
|
138
|
+
console.log(` ${ok ? "✓" : "✗"} ${desc}: cost=$${cost} → score=${score.toFixed(3)} (expected ~${expected})`);
|
|
139
|
+
if (ok) passed++; else failed++;
|
|
140
|
+
} else {
|
|
141
|
+
// Verification test
|
|
142
|
+
const compareCost = parseFloat(desc.split(" ")[0]);
|
|
143
|
+
const compareScore = logScaleCostScore(compareCost);
|
|
144
|
+
const ok = score > compareScore;
|
|
145
|
+
console.log(` ${ok ? "✓" : "✗"} ${desc}: ${cost} > ${compareCost} (${score.toFixed(3)} > ${compareScore.toFixed(3)})`);
|
|
146
|
+
if (ok) passed++; else failed++;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
console.log("=".repeat(50));
|
|
151
|
+
console.log(`Results: ${passed} passed, ${failed} failed`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Run if called directly
|
|
155
|
+
if (require.main === module) {
|
|
156
|
+
runValidationTests();
|
|
157
|
+
}
|