adaptive-memory-multi-model-router 2.14.60 → 2.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/ISSUE_TEMPLATE/checklist.md +35 -0
- package/.github/workflows/ci.yml +9 -6
- package/POPULARITY_BOOSTERS.md +18 -0
- package/README.md +130 -1067
- package/assets/chart-accuracy-by-tier.svg +63 -0
- package/assets/chart-confusion-matrix.svg +98 -0
- package/assets/chart-cost-comparison.svg +66 -0
- package/assets/chart-latency-overhead.svg +102 -0
- package/assets/chart-routerena-leaderboard.svg +76 -0
- package/dist/analytics/costAnalytics.d.ts +1 -0
- package/dist/benchmark/comprehensive.d.ts +4 -53
- package/dist/benchmark/comprehensive.d.ts.map +1 -0
- package/dist/benchmark/comprehensive.js +112 -214
- package/dist/benchmark/comprehensive.js.map +1 -1
- package/dist/benchmark/reproducible.d.ts +1 -0
- package/dist/cache/semanticCache.d.ts +1 -0
- package/dist/cli/setupWizard.d.ts +6 -1
- package/dist/cli/setupWizard.d.ts.map +1 -1
- package/dist/cli/setupWizard.js +6 -9
- package/dist/cli/setupWizard.js.map +1 -1
- package/dist/cli.js +15 -14
- package/dist/cost/budgetEnforcer.d.ts +1 -0
- package/dist/cost/costTracker.d.ts +1 -0
- package/dist/ensemble/multiRoundDialog.d.ts +1 -0
- package/dist/ensemble/multiRoundDialog.d.ts.map +1 -0
- package/dist/ensemble/shapleyValue.d.ts +1 -0
- package/dist/ensemble/shapleyValue.d.ts.map +1 -0
- package/dist/ensemble.d.ts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/integrations/langchainAdapter.d.ts +1 -0
- package/dist/integrations/oauth.d.ts +1 -0
- package/dist/integrations/scienceAdapter.d.ts +1 -0
- package/dist/integrations/scienceAdapter.d.ts.map +1 -0
- package/dist/memory/autoFetch.d.ts +1 -0
- package/dist/memory/hybridMemory.d.ts +1 -0
- package/dist/memory/hybridMemory.d.ts.map +1 -0
- package/dist/memory/memoryTree.d.ts +1 -0
- package/dist/memory/memoryTree.d.ts.map +1 -1
- package/dist/memory/obsidianVault.d.ts +1 -0
- package/dist/memory/obsidianVault.d.ts.map +1 -1
- package/dist/memory/reasoningBank.d.ts +1 -0
- package/dist/memory/reasoningBank.d.ts.map +1 -0
- package/dist/observability/changeWatch.d.ts +1 -0
- package/dist/observability/fatigueDetector.d.ts +1 -0
- package/dist/observability/index.d.ts +1 -0
- package/dist/observability/metrics.d.ts +1 -0
- package/dist/observability/metrics.d.ts.map +1 -1
- package/dist/observability/middleware.d.ts +1 -0
- package/dist/observability/tracer.d.ts +1 -0
- package/dist/observability/tracer.d.ts.map +1 -1
- package/dist/observability/types.d.ts +1 -0
- package/dist/providers/providerConfig.d.ts +1 -0
- package/dist/providers/providerConfig.d.ts.map +1 -1
- package/dist/routing/advancedRouter.d.ts +2 -1
- package/dist/routing/advancedRouter.d.ts.map +1 -1
- package/dist/routing/crossModelValidation.d.ts +1 -0
- package/dist/routing/providerHealth.d.ts +1 -0
- package/dist/routing/providerHealth.d.ts.map +1 -1
- package/dist/routing/providerRetry.d.ts +1 -0
- package/dist/sdk.d.ts +1 -0
- package/dist/security/guardrails.d.ts +1 -0
- package/dist/security/guardrails.d.ts.map +1 -1
- package/dist/server/dashboard.d.ts +1 -0
- package/dist/server/handlers/chatHandler.d.ts +11 -0
- package/dist/server/handlers/chatHandler.d.ts.map +1 -0
- package/dist/server/handlers/chatHandler.js +159 -0
- package/dist/server/handlers/chatHandler.js.map +1 -0
- package/dist/server/handlers/completionsHandler.d.ts +10 -0
- package/dist/server/handlers/completionsHandler.d.ts.map +1 -0
- package/dist/server/handlers/completionsHandler.js +124 -0
- package/dist/server/handlers/completionsHandler.js.map +1 -0
- package/dist/server/handlers/embeddingsHandler.d.ts +17 -0
- package/dist/server/handlers/embeddingsHandler.d.ts.map +1 -0
- package/dist/server/handlers/embeddingsHandler.js +235 -0
- package/dist/server/handlers/embeddingsHandler.js.map +1 -0
- package/dist/server/handlers/healthHandler.d.ts +10 -0
- package/dist/server/handlers/healthHandler.d.ts.map +1 -0
- package/dist/server/handlers/healthHandler.js +49 -0
- package/dist/server/handlers/healthHandler.js.map +1 -0
- package/dist/server/handlers/metricsHandler.d.ts +11 -0
- package/dist/server/handlers/metricsHandler.d.ts.map +1 -0
- package/dist/server/handlers/metricsHandler.js +24 -0
- package/dist/server/handlers/metricsHandler.js.map +1 -0
- package/dist/server/handlers/modelsHandler.d.ts +10 -0
- package/dist/server/handlers/modelsHandler.d.ts.map +1 -0
- package/dist/server/handlers/modelsHandler.js +19 -0
- package/dist/server/handlers/modelsHandler.js.map +1 -0
- package/dist/server/metrics.d.ts +96 -0
- package/dist/server/metrics.d.ts.map +1 -0
- package/dist/server/metrics.js +267 -0
- package/dist/server/metrics.js.map +1 -0
- package/dist/server/modelMapper.d.ts +1 -0
- package/dist/server/proxyServer.d.ts +53 -17
- package/dist/server/proxyServer.d.ts.map +1 -1
- package/dist/server/proxyServer.js +66 -303
- package/dist/server/proxyServer.js.map +1 -1
- package/dist/server/router.d.ts +49 -0
- package/dist/server/router.d.ts.map +1 -0
- package/dist/server/router.js +120 -0
- package/dist/server/router.js.map +1 -0
- package/dist/server/state.d.ts +30 -0
- package/dist/server/state.d.ts.map +1 -0
- package/dist/server/state.js +18 -0
- package/dist/server/state.js.map +1 -0
- package/dist/skills/__tests__/skill_manager.test.d.ts +3 -0
- package/dist/skills/__tests__/skill_manager.test.d.ts.map +1 -1
- package/dist/skills/__tests__/skill_manager.test.js +3 -6
- package/dist/skills/__tests__/skill_manager.test.js.map +1 -1
- package/dist/tui/dashboard.d.ts +1 -0
- package/dist/tui/index.d.ts +1 -0
- package/dist/utils/costUtils.d.ts +1 -0
- package/dist/utils/reliability.js +4 -18
- package/dist/utils/sorting.d.ts +1 -0
- package/dist/utils/tokenUtils.d.ts +1 -0
- package/docs/assets/chart-accuracy-by-tier.svg +63 -0
- package/docs/assets/chart-confusion-matrix.svg +98 -0
- package/docs/assets/chart-cost-comparison.svg +66 -0
- package/docs/assets/chart-latency-overhead.svg +102 -0
- package/docs/assets/chart-routerena-leaderboard.svg +76 -0
- package/docs/index.html +72 -80
- package/docs/llms-full.txt +156 -184
- package/docs/llms.txt +77 -42
- package/hf-space/app.py +3 -3
- package/llms-full.txt +156 -184
- package/llms.txt +77 -42
- package/mcp-server/package.json +6 -0
- package/mcp-server/tsconfig.json +3 -2
- package/package.json +5 -5
- package/src/server/handlers/chatHandler.ts +173 -0
- package/src/server/handlers/completionsHandler.ts +120 -0
- package/src/server/handlers/embeddingsHandler.ts +271 -0
- package/src/server/handlers/healthHandler.ts +57 -0
- package/src/server/handlers/metricsHandler.ts +30 -0
- package/src/server/handlers/modelsHandler.ts +25 -0
- package/src/server/metrics.ts +303 -0
- package/src/server/proxyServer.ts +71 -394
- package/src/server/router.ts +157 -0
- package/src/server/state.ts +34 -0
- package/src/skills/__tests__/skill_manager.test.ts +3 -3
- package/NEW_OPPORTUNITIES.md +0 -163
- package/NEW_SUBMISSIONS.md +0 -80
- package/PRIORITY_REDDIT_TARGETS.md +0 -53
- package/VISIBILITY_PLAN.md +0 -146
- package/articles/TWITTER_THREAD_IMPLICATIONS.md +0 -182
- package/articles/TWITTER_THREAD_VAULT.md +0 -164
- package/index.html +0 -667
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 480 280" width="480" height="280">
|
|
2
|
+
<defs>
|
|
3
|
+
<linearGradient id="barGrad" x1="0%" y1="0%" x2="0%" y2="100%">
|
|
4
|
+
<stop offset="0%" stop-color="#22c55e"/>
|
|
5
|
+
<stop offset="100%" stop-color="#16a34a"/>
|
|
6
|
+
</linearGradient>
|
|
7
|
+
</defs>
|
|
8
|
+
|
|
9
|
+
<!-- Background -->
|
|
10
|
+
<rect width="480" height="280" fill="#fefefe"/>
|
|
11
|
+
|
|
12
|
+
<!-- Title -->
|
|
13
|
+
<text x="240" y="28" text-anchor="middle" fill="#111" font-family="system-ui,sans-serif" font-size="14" font-weight="600">Routing Accuracy by Tier (n=200)</text>
|
|
14
|
+
|
|
15
|
+
<!-- Y-axis -->
|
|
16
|
+
<line x1="60" y1="50" x2="60" y2="230" stroke="#333" stroke-width="1"/>
|
|
17
|
+
<line x1="60" y1="230" x2="460" y2="230" stroke="#333" stroke-width="1"/>
|
|
18
|
+
|
|
19
|
+
<!-- Y-axis labels -->
|
|
20
|
+
<g font-family="system-ui,sans-serif" font-size="10" fill="#555" text-anchor="end">
|
|
21
|
+
<text x="52" y="54">100%</text>
|
|
22
|
+
<text x="52" y="99">80%</text>
|
|
23
|
+
<text x="52" y="144">60%</text>
|
|
24
|
+
<text x="52" y="189">40%</text>
|
|
25
|
+
<text x="52" y="234">0%</text>
|
|
26
|
+
</g>
|
|
27
|
+
|
|
28
|
+
<!-- Grid lines -->
|
|
29
|
+
<g stroke="#e5e7eb" stroke-width="1" stroke-dasharray="4,3">
|
|
30
|
+
<line x1="60" y1="55" x2="460" y2="55"/>
|
|
31
|
+
<line x1="60" y1="100" x2="460" y2="100"/>
|
|
32
|
+
<line x1="60" y1="145" x2="460" y2="145"/>
|
|
33
|
+
<line x1="60" y1="190" x2="460" y2="190"/>
|
|
34
|
+
</g>
|
|
35
|
+
|
|
36
|
+
<!-- Bars: correct/total -->
|
|
37
|
+
<!-- free: 48/50 = 96% -->
|
|
38
|
+
<rect x="100" y="55" width="60" height="175" fill="url(#barGrad)" rx="2"/>
|
|
39
|
+
<text x="130" y="245" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">free</text>
|
|
40
|
+
<text x="130" y="48" text-anchor="middle" fill="#16a34a" font-family="system-ui,sans-serif" font-size="11" font-weight="600">96%</text>
|
|
41
|
+
<text x="130" y="238" text-anchor="middle" fill="#888" font-family="system-ui,sans-serif" font-size="9">48/50</text>
|
|
42
|
+
|
|
43
|
+
<!-- cheap: 45/60 = 75% -->
|
|
44
|
+
<rect x="200" y="85" width="60" height="145" fill="url(#barGrad)" rx="2"/>
|
|
45
|
+
<text x="230" y="245" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">cheap</text>
|
|
46
|
+
<text x="230" y="78" text-anchor="middle" fill="#16a34a" font-family="system-ui,sans-serif" font-size="11" font-weight="600">75%</text>
|
|
47
|
+
<text x="230" y="238" text-anchor="middle" fill="#888" font-family="system-ui,sans-serif" font-size="9">45/60</text>
|
|
48
|
+
|
|
49
|
+
<!-- mid: 18/50 = 36% -->
|
|
50
|
+
<rect x="300" y="169" width="60" height="61" fill="url(#barGrad)" rx="2"/>
|
|
51
|
+
<text x="330" y="245" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">mid</text>
|
|
52
|
+
<text x="330" y="162" text-anchor="middle" fill="#16a34a" font-family="system-ui,sans-serif" font-size="11" font-weight="600">36%</text>
|
|
53
|
+
<text x="330" y="238" text-anchor="middle" fill="#888" font-family="system-ui,sans-serif" font-size="9">18/50</text>
|
|
54
|
+
|
|
55
|
+
<!-- premium: 23/40 = 57.5% -->
|
|
56
|
+
<rect x="400" y="124" width="60" height="106" fill="url(#barGrad)" rx="2"/>
|
|
57
|
+
<text x="430" y="245" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">premium</text>
|
|
58
|
+
<text x="430" y="117" text-anchor="middle" fill="#16a34a" font-family="system-ui,sans-serif" font-size="11" font-weight="600">57%</text>
|
|
59
|
+
<text x="430" y="238" text-anchor="middle" fill="#888" font-family="system-ui,sans-serif" font-size="9">23/40</text>
|
|
60
|
+
|
|
61
|
+
<!-- Overall -->
|
|
62
|
+
<text x="240" y="268" text-anchor="middle" fill="#888" font-family="system-ui,sans-serif" font-size="9">Exact tier match: 67% | Within 1 tier: 96%</text>
|
|
63
|
+
</svg>
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 420 340" width="420" height="340">
|
|
2
|
+
<rect width="420" height="340" fill="#fefefe"/>
|
|
3
|
+
|
|
4
|
+
<!-- Title -->
|
|
5
|
+
<text x="210" y="22" text-anchor="middle" fill="#111" font-family="system-ui,sans-serif" font-size="13" font-weight="600">Confusion Matrix — Routing Decisions (n=200)</text>
|
|
6
|
+
<text x="210" y="38" text-anchor="middle" fill="#666" font-family="system-ui,sans-serif" font-size="9">Predicted tier vs actual tier</text>
|
|
7
|
+
|
|
8
|
+
<!-- Matrix: 4x4, each cell 60x45 -->
|
|
9
|
+
<!-- Actual label on left, Predicted on top -->
|
|
10
|
+
<!-- free: 50, cheap: 60, mid: 50, premium: 40 -->
|
|
11
|
+
|
|
12
|
+
<!-- Column headers -->
|
|
13
|
+
<text x="120" y="58" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">free</text>
|
|
14
|
+
<text x="180" y="58" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">cheap</text>
|
|
15
|
+
<text x="240" y="58" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">mid</text>
|
|
16
|
+
<text x="300" y="58" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">premium</text>
|
|
17
|
+
|
|
18
|
+
<!-- Row: free (actual=50) | predicted: free=48, cheap=2, mid=0, premium=0 -->
|
|
19
|
+
<text x="85" y="100" text-anchor="end" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">free</text>
|
|
20
|
+
<text x="85" y="115" text-anchor="end" fill="#888" font-family="system-ui,sans-serif" font-size="8">(n=50)</text>
|
|
21
|
+
<!-- free→free: 48/50=96% -->
|
|
22
|
+
<rect x="105" y="80" width="55" height="35" fill="#16a34a" rx="2"/>
|
|
23
|
+
<text x="132" y="102" text-anchor="middle" fill="#fff" font-family="system-ui,sans-serif" font-size="13" font-weight="600">48</text>
|
|
24
|
+
<!-- free→cheap: 2 -->
|
|
25
|
+
<rect x="165" y="80" width="55" height="35" fill="#bbf7d0" rx="2"/>
|
|
26
|
+
<text x="192" y="102" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">2</text>
|
|
27
|
+
<!-- free→mid: 0 -->
|
|
28
|
+
<rect x="225" y="80" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
29
|
+
<text x="252" y="102" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">0</text>
|
|
30
|
+
<!-- free→premium: 0 -->
|
|
31
|
+
<rect x="285" y="80" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
32
|
+
<text x="312" y="102" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">0</text>
|
|
33
|
+
|
|
34
|
+
<!-- Row: cheap (actual=60) | predicted: free=13, cheap=45, mid=1, premium=1 -->
|
|
35
|
+
<text x="85" y="145" text-anchor="end" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">cheap</text>
|
|
36
|
+
<text x="85" y="160" text-anchor="end" fill="#888" font-family="system-ui,sans-serif" font-size="8">(n=60)</text>
|
|
37
|
+
<!-- cheap→free: 13 -->
|
|
38
|
+
<rect x="105" y="125" width="55" height="35" fill="#fef08a" rx="2"/>
|
|
39
|
+
<text x="132" y="147" text-anchor="middle" fill="#713f12" font-family="system-ui,sans-serif" font-size="12">13</text>
|
|
40
|
+
<!-- cheap→cheap: 45/60=75% -->
|
|
41
|
+
<rect x="165" y="125" width="55" height="35" fill="#16a34a" rx="2"/>
|
|
42
|
+
<text x="192" y="147" text-anchor="middle" fill="#fff" font-family="system-ui,sans-serif" font-size="13" font-weight="600">45</text>
|
|
43
|
+
<!-- cheap→mid: 1 -->
|
|
44
|
+
<rect x="225" y="125" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
45
|
+
<text x="252" y="147" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">1</text>
|
|
46
|
+
<!-- cheap→premium: 1 -->
|
|
47
|
+
<rect x="285" y="125" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
48
|
+
<text x="312" y="147" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">1</text>
|
|
49
|
+
|
|
50
|
+
<!-- Row: mid (actual=50) | predicted: free=1, cheap=22, mid=18, premium=9 -->
|
|
51
|
+
<text x="85" y="190" text-anchor="end" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">mid</text>
|
|
52
|
+
<text x="85" y="205" text-anchor="end" fill="#888" font-family="system-ui,sans-serif" font-size="8">(n=50)</text>
|
|
53
|
+
<!-- mid→free: 1 -->
|
|
54
|
+
<rect x="105" y="170" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
55
|
+
<text x="132" y="192" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">1</text>
|
|
56
|
+
<!-- mid→cheap: 22 (most common error) -->
|
|
57
|
+
<rect x="165" y="170" width="55" height="35" fill="#fef08a" rx="2"/>
|
|
58
|
+
<text x="192" y="192" text-anchor="middle" fill="#713f12" font-family="system-ui,sans-serif" font-size="12">22</text>
|
|
59
|
+
<!-- mid→mid: 18/50=36% -->
|
|
60
|
+
<rect x="225" y="170" width="55" height="35" fill="#16a34a" rx="2"/>
|
|
61
|
+
<text x="252" y="192" text-anchor="middle" fill="#fff" font-family="system-ui,sans-serif" font-size="13" font-weight="600">18</text>
|
|
62
|
+
<!-- mid→premium: 9 -->
|
|
63
|
+
<rect x="285" y="170" width="55" height="35" fill="#fef08a" rx="2"/>
|
|
64
|
+
<text x="312" y="192" text-anchor="middle" fill="#713f12" font-family="system-ui,sans-serif" font-size="12">9</text>
|
|
65
|
+
|
|
66
|
+
<!-- Row: premium (actual=40) | predicted: free=2, cheap=4, mid=11, premium=23 -->
|
|
67
|
+
<text x="85" y="235" text-anchor="end" fill="#333" font-family="system-ui,sans-serif" font-size="10" font-weight="600">premium</text>
|
|
68
|
+
<text x="85" y="250" text-anchor="end" fill="#888" font-family="system-ui,sans-serif" font-size="8">(n=40)</text>
|
|
69
|
+
<!-- premium→free: 2 -->
|
|
70
|
+
<rect x="105" y="215" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
71
|
+
<text x="132" y="237" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">2</text>
|
|
72
|
+
<!-- premium→cheap: 4 -->
|
|
73
|
+
<rect x="165" y="215" width="55" height="35" fill="#d1fae5" rx="2"/>
|
|
74
|
+
<text x="192" y="237" text-anchor="middle" fill="#16634a" font-family="system-ui,sans-serif" font-size="12">4</text>
|
|
75
|
+
<!-- premium→mid: 11 -->
|
|
76
|
+
<rect x="225" y="215" width="55" height="35" fill="#fef08a" rx="2"/>
|
|
77
|
+
<text x="252" y="237" text-anchor="middle" fill="#713f12" font-family="system-ui,sans-serif" font-size="12">11</text>
|
|
78
|
+
<!-- premium→premium: 23/40=57.5% -->
|
|
79
|
+
<rect x="285" y="215" width="55" height="35" fill="#16a34a" rx="2"/>
|
|
80
|
+
<text x="312" y="237" text-anchor="middle" fill="#fff" font-family="system-ui,sans-serif" font-size="13" font-weight="600">23</text>
|
|
81
|
+
|
|
82
|
+
<!-- Y-axis label (actual) -->
|
|
83
|
+
<text x="18" y="165" text-anchor="middle" fill="#555" font-family="system-ui,sans-serif" font-size="10" transform="rotate(-90,18,165)">Actual Tier</text>
|
|
84
|
+
|
|
85
|
+
<!-- X-axis label (predicted) -->
|
|
86
|
+
<text x="210" y="268" text-anchor="middle" fill="#555" font-family="system-ui,sans-serif" font-size="10">Predicted Tier</text>
|
|
87
|
+
|
|
88
|
+
<!-- Legend -->
|
|
89
|
+
<rect x="105" y="278" width="12" height="12" fill="#16a34a" rx="2"/>
|
|
90
|
+
<text x="122" y="288" fill="#333" font-family="system-ui,sans-serif" font-size="9">Correct</text>
|
|
91
|
+
<rect x="165" y="278" width="12" height="12" fill="#fef08a" rx="2"/>
|
|
92
|
+
<text x="182" y="288" fill="#333" font-family="system-ui,sans-serif" font-size="9">Off by 1 tier</text>
|
|
93
|
+
<rect x="245" y="278" width="12" height="12" fill="#d1fae5" rx="2"/>
|
|
94
|
+
<text x="262" y="288" fill="#333" font-family="system-ui,sans-serif" font-size="9">Off by 2+ tiers</text>
|
|
95
|
+
|
|
96
|
+
<!-- Summary -->
|
|
97
|
+
<text x="210" y="310" text-anchor="middle" fill="#555" font-family="system-ui,sans-serif" font-size="9">Exact match: 67% (134/200) | Off by 1: 29% (58/200) | Off by 2+: 4% (8/200)</text>
|
|
98
|
+
</svg>
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 420 280" width="420" height="280">
|
|
2
|
+
<defs>
|
|
3
|
+
<linearGradient id="costBg" x1="0%" y1="0%" x2="0%" y2="100%">
|
|
4
|
+
<stop offset="0%" stop-color="#1e293b"/>
|
|
5
|
+
<stop offset="100%" stop-color="#0f172a"/>
|
|
6
|
+
</linearGradient>
|
|
7
|
+
</defs>
|
|
8
|
+
|
|
9
|
+
<rect width="420" height="280" fill="url(#costBg)" rx="8"/>
|
|
10
|
+
|
|
11
|
+
<!-- Title -->
|
|
12
|
+
<text x="210" y="28" text-anchor="middle" fill="#f1f5f9" font-family="system-ui,sans-serif" font-size="14" font-weight="600">Cost per 1K Tokens — Router Comparison</text>
|
|
13
|
+
<text x="210" y="44" text-anchor="middle" fill="#64748b" font-family="system-ui,sans-serif" font-size="10">Lower is better | All values from RouterArena evaluation (PR#144)</text>
|
|
14
|
+
|
|
15
|
+
<!-- Y-axis: 0 to $0.30, mapped 190px -->
|
|
16
|
+
<!-- Scale: $0.30 = 190px → $1 = 633px, $0.01 = 6.33px -->
|
|
17
|
+
|
|
18
|
+
<g transform="translate(55,58)">
|
|
19
|
+
<!-- Y-axis -->
|
|
20
|
+
<line x1="40" y1="0" x2="40" y2="190" stroke="#334155" stroke-width="1.5"/>
|
|
21
|
+
<line x1="40" y1="190" x2="360" y2="190" stroke="#334155" stroke-width="1.5"/>
|
|
22
|
+
|
|
23
|
+
<!-- Y-axis gridlines (every $0.05) -->
|
|
24
|
+
<g stroke="#1e293b" stroke-width="1">
|
|
25
|
+
<line x1="40" y1="0" x2="360" y2="0"/> <!-- $0.30 -->
|
|
26
|
+
<line x1="40" y1="31.7" x2="360" y2="31.7"/> <!-- $0.25 -->
|
|
27
|
+
<line x1="40" y1="63.3" x2="360" y2="63.3"/> <!-- $0.20 -->
|
|
28
|
+
<line x1="40" y1="95" x2="360" y2="95"/> <!-- $0.15 -->
|
|
29
|
+
<line x1="40" y1="126.7" x2="360" y2="126.7"/> <!-- $0.10 -->
|
|
30
|
+
<line x1="40" y1="158.3" x2="360" y2="158.3"/> <!-- $0.05 -->
|
|
31
|
+
<line x1="40" y1="190" x2="360" y2="190"/> <!-- $0 -->
|
|
32
|
+
</g>
|
|
33
|
+
|
|
34
|
+
<!-- Y-axis labels -->
|
|
35
|
+
<g fill="#64748b" font-family="system-ui,sans-serif" font-size="10" text-anchor="end">
|
|
36
|
+
<text x="35" y="4">$0.30</text>
|
|
37
|
+
<text x="35" y="66">$0.20</text>
|
|
38
|
+
<text x="35" y="127">$0.10</text>
|
|
39
|
+
<text x="35" y="194">$0</text>
|
|
40
|
+
</g>
|
|
41
|
+
|
|
42
|
+
<!-- Bar 1: A3M Router $0.0768 = 49px → y=141, h=49 -->
|
|
43
|
+
<rect x="60" y="141" width="70" height="49" fill="#22c55e" rx="3"/>
|
|
44
|
+
<text x="95" y="135" text-anchor="middle" fill="#4ade80" font-family="system-ui,sans-serif" font-size="12" font-weight="600">$0.0768</text>
|
|
45
|
+
<text x="95" y="207" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="10">A3M Router</text>
|
|
46
|
+
<text x="95" y="219" text-anchor="middle" fill="#64748b" font-family="system-ui,sans-serif" font-size="8">(this work)</text>
|
|
47
|
+
|
|
48
|
+
<!-- Bar 2: Sqwish $0.18 = 114px → y=76, h=114 -->
|
|
49
|
+
<rect x="160" y="76" width="70" height="114" fill="#94a3b8" rx="3" opacity="0.7"/>
|
|
50
|
+
<text x="195" y="70" text-anchor="middle" fill="#cbd5e1" font-family="system-ui,sans-serif" font-size="12" font-weight="600">$0.18</text>
|
|
51
|
+
<text x="195" y="207" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="10">Sqwish</text>
|
|
52
|
+
|
|
53
|
+
<!-- Bar 3: Azure-Model-Router $0.22 = 139px → y=51, h=139 -->
|
|
54
|
+
<rect x="260" y="51" width="70" height="139" fill="#94a3b8" rx="3" opacity="0.5"/>
|
|
55
|
+
<text x="295" y="45" text-anchor="middle" fill="#cbd5e1" font-family="system-ui,sans-serif" font-size="12" font-weight="600">$0.22</text>
|
|
56
|
+
<text x="295" y="207" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="10">Azure-Router</text>
|
|
57
|
+
|
|
58
|
+
<!-- Bar 4: RouteLLM $0.27 = 171px → y=19, h=171 -->
|
|
59
|
+
<rect x="360" y="19" width="70" height="171" fill="#94a3b8" rx="3" opacity="0.35"/>
|
|
60
|
+
<text x="395" y="13" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="12" font-weight="600">$0.27</text>
|
|
61
|
+
<text x="395" y="207" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="10">RouteLLM</text>
|
|
62
|
+
</g>
|
|
63
|
+
|
|
64
|
+
<!-- A3M savings annotation -->
|
|
65
|
+
<text x="210" y="265" text-anchor="middle" fill="#64748b" font-family="system-ui,sans-serif" font-size="9">A3M is 3.5x cheaper than RouteLLM | 2.3x cheaper than Sqwish | 2.9x cheaper than Azure-Router</text>
|
|
66
|
+
</svg>
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 480 280" width="480" height="280">
|
|
2
|
+
<defs>
|
|
3
|
+
<linearGradient id="latBg" x1="0%" y1="0%" x2="0%" y2="100%">
|
|
4
|
+
<stop offset="0%" stop-color="#f8fafc"/>
|
|
5
|
+
<stop offset="100%" stop-color="#f1f5f9"/>
|
|
6
|
+
</linearGradient>
|
|
7
|
+
<marker id="arrow" markerWidth="6" markerHeight="6" refX="3" refY="3" orient="auto">
|
|
8
|
+
<path d="M0,0 L6,3 L0,6 Z" fill="#64748b"/>
|
|
9
|
+
</marker>
|
|
10
|
+
</defs>
|
|
11
|
+
|
|
12
|
+
<!-- Background -->
|
|
13
|
+
<rect width="480" height="280" fill="url(#latBg)"/>
|
|
14
|
+
|
|
15
|
+
<!-- Title -->
|
|
16
|
+
<text x="240" y="26" text-anchor="middle" fill="#111" font-family="system-ui,sans-serif" font-size="14" font-weight="600">Latency Breakdown — Routing Overhead</text>
|
|
17
|
+
<text x="240" y="43" text-anchor="middle" fill="#666" font-family="system-ui,sans-serif" font-size="10">Groq endpoint | Local measurement (May 2026)</text>
|
|
18
|
+
|
|
19
|
+
<!-- Chart area: y=58 (top) to y=248 (baseline 0ms) -->
|
|
20
|
+
<!-- Scale: 0.5px per ms. Max 380ms = 190px (fits 374ms bar) -->
|
|
21
|
+
|
|
22
|
+
<!-- Y-axis -->
|
|
23
|
+
<line x1="52" y1="58" x2="52" y2="248" stroke="#333" stroke-width="1"/>
|
|
24
|
+
<line x1="52" y1="248" x2="460" y2="248" stroke="#333" stroke-width="1"/>
|
|
25
|
+
|
|
26
|
+
<!-- Y-axis gridlines (every 50ms = 25px) -->
|
|
27
|
+
<g stroke="#e5e7eb" stroke-width="1" stroke-dasharray="4,3">
|
|
28
|
+
<line x1="52" y1="223" x2="460" y2="223"/> <!-- 50ms -->
|
|
29
|
+
<line x1="52" y1="198" x2="460" y2="198"/> <!-- 100ms -->
|
|
30
|
+
<line x1="52" y1="173" x2="460" y2="173"/> <!-- 150ms -->
|
|
31
|
+
<line x1="52" y1="148" x2="460" y2="148"/> <!-- 200ms -->
|
|
32
|
+
<line x1="52" y1="123" x2="460" y2="123"/> <!-- 250ms -->
|
|
33
|
+
<line x1="52" y1="98" x2="460" y2="98"/> <!-- 300ms -->
|
|
34
|
+
<line x1="52" y1="73" x2="460" y2="73"/> <!-- 350ms -->
|
|
35
|
+
<line x1="52" y1="58" x2="460" y2="58"/> <!-- 380ms -->
|
|
36
|
+
</g>
|
|
37
|
+
|
|
38
|
+
<!-- Y-axis labels (aligned with gridlines) -->
|
|
39
|
+
<g font-family="system-ui,sans-serif" font-size="10" fill="#64748b" text-anchor="end">
|
|
40
|
+
<text x="46" y="61">380ms</text>
|
|
41
|
+
<text x="46" y="76">350ms</text>
|
|
42
|
+
<text x="46" y="101">300ms</text>
|
|
43
|
+
<text x="46" y="126">250ms</text>
|
|
44
|
+
<text x="46" y="151">200ms</text>
|
|
45
|
+
<text x="46" y="176">150ms</text>
|
|
46
|
+
<text x="46" y="201">100ms</text>
|
|
47
|
+
<text x="46" y="226">50ms</text>
|
|
48
|
+
<text x="46" y="252">0ms</text>
|
|
49
|
+
</g>
|
|
50
|
+
|
|
51
|
+
<!-- === BARS === -->
|
|
52
|
+
<!-- Groq Direct: 138ms = 69px → top at y=179 -->
|
|
53
|
+
<rect x="70" y="179" width="70" height="69" fill="#94a3b8" rx="3"/>
|
|
54
|
+
<text x="105" y="173" text-anchor="middle" fill="#475569" font-family="system-ui,sans-serif" font-size="11" font-weight="600">138ms</text>
|
|
55
|
+
<text x="105" y="265" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">Direct Groq</text>
|
|
56
|
+
|
|
57
|
+
<!-- A3M Forced: 234ms = 117px → top at y=131 (138+96 overhead) -->
|
|
58
|
+
<rect x="185" y="131" width="70" height="117" fill="#3b82f6" rx="3"/>
|
|
59
|
+
<text x="220" y="125" text-anchor="middle" fill="#1d4ed8" font-family="system-ui,sans-serif" font-size="11" font-weight="600">234ms</text>
|
|
60
|
+
<text x="220" y="265" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">A3M Forced</text>
|
|
61
|
+
|
|
62
|
+
<!-- A3M Auto: 374ms = 187px → top at y=61 (234+140 overhead) -->
|
|
63
|
+
<rect x="300" y="61" width="70" height="187" fill="#22c55e" rx="3"/>
|
|
64
|
+
<text x="335" y="55" text-anchor="middle" fill="#15803d" font-family="system-ui,sans-serif" font-size="11" font-weight="600">374ms</text>
|
|
65
|
+
<text x="335" y="265" text-anchor="middle" fill="#333" font-family="system-ui,sans-serif" font-size="10">A3M Auto</text>
|
|
66
|
+
|
|
67
|
+
<!-- === OVERHEAD ANNOTATIONS === -->
|
|
68
|
+
|
|
69
|
+
<!-- +96ms bracket: from Groq top (y=179) to Forced top (y=131) = 48px = 96ms ✓ -->
|
|
70
|
+
<line x1="162" y1="179" x2="162" y2="131" stroke="#64748b" stroke-width="1.5"/>
|
|
71
|
+
<line x1="155" y1="179" x2="169" y2="179" stroke="#64748b" stroke-width="1.5"/>
|
|
72
|
+
<line x1="155" y1="131" x2="169" y2="131" stroke="#64748b" stroke-width="1.5"/>
|
|
73
|
+
<text x="177" y="159" fill="#64748b" font-family="system-ui,sans-serif" font-size="10" font-weight="600">+96ms</text>
|
|
74
|
+
<text x="177" y="170" fill="#64748b" font-family="system-ui,sans-serif" font-size="8">(proxy)</text>
|
|
75
|
+
|
|
76
|
+
<!-- +140ms bracket: from Forced top (y=131) to Auto top (y=61) = 70px = 140ms ✓ -->
|
|
77
|
+
<line x1="277" y1="131" x2="277" y2="61" stroke="#64748b" stroke-width="1.5"/>
|
|
78
|
+
<line x1="270" y1="131" x2="284" y2="131" stroke="#64748b" stroke-width="1.5"/>
|
|
79
|
+
<line x1="270" y1="61" x2="284" y2="61" stroke="#64748b" stroke-width="1.5"/>
|
|
80
|
+
<text x="292" y="100" fill="#64748b" font-family="system-ui,sans-serif" font-size="10" font-weight="600">+140ms</text>
|
|
81
|
+
<text x="292" y="111" fill="#64748b" font-family="system-ui,sans-serif" font-size="8">(routing)</text>
|
|
82
|
+
|
|
83
|
+
<!-- +236ms bracket: from Groq top (y=179) to Auto top (y=61) = 118px = 236ms ✓ -->
|
|
84
|
+
<line x1="388" y1="179" x2="388" y2="61" stroke="#22c55e" stroke-width="1.5"/>
|
|
85
|
+
<line x1="381" y1="179" x2="395" y2="179" stroke="#22c55e" stroke-width="1.5"/>
|
|
86
|
+
<line x1="381" y1="61" x2="395" y2="61" stroke="#22c55e" stroke-width="1.5"/>
|
|
87
|
+
<text x="403" y="124" fill="#15803d" font-family="system-ui,sans-serif" font-size="10" font-weight="600">+236ms</text>
|
|
88
|
+
<text x="403" y="135" fill="#15803d" font-family="system-ui,sans-serif" font-size="8">(total)</text>
|
|
89
|
+
|
|
90
|
+
<!-- Baseline annotation (dotted line at Groq level) -->
|
|
91
|
+
<line x1="52" y1="179" x2="460" y2="179" stroke="#94a3b8" stroke-width="1" stroke-dasharray="3,3"/>
|
|
92
|
+
<text x="54" y="183" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="8">138ms baseline</text>
|
|
93
|
+
|
|
94
|
+
<!-- Legend -->
|
|
95
|
+
<rect x="370" y="223" width="12" height="12" fill="#94a3b8" rx="2"/>
|
|
96
|
+
<text x="387" y="233" fill="#333" font-family="system-ui,sans-serif" font-size="9">Baseline</text>
|
|
97
|
+
<rect x="370" y="240" width="12" height="12" fill="#3b82f6" rx="2"/>
|
|
98
|
+
<text x="387" y="250" fill="#333" font-family="system-ui,sans-serif" font-size="9">Proxy overhead</text>
|
|
99
|
+
|
|
100
|
+
<!-- Note -->
|
|
101
|
+
<text x="240" y="276" text-anchor="middle" fill="#888" font-family="system-ui,sans-serif" font-size="9">236ms total overhead is less than the time it takes to blink (~300ms)</text>
|
|
102
|
+
</svg>
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 520 300" width="520" height="300">
|
|
2
|
+
<defs>
|
|
3
|
+
<linearGradient id="rankBg" x1="0%" y1="0%" x2="0%" y2="100%">
|
|
4
|
+
<stop offset="0%" stop-color="#0f172a"/>
|
|
5
|
+
<stop offset="100%" stop-color="#1e293b"/>
|
|
6
|
+
</linearGradient>
|
|
7
|
+
</defs>
|
|
8
|
+
|
|
9
|
+
<rect width="520" height="300" fill="url(#rankBg)" rx="8"/>
|
|
10
|
+
|
|
11
|
+
<!-- Title -->
|
|
12
|
+
<text x="260" y="28" text-anchor="middle" fill="#f1f5f9" font-family="system-ui,sans-serif" font-size="14" font-weight="600">A3M Router — RouterArena Evaluation (PR#144)</text>
|
|
13
|
+
<text x="260" y="44" text-anchor="middle" fill="#64748b" font-family="system-ui,sans-serif" font-size="10">ICLR 2025 | 8,400 queries | Evaluated against baseline routers</text>
|
|
14
|
+
|
|
15
|
+
<!-- Merged badge (PR#144 = premium tier, merged) -->
|
|
16
|
+
<rect x="378" y="14" width="58" height="18" rx="9" fill="#22c55e" fill-opacity="0.2" stroke="#22c55e" stroke-width="1"/>
|
|
17
|
+
<text x="407" y="26" text-anchor="middle" fill="#4ade80" font-family="system-ui,sans-serif" font-size="9" font-weight="600">Merged</text>
|
|
18
|
+
|
|
19
|
+
<!-- Headers -->
|
|
20
|
+
<text x="20" y="65" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="9" font-weight="600">RANK</text>
|
|
21
|
+
<text x="60" y="65" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="9" font-weight="600">ROUTER</text>
|
|
22
|
+
<text x="250" y="65" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="9" font-weight="600">SCORE</text>
|
|
23
|
+
<text x="310" y="65" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="9" font-weight="600">ACC</text>
|
|
24
|
+
<text x="365" y="65" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="9" font-weight="600">COST/1K</text>
|
|
25
|
+
<text x="430" y="65" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="9" font-weight="600">ROBUST</text>
|
|
26
|
+
|
|
27
|
+
<line x1="15" y1="70" x2="505" y2="70" stroke="#334155" stroke-width="1"/>
|
|
28
|
+
|
|
29
|
+
<!-- Row 1: A3M Router -->
|
|
30
|
+
<rect x="12" y="78" width="496" height="36" rx="4" fill="#166534" fill-opacity="0.3"/>
|
|
31
|
+
<text x="28" y="100" text-anchor="middle" fill="#fbbf24" font-family="system-ui,sans-serif" font-size="13" font-weight="700">1</text>
|
|
32
|
+
<text x="55" y="100" fill="#f1f5f9" font-family="system-ui,sans-serif" font-size="12" font-weight="600">A3M Router (this work)</text>
|
|
33
|
+
<text x="250" y="100" fill="#4ade80" font-family="system-ui,sans-serif" font-size="12" font-weight="700">0.9404</text>
|
|
34
|
+
<text x="310" y="100" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="12">96.77%</text>
|
|
35
|
+
<text x="365" y="100" fill="#4ade80" font-family="system-ui,sans-serif" font-size="12">$0.0768</text>
|
|
36
|
+
<text x="430" y="100" fill="#4ade80" font-family="system-ui,sans-serif" font-size="12">1.0000</text>
|
|
37
|
+
|
|
38
|
+
<!-- Row 2: Sqwish -->
|
|
39
|
+
<rect x="12" y="118" width="496" height="32" rx="4" fill="none"/>
|
|
40
|
+
<text x="28" y="138" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="12">2</text>
|
|
41
|
+
<text x="55" y="138" fill="#e2e8f0" font-family="system-ui,sans-serif" font-size="11">Sqwish</text>
|
|
42
|
+
<text x="250" y="138" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">0.7527</text>
|
|
43
|
+
<text x="310" y="138" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">75.27%</text>
|
|
44
|
+
<text x="365" y="138" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">$0.18</text>
|
|
45
|
+
<text x="430" y="138" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">—</text>
|
|
46
|
+
|
|
47
|
+
<!-- Row 3: Azure-Model-Router -->
|
|
48
|
+
<rect x="12" y="154" width="496" height="32" rx="4" fill="none"/>
|
|
49
|
+
<text x="28" y="174" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="12">3</text>
|
|
50
|
+
<text x="55" y="174" fill="#e2e8f0" font-family="system-ui,sans-serif" font-size="11">Azure-Model-Router</text>
|
|
51
|
+
<text x="250" y="174" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">0.7187</text>
|
|
52
|
+
<text x="310" y="174" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">71.87%</text>
|
|
53
|
+
<text x="365" y="174" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">$0.22</text>
|
|
54
|
+
<text x="430" y="174" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">—</text>
|
|
55
|
+
|
|
56
|
+
<!-- Row 4: GPT-5 -->
|
|
57
|
+
<rect x="12" y="190" width="496" height="32" rx="4" fill="none"/>
|
|
58
|
+
<text x="28" y="210" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="12">4</text>
|
|
59
|
+
<text x="55" y="210" fill="#e2e8f0" font-family="system-ui,sans-serif" font-size="11">GPT-5 (non-routing baseline)</text>
|
|
60
|
+
<text x="250" y="210" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">0.6432</text>
|
|
61
|
+
<text x="310" y="210" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">64.32%</text>
|
|
62
|
+
<text x="365" y="210" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">$10.02</text>
|
|
63
|
+
<text x="430" y="210" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">—</text>
|
|
64
|
+
|
|
65
|
+
<!-- Row 5: RouteLLM -->
|
|
66
|
+
<rect x="12" y="226" width="496" height="32" rx="4" fill="none"/>
|
|
67
|
+
<text x="28" y="246" text-anchor="middle" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="12">5</text>
|
|
68
|
+
<text x="55" y="246" fill="#e2e8f0" font-family="system-ui,sans-serif" font-size="11">RouteLLM</text>
|
|
69
|
+
<text x="250" y="246" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">0.4807</text>
|
|
70
|
+
<text x="310" y="246" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">48.07%</text>
|
|
71
|
+
<text x="365" y="246" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">$0.27</text>
|
|
72
|
+
<text x="430" y="246" fill="#94a3b8" font-family="system-ui,sans-serif" font-size="11">—</text>
|
|
73
|
+
|
|
74
|
+
<!-- Source note -->
|
|
75
|
+
<text x="260" y="280" text-anchor="middle" fill="#475569" font-family="system-ui,sans-serif" font-size="9">Source: RouteWorks/RouterArena#144 (ICLR 2025) | 8,400 queries | Score: RouterArena composite metric</text>
|
|
76
|
+
</svg>
|
|
@@ -1,56 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A3M Router — Comprehensive Local Benchmark Suite
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* 1. Routing Accuracy (tier assignment)
|
|
6
|
-
* 2. Memory Persistence (cross-session recall)
|
|
7
|
-
* 3. Robustness (failover, circuit breaker, provider health)
|
|
8
|
-
* 4. Cost Efficiency (vs always-premium baseline)
|
|
9
|
-
*
|
|
10
|
-
* Run: npx ts-node src/benchmark/comprehensive.ts
|
|
3
|
+
* Tests: Routing Accuracy, Memory Persistence, Robustness, Cost Efficiency
|
|
4
|
+
* Run: npx ts-node -P tsconfig.build.json src/benchmark/comprehensive.ts
|
|
11
5
|
*/
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
actualTier: string;
|
|
15
|
-
routedTier: string;
|
|
16
|
-
model: string;
|
|
17
|
-
complexity: number;
|
|
18
|
-
cost: number;
|
|
19
|
-
correct: boolean;
|
|
20
|
-
offByOne: boolean;
|
|
21
|
-
}
|
|
22
|
-
declare function runRoutingAccuracy(): {
|
|
23
|
-
results: RoutingResult[];
|
|
24
|
-
summary: any;
|
|
25
|
-
};
|
|
26
|
-
interface MemoryTestResult {
|
|
27
|
-
test: string;
|
|
28
|
-
passed: boolean;
|
|
29
|
-
details: string;
|
|
30
|
-
}
|
|
31
|
-
declare function runMemoryBenchmark(): {
|
|
32
|
-
results: MemoryTestResult[];
|
|
33
|
-
summary: any;
|
|
34
|
-
};
|
|
35
|
-
interface RobustnessResult {
|
|
36
|
-
test: string;
|
|
37
|
-
passed: boolean;
|
|
38
|
-
details: string;
|
|
39
|
-
}
|
|
40
|
-
declare function runRobustnessBenchmark(): {
|
|
41
|
-
results: RobustnessResult[];
|
|
42
|
-
summary: any;
|
|
43
|
-
};
|
|
44
|
-
interface CostResult {
|
|
45
|
-
scenario: string;
|
|
46
|
-
a3mCost: number;
|
|
47
|
-
alwaysPremiumCost: number;
|
|
48
|
-
savings: number;
|
|
49
|
-
savingsPct: number;
|
|
50
|
-
}
|
|
51
|
-
declare function runCostBenchmark(): {
|
|
52
|
-
results: CostResult[];
|
|
53
|
-
summary: any;
|
|
54
|
-
};
|
|
55
|
-
export declare function runComprehensiveBenchmark(): void;
|
|
56
|
-
export { runRoutingAccuracy, runMemoryBenchmark, runRobustnessBenchmark, runCostBenchmark };
|
|
6
|
+
export {};
|
|
7
|
+
//# sourceMappingURL=comprehensive.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"comprehensive.d.ts","sourceRoot":"","sources":["../../src/benchmark/comprehensive.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|