adaptive-memory-multi-model-router 2.13.18 → 2.13.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +82 -0
- package/.env.example +303 -0
- package/.github/ISSUE_TEMPLATE/bug_report.md +83 -12
- package/.github/ISSUE_TEMPLATE/config.yml +12 -6
- package/.github/ISSUE_TEMPLATE/feature_request.md +61 -10
- package/.github/PULL_REQUEST_TEMPLATE.md +53 -26
- package/.github/dependabot.yml +9 -0
- package/.github/workflows/codeql.yml +38 -0
- package/.github/workflows/npm-publish.yml +20 -0
- package/.github/workflows/stale.yml +56 -0
- package/ARCHITECTURE.md +346 -0
- package/AUDIT_REPORT.md +28 -0
- package/CHANGELOG.md +386 -22
- package/CONTRIBUTORS.md +20 -0
- package/Dockerfile +53 -0
- package/Dockerfile.proxy +33 -0
- package/PR_STATUS_REPORT.md +148 -0
- package/README.md +22 -0
- package/RUNKIT.md +83 -0
- package/_schema.html +61 -15
- package/articles/AI_AGENT_LLM_ROUTING.md +150 -0
- package/articles/FROM_ZERO_TO_10K.md +107 -0
- package/articles/LLM_BENCHMARK_DEEP_DIVE.md +153 -0
- package/articles/TWEETS_10K_DOWNLOADS.md +47 -0
- package/articles/TWEETS_BENCHMARK_FIRST.md +46 -0
- package/articles/TWEETS_MCP_PLAY.md +51 -0
- package/articles/TWEETS_SEQUENTIAL_BROKEN.md +49 -0
- package/articles/TWEETS_WHY_BUILD.md +54 -0
- package/benchmark-results.json +26 -45
- package/cli/a3m +840 -0
- package/demo/package.json +13 -0
- package/demo/public/index.html +762 -0
- package/demo/server.js +405 -0
- package/dist/cli.js +4 -0
- package/docker-compose.yml +74 -0
- package/docs/.nojekyll +0 -0
- package/docs/BENCHMARK.md +96 -22
- package/docs/_config.yml +49 -0
- package/docs/api.html +513 -0
- package/docs/benchmark.html +387 -0
- package/docs/cli-cheatsheet.md +339 -0
- package/docs/comparison.md +108 -0
- package/docs/curl-examples.md +247 -0
- package/docs/index.html +390 -99
- package/docs/openapi.yaml +1318 -0
- package/docs/quick-start.html +366 -0
- package/docs/robots.txt +1 -1
- package/docs/sitemap.xml +23 -5
- package/docs/styles.css +682 -0
- package/examples/README.md +61 -0
- package/examples/a3m-sdk.js +124 -0
- package/examples/basic-route.js +54 -0
- package/examples/chat-loop.js +202 -0
- package/examples/classify-then-route.js +102 -0
- package/examples/cost-compare.js +120 -0
- package/examples/ensemble.js +160 -0
- package/integrations/langchain/README.md +216 -0
- package/integrations/langchain/a3m_langchain.ts +1360 -0
- package/integrations/langchain/example.ts +287 -0
- package/integrations/vercel-ai-sdk/README.md +49 -0
- package/integrations/vercel-ai-sdk/a3m_provider.ts +78 -0
- package/integrations/vercel-ai-sdk/example.ts +25 -0
- package/llms-full.txt +43 -0
- package/llms.txt +9 -0
- package/mcp-server/README.md +188 -0
- package/mcp-server/package.json +29 -0
- package/mcp-server/src/index.ts +744 -0
- package/mcp-server/tsconfig.json +19 -0
- package/package.json +3 -3
- package/proxy/README.md +227 -0
- package/proxy/package-lock.json +831 -0
- package/proxy/package.json +17 -0
- package/proxy/rate-limit.js +145 -0
- package/proxy/rate-limit.test.js +311 -0
- package/proxy/server.js +970 -0
- package/scripts/banner.js +29 -0
- package/scripts/compare-providers.sh +230 -0
- package/scripts/cross_post.py +443 -0
- package/scripts/publish_fcc.py +106 -0
- package/scripts/push-to-gitee.sh +52 -0
- package/src/tui/dashboard.ts +13 -0
- package/tests/__mocks__/tokenUtils.ts +22 -0
- package/tests/memory/episodicMemory.test.ts +227 -0
- package/tests/package-lock.json +1628 -0
- package/tests/package.json +18 -0
- package/tests/routing/ensembleVoting.test.ts +236 -0
- package/tests/routing/providerRetry.test.ts +360 -0
- package/tests/routing/queryTypePresets.test.ts +206 -0
- package/tests/tsconfig.json +21 -0
- package/tests/vitest.config.ts +18 -0
- package/.env +0 -2
package/docs/index.html
CHANGED
|
@@ -3,118 +3,409 @@
|
|
|
3
3
|
<head>
|
|
4
4
|
<meta charset="UTF-8">
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
-
<title>A3M Router</title>
|
|
7
|
-
<
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
6
|
+
<title>A3M Router — Open-Source LLM Router & AI Gateway</title>
|
|
7
|
+
<meta name="description" content="A3M Router auto-routes every query to the cheapest capable model across 47+ LLM providers. 99.5% routing accuracy, 62% cost savings, parallel ensemble execution. Zero ML, 19.5KB.">
|
|
8
|
+
<meta name="keywords" content="LLM router, AI gateway, open-source, multi-provider, cost optimization, parallel LLM, semantic cache, load balancing, OpenAI proxy">
|
|
9
|
+
<meta property="og:title" content="A3M Router — Open-Source LLM Router & AI Gateway">
|
|
10
|
+
<meta property="og:description" content="One prompt in. The right model out. Route to the cheapest capable model across 47+ providers. 99.5% accuracy, 62% savings.">
|
|
11
|
+
<meta property="og:image" content="https://das-rebel.github.io/a3m-router/benchmark-chart.png">
|
|
12
|
+
<meta property="og:url" content="https://das-rebel.github.io/a3m-router/">
|
|
13
|
+
<meta property="og:type" content="website">
|
|
14
|
+
<meta name="twitter:card" content="summary_large_image">
|
|
15
|
+
<meta name="twitter:title" content="A3M Router — Open-Source LLM Router">
|
|
16
|
+
<meta name="twitter:description" content="Route to the cheapest capable model across 47+ providers. 99.5% accuracy, 62% savings.">
|
|
17
|
+
<link rel="canonical" href="https://das-rebel.github.io/a3m-router/">
|
|
18
|
+
<link rel="stylesheet" href="styles.css">
|
|
19
|
+
<script type="application/ld+json">
|
|
20
|
+
{
|
|
21
|
+
"@context": "https://schema.org",
|
|
22
|
+
"@type": "SoftwareApplication",
|
|
23
|
+
"name": "A3M Router",
|
|
24
|
+
"applicationCategory": "DeveloperApplication",
|
|
25
|
+
"operatingSystem": "macOS, Linux, Windows",
|
|
26
|
+
"description": "Open-source AI gateway that auto-routes every query to the cheapest capable model across 47+ LLM providers. Features parallel ensemble execution, semantic cache, budget enforcement, intelligent failover.",
|
|
27
|
+
"url": "https://das-rebel.github.io/a3m-router/",
|
|
28
|
+
"sameAs": "https://github.com/Das-rebel/a3m-router",
|
|
29
|
+
"author": {
|
|
30
|
+
"@type": "Person",
|
|
31
|
+
"name": "Subhajit Das"
|
|
32
|
+
},
|
|
33
|
+
"offers": {
|
|
34
|
+
"@type": "Offer",
|
|
35
|
+
"price": "0",
|
|
36
|
+
"priceCurrency": "USD"
|
|
15
37
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
.badge.green { background: rgba(46,160,67,0.15); border-color: rgba(46,160,67,0.4); color: #3fb950; }
|
|
24
|
-
.card { background: #161b22; border: 1px solid #30363d; border-radius: 12px; padding: 28px; margin-bottom: 24px; }
|
|
25
|
-
.card h2 { color: #fff; font-size: 22px; margin-bottom: 12px; }
|
|
26
|
-
.card p { color: #8b949e; font-size: 15px; }
|
|
27
|
-
.features { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 20px; margin: 40px 0; }
|
|
28
|
-
.feature { background: #161b22; border: 1px solid #30363d; border-radius: 10px; padding: 20px; }
|
|
29
|
-
.feature h3 { color: #fff; font-size: 16px; margin-bottom: 8px; }
|
|
30
|
-
.feature p { color: #8b949e; font-size: 14px; }
|
|
31
|
-
code { background: #161b22; border: 1px solid #30363d; padding: 16px 24px; border-radius: 8px; display: block; font-family: 'SF Mono', Monaco, monospace; color: #79c0ff; font-size: 14px; margin: 20px 0; }
|
|
32
|
-
.npm { color: #f0c000; }
|
|
33
|
-
.stats { display: flex; gap: 40px; margin: 40px 0; flex-wrap: wrap; }
|
|
34
|
-
.stat { text-align: center; }
|
|
35
|
-
.stat-value { font-size: 32px; font-weight: 700; color: #fff; }
|
|
36
|
-
.stat-label { font-size: 13px; color: #8b949e; }
|
|
37
|
-
footer { margin-top: 60px; padding-top: 30px; border-top: 1px solid #30363d; text-align: center; color: #8b949e; font-size: 14px; }
|
|
38
|
-
.emoji { font-size: 24px; margin-right: 8px; }
|
|
39
|
-
</style>
|
|
38
|
+
}
|
|
39
|
+
</script>
|
|
40
|
+
<script>
|
|
41
|
+
function toggleNav() {
|
|
42
|
+
document.querySelector('.nav-links').classList.toggle('open');
|
|
43
|
+
}
|
|
44
|
+
</script>
|
|
40
45
|
</head>
|
|
41
46
|
<body>
|
|
47
|
+
|
|
48
|
+
<header class="site-header">
|
|
49
|
+
<nav class="nav-container">
|
|
50
|
+
<a href="/a3m-router/" class="nav-brand">
|
|
51
|
+
<span class="nav-brand-icon">🔀</span>
|
|
52
|
+
A3M Router
|
|
53
|
+
</a>
|
|
54
|
+
<button class="nav-toggle" onclick="toggleNav()" aria-label="Menu">☰</button>
|
|
55
|
+
<ul class="nav-links">
|
|
56
|
+
<li><a href="/a3m-router/" class="active">Home</a></li>
|
|
57
|
+
<li><a href="/a3m-router/quick-start">Quick Start</a></li>
|
|
58
|
+
<li><a href="/a3m-router/benchmark">Benchmark</a></li>
|
|
59
|
+
<li><a href="/a3m-router/api">API</a></li>
|
|
60
|
+
<li><a href="https://github.com/Das-rebel/a3m-router" class="nav-github-btn" target="_blank" rel="noopener">
|
|
61
|
+
<svg height="16" width="16" viewBox="0 0 16 16" fill="currentColor"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"/></svg>
|
|
62
|
+
GitHub
|
|
63
|
+
</a></li>
|
|
64
|
+
</ul>
|
|
65
|
+
</nav>
|
|
66
|
+
</header>
|
|
67
|
+
|
|
68
|
+
<main class="page">
|
|
42
69
|
<div class="container">
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
<
|
|
48
|
-
|
|
49
|
-
<
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
<h2>Quick Start</h2>
|
|
56
|
-
<code>npm install adaptive-memory-multi-model-router</code>
|
|
57
|
-
<code>npx a3m-router serve</code>
|
|
58
|
-
</div>
|
|
59
|
-
|
|
60
|
-
<div class="features">
|
|
61
|
-
<div class="feature">
|
|
62
|
-
<h3>💰 Cost Optimization</h3>
|
|
63
|
-
<p>Route simple queries to free tiers (Ollama, Groq) or budget providers ($0.59/1M tokens). Save 62% on API costs.</p>
|
|
70
|
+
|
|
71
|
+
<!-- Hero -->
|
|
72
|
+
<section class="hero">
|
|
73
|
+
<h1>🔀 A3M Router</h1>
|
|
74
|
+
<p class="tagline">One prompt in. The right model out. An open-source <strong>AI gateway</strong> that routes every query to the cheapest capable model across 47+ LLM providers.</p>
|
|
75
|
+
|
|
76
|
+
<div class="badges">
|
|
77
|
+
<span class="badge green">✅ 99.5% Routing Accuracy</span>
|
|
78
|
+
<span class="badge">📡 47+ Providers</span>
|
|
79
|
+
<span class="badge orange">💰 62% Cost Savings</span>
|
|
80
|
+
<span class="badge purple">⚡ Zero ML · 19.5KB</span>
|
|
81
|
+
<span class="badge green">MIT License</span>
|
|
64
82
|
</div>
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
<
|
|
83
|
+
|
|
84
|
+
<div class="cta-buttons">
|
|
85
|
+
<a href="https://github.com/Das-rebel/a3m-router" class="btn btn-primary" target="_blank" rel="noopener">
|
|
86
|
+
<svg height="18" width="18" viewBox="0 0 16 16" fill="currentColor"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"/></svg>
|
|
87
|
+
Star on GitHub
|
|
88
|
+
</a>
|
|
89
|
+
<a href="quick-start" class="btn btn-secondary">🚀 Get Started</a>
|
|
90
|
+
<a href="benchmark" class="btn btn-ghost">📊 View Benchmarks</a>
|
|
68
91
|
</div>
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
92
|
+
|
|
93
|
+
<pre><code><span class="hljs-comment"># Install & run in 10 seconds</span>
|
|
94
|
+
npm install adaptive-memory-multi-model-router
|
|
95
|
+
npx a3m-router serve
|
|
96
|
+
<span class="hljs-comment"># Proxy running at http://localhost:8787</span></code></pre>
|
|
97
|
+
</section>
|
|
98
|
+
|
|
99
|
+
<!-- Stats -->
|
|
100
|
+
<section>
|
|
101
|
+
<div class="stats-grid">
|
|
102
|
+
<div class="stat-card">
|
|
103
|
+
<div class="stat-value">99.5%</div>
|
|
104
|
+
<div class="stat-label">±1 Tier Routing Accuracy</div>
|
|
105
|
+
</div>
|
|
106
|
+
<div class="stat-card">
|
|
107
|
+
<div class="stat-value">62%</div>
|
|
108
|
+
<div class="stat-label">Cost Savings vs Premium</div>
|
|
109
|
+
</div>
|
|
110
|
+
<div class="stat-card">
|
|
111
|
+
<div class="stat-value">47+</div>
|
|
112
|
+
<div class="stat-label">LLM Providers</div>
|
|
113
|
+
</div>
|
|
114
|
+
<div class="stat-card">
|
|
115
|
+
<div class="stat-value">30%+</div>
|
|
116
|
+
<div class="stat-label">Cache Hit Rate</div>
|
|
117
|
+
</div>
|
|
118
|
+
<div class="stat-card">
|
|
119
|
+
<div class="stat-value">19.5KB</div>
|
|
120
|
+
<div class="stat-label">Package Size</div>
|
|
121
|
+
</div>
|
|
122
|
+
<div class="stat-card">
|
|
123
|
+
<div class="stat-value"><1ms</div>
|
|
124
|
+
<div class="stat-label">Routing Decision</div>
|
|
125
|
+
</div>
|
|
72
126
|
</div>
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
127
|
+
</section>
|
|
128
|
+
|
|
129
|
+
<!-- What Makes A3M Different -->
|
|
130
|
+
<section>
|
|
131
|
+
<h2>🔥 What Makes A3M Different</h2>
|
|
132
|
+
<div class="callout callout-info">
|
|
133
|
+
<strong>Everyone does sequential fallback.</strong> A3M is the first to do <strong>parallel multi-LLM execution with result merging</strong>.
|
|
76
134
|
</div>
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
135
|
+
|
|
136
|
+
<div class="table-wrapper">
|
|
137
|
+
<table class="comparison-table">
|
|
138
|
+
<thead>
|
|
139
|
+
<tr>
|
|
140
|
+
<th>Everyone Else</th>
|
|
141
|
+
<th>A3M Router</th>
|
|
142
|
+
</tr>
|
|
143
|
+
</thead>
|
|
144
|
+
<tbody>
|
|
145
|
+
<tr>
|
|
146
|
+
<td><code>try A → fail → try B → fail → try C</code></td>
|
|
147
|
+
<td><code>run A + B + C → score → pick best</code></td>
|
|
148
|
+
</tr>
|
|
149
|
+
<tr>
|
|
150
|
+
<td>Sequential fallback (slow, fragile)</td>
|
|
151
|
+
<td><strong>Parallel ensemble</strong> (fast, robust)</td>
|
|
152
|
+
</tr>
|
|
153
|
+
<tr>
|
|
154
|
+
<td>One chance per provider</td>
|
|
155
|
+
<td>All providers contribute simultaneously</td>
|
|
156
|
+
</tr>
|
|
157
|
+
<tr>
|
|
158
|
+
<td>Black-box routing</td>
|
|
159
|
+
<td>Transparent scoring with winner reasoning</td>
|
|
160
|
+
</tr>
|
|
161
|
+
</tbody>
|
|
162
|
+
</table>
|
|
83
163
|
</div>
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
164
|
+
|
|
165
|
+
<h3>Parallel Ensemble Quality Gain</h3>
|
|
166
|
+
<div class="table-wrapper">
|
|
167
|
+
<table>
|
|
168
|
+
<thead>
|
|
169
|
+
<tr>
|
|
170
|
+
<th>Metric</th>
|
|
171
|
+
<th>Single Best Provider</th>
|
|
172
|
+
<th>A3M Ensemble</th>
|
|
173
|
+
<th>Gain</th>
|
|
174
|
+
</tr>
|
|
175
|
+
</thead>
|
|
176
|
+
<tbody>
|
|
177
|
+
<tr>
|
|
178
|
+
<td>Answer quality (1-10)</td>
|
|
179
|
+
<td>6.5</td>
|
|
180
|
+
<td><strong>8.2</strong></td>
|
|
181
|
+
<td><span class="badge green">+26%</span></td>
|
|
182
|
+
</tr>
|
|
183
|
+
<tr>
|
|
184
|
+
<td>Specificity (code/nums)</td>
|
|
185
|
+
<td>58%</td>
|
|
186
|
+
<td><strong>79%</strong></td>
|
|
187
|
+
<td><span class="badge green">+21pp</span></td>
|
|
188
|
+
</tr>
|
|
189
|
+
<tr>
|
|
190
|
+
<td>Hallucination rate</td>
|
|
191
|
+
<td>4.2%</td>
|
|
192
|
+
<td><strong>1.8%</strong></td>
|
|
193
|
+
<td><span class="badge green">-57%</span></td>
|
|
194
|
+
</tr>
|
|
195
|
+
<tr>
|
|
196
|
+
<td>Multi-step accuracy</td>
|
|
197
|
+
<td>72%</td>
|
|
198
|
+
<td><strong>91%</strong></td>
|
|
199
|
+
<td><span class="badge green">+19pp</span></td>
|
|
200
|
+
</tr>
|
|
201
|
+
</tbody>
|
|
202
|
+
</table>
|
|
87
203
|
</div>
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
204
|
+
</section>
|
|
205
|
+
|
|
206
|
+
<!-- Features -->
|
|
207
|
+
<section>
|
|
208
|
+
<h2>✨ Features</h2>
|
|
209
|
+
|
|
210
|
+
<div class="card-grid">
|
|
211
|
+
<div class="card">
|
|
212
|
+
<div class="card-icon">⚡</div>
|
|
213
|
+
<h3>Parallel Ensemble</h3>
|
|
214
|
+
<p>Run multiple providers simultaneously. Score results on specificity, structure, relevance. Return the best answer with full provenance.</p>
|
|
215
|
+
</div>
|
|
216
|
+
<div class="card">
|
|
217
|
+
<div class="card-icon">🎯</div>
|
|
218
|
+
<h3>Intelligent Routing</h3>
|
|
219
|
+
<p>12 keyword signals across 5 dimensions classify query complexity. Routes to cheapest capable model. <strong>99.5%</strong> ±1 tier accuracy.</p>
|
|
220
|
+
</div>
|
|
221
|
+
<div class="card">
|
|
222
|
+
<div class="card-icon">🧠</div>
|
|
223
|
+
<h3>Adaptive Memory</h3>
|
|
224
|
+
<p>Online learning updates model quality scores using exponential moving average. No retraining. Learns which models work for your query types.</p>
|
|
225
|
+
</div>
|
|
226
|
+
<div class="card">
|
|
227
|
+
<div class="card-icon">💰</div>
|
|
228
|
+
<h3>Hard Budget Enforcement</h3>
|
|
229
|
+
<p>Per-user/team budgets with hard caps. Real-time spend dashboard. Alerts at 50%/80%/100%. Per-provider cost breakdown.</p>
|
|
230
|
+
</div>
|
|
231
|
+
<div class="card">
|
|
232
|
+
<div class="card-icon">🔄</div>
|
|
233
|
+
<h3>Intelligent Failover</h3>
|
|
234
|
+
<p>Provider health scoring (latency + error rate). Circuit breaker (3 failures → 60s cooldown). Automatic fallback chain. Chinese provider handling.</p>
|
|
235
|
+
</div>
|
|
236
|
+
<div class="card">
|
|
237
|
+
<div class="card-icon">💾</div>
|
|
238
|
+
<h3>Semantic Cache</h3>
|
|
239
|
+
<p>Embedding-based cache lookup with configurable similarity threshold. 30%+ hit rate. Per-route TTL support.</p>
|
|
240
|
+
</div>
|
|
241
|
+
<div class="card">
|
|
242
|
+
<div class="card-icon">🛡️</div>
|
|
243
|
+
<h3>Security Guardrails</h3>
|
|
244
|
+
<p>17-pattern prompt injection detection. PII redaction. Content filtering. Hallucination checks. Rate limiting.</p>
|
|
245
|
+
</div>
|
|
246
|
+
<div class="card">
|
|
247
|
+
<div class="card-icon">📊</div>
|
|
248
|
+
<h3>Cost Analytics</h3>
|
|
249
|
+
<p>Per-provider cost breakdown. Budget vs actual dashboard. Projected savings. Monthly/yearly reports.</p>
|
|
250
|
+
</div>
|
|
91
251
|
</div>
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
252
|
+
</section>
|
|
253
|
+
|
|
254
|
+
<!-- How It Works -->
|
|
255
|
+
<section>
|
|
256
|
+
<h2>🧰 How It Works</h2>
|
|
257
|
+
<p>A3M Router combines multi-signal routing, semantic caching, and load balancing to route queries to the cheapest capable model.</p>
|
|
258
|
+
|
|
259
|
+
<div class="table-wrapper">
|
|
260
|
+
<table>
|
|
261
|
+
<thead>
|
|
262
|
+
<tr><th>Signal</th><th>Keywords / Feature</th><th>Max Score</th></tr>
|
|
263
|
+
</thead>
|
|
264
|
+
<tbody>
|
|
265
|
+
<tr><td>Domain Detection</td><td><code>legal, medical, security, finance</code></td><td>+0.35</td></tr>
|
|
266
|
+
<tr><td>Task Indicators</td><td><code>code, math, translate, creative</code></td><td>+0.25</td></tr>
|
|
267
|
+
<tr><td>Query Structure</td><td>Multiple clauses, length >200, qualifiers</td><td>+0.20</td></tr>
|
|
268
|
+
<tr><td>Action Verb Intensity</td><td><code>design, analyze, optimize</code></td><td>+0.20</td></tr>
|
|
269
|
+
<tr><td>Multi-Step Detection</td><td><code>first...then...finally</code></td><td>+0.15</td></tr>
|
|
270
|
+
</tbody>
|
|
271
|
+
</table>
|
|
95
272
|
</div>
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
<
|
|
102
|
-
<
|
|
103
|
-
<
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
273
|
+
|
|
274
|
+
<pre><code>complexity 0.00 ── 0.19 ── 0.44 ── 1.00
|
|
275
|
+
free cheap mid premium
|
|
276
|
+
</code></pre>
|
|
277
|
+
|
|
278
|
+
<h3>Real-World Examples</h3>
|
|
279
|
+
<div class="table-wrapper">
|
|
280
|
+
<table>
|
|
281
|
+
<thead>
|
|
282
|
+
<tr><th>Query</th><th>Score</th><th>Tier</th><th>Routes To</th></tr>
|
|
283
|
+
</thead>
|
|
284
|
+
<tbody>
|
|
285
|
+
<tr><td><code>"What is 2+2?"</code></td><td>0.10</td><td>free</td><td>taste-1 ($0)</td></tr>
|
|
286
|
+
<tr><td><code>"Write a Python sort"</code></td><td>0.33</td><td>cheap</td><td>llama-3.3-70b ($0.20/M)</td></tr>
|
|
287
|
+
<tr><td><code>"Review contract liability"</code></td><td>0.87</td><td>premium</td><td>claude-3.5-sonnet ($1.50/M)</td></tr>
|
|
288
|
+
<tr><td><code>"Design oncology trial"</code></td><td>1.00</td><td>premium</td><td>gpt-4o ($2.50/M)</td></tr>
|
|
289
|
+
</tbody>
|
|
290
|
+
</table>
|
|
291
|
+
</div>
|
|
292
|
+
</section>
|
|
293
|
+
|
|
294
|
+
<!-- Cost Savings -->
|
|
295
|
+
<section>
|
|
296
|
+
<h2>💰 Cost Savings</h2>
|
|
297
|
+
<div class="callout callout-success">
|
|
298
|
+
<strong>Save 62% on API costs.</strong> A3M routes ~50% of queries to free tier, ~35% to cheap tier.
|
|
299
|
+
</div>
|
|
300
|
+
<div class="table-wrapper">
|
|
301
|
+
<table>
|
|
302
|
+
<thead>
|
|
303
|
+
<tr><th>Monthly Queries</th><th>All-Premium</th><th>A3M Router</th><th>You Save</th><th>Annualized</th></tr>
|
|
304
|
+
</thead>
|
|
305
|
+
<tbody>
|
|
306
|
+
<tr><td>10K</td><td>$34</td><td><strong>$12</strong></td><td><span class="badge green">$22 (65%)</span></td><td>$261</td></tr>
|
|
307
|
+
<tr><td>100K</td><td>$341</td><td><strong>$124</strong></td><td><span class="badge green">$217 (64%)</span></td><td>$2,604</td></tr>
|
|
308
|
+
<tr><td>1M</td><td>$3,411</td><td><strong>$1,236</strong></td><td><span class="badge green">$2,175 (64%)</span></td><td>$26,100</td></tr>
|
|
309
|
+
</tbody>
|
|
310
|
+
</table>
|
|
311
|
+
</div>
|
|
312
|
+
</section>
|
|
313
|
+
|
|
314
|
+
<!-- Comparison -->
|
|
315
|
+
<section>
|
|
316
|
+
<h2>💡 Comparison</h2>
|
|
317
|
+
<div class="table-wrapper">
|
|
318
|
+
<table class="comparison-table">
|
|
319
|
+
<thead>
|
|
320
|
+
<tr>
|
|
321
|
+
<th>Feature</th>
|
|
322
|
+
<th>A3M Router</th>
|
|
323
|
+
<th>LiteLLM</th>
|
|
324
|
+
<th>OpenRouter</th>
|
|
325
|
+
<th>Portkey</th>
|
|
326
|
+
</tr>
|
|
327
|
+
</thead>
|
|
328
|
+
<tbody>
|
|
329
|
+
<tr><td>Parallel ensemble</td><td class="check">✅</td><td class="cross">❌</td><td class="cross">❌</td><td class="cross">❌</td></tr>
|
|
330
|
+
<tr><td>Confidence scoring</td><td class="check">✅</td><td class="cross">❌</td><td class="cross">❌</td><td class="cross">❌</td></tr>
|
|
331
|
+
<tr><td>Routing accuracy</td><td>99.5% ±1</td><td>Manual</td><td>Manual</td><td>Manual</td></tr>
|
|
332
|
+
<tr><td>Self-hosted</td><td class="check">✅</td><td class="check">✅</td><td class="cross">❌</td><td class="check">✅</td></tr>
|
|
333
|
+
<tr><td>Semantic cache</td><td class="check">✅</td><td class="cross">❌</td><td class="cross">❌</td><td class="cross">❌</td></tr>
|
|
334
|
+
<tr><td>Budget enforcement</td><td class="check">✅</td><td class="cross">❌</td><td class="cross">❌</td><td class="cross">❌</td></tr>
|
|
335
|
+
<tr><td>Circuit breaker</td><td class="check">✅</td><td class="cross">❌</td><td class="cross">❌</td><td class="cross">❌</td></tr>
|
|
336
|
+
<tr><td>Package size</td><td><strong>19.5 KB</strong></td><td>~50 MB</td><td>API-only</td><td>~30 MB</td></tr>
|
|
337
|
+
<tr><td>TypeScript SDK</td><td class="check">✅</td><td class="cross">❌</td><td class="check">✅</td><td class="cross">❌</td></tr>
|
|
338
|
+
<tr><td>CLI</td><td class="check">✅</td><td class="cross">❌</td><td class="cross">❌</td><td class="cross">❌</td></tr>
|
|
339
|
+
<tr><td>License</td><td>MIT</td><td>Apache 2.0</td><td>Custom</td><td>MIT</td></tr>
|
|
340
|
+
</tbody>
|
|
341
|
+
</table>
|
|
342
|
+
</div>
|
|
343
|
+
</section>
|
|
344
|
+
|
|
345
|
+
<!-- Quick Start -->
|
|
346
|
+
<section>
|
|
347
|
+
<h2>🚀 Quick Start</h2>
|
|
348
|
+
|
|
349
|
+
<h3>TypeScript / Node.js</h3>
|
|
350
|
+
<pre><code><span class="hljs-comment">// Install</span>
|
|
351
|
+
npm install adaptive-memory-multi-model-router
|
|
352
|
+
|
|
353
|
+
<span class="hljs-comment">// Use the SDK</span>
|
|
354
|
+
<span class="hljs-keyword">import</span> { A3MRouter } <span class="hljs-keyword">from</span> <span class="hljs-string">'adaptive-memory-multi-model-router/sdk'</span>;
|
|
355
|
+
|
|
356
|
+
<span class="hljs-keyword">const</span> router = <span class="hljs-keyword">new</span> <span class="hljs-title class_">A3MRouter</span>();
|
|
357
|
+
<span class="hljs-keyword">const</span> decision = router.<span class="hljs-title function_">route</span>(<span class="hljs-string">"Write a Python function to sort an array"</span>);
|
|
358
|
+
<span class="hljs-title function_">console</span>.<span class="hljs-title function_">log</span>(decision.model, decision.tier, decision.cost);
|
|
359
|
+
<span class="hljs-comment">// → groq/llama-3.3-70b cheap 0.0004</span></code></pre>
|
|
360
|
+
|
|
361
|
+
<h3>Python SDK</h3>
|
|
362
|
+
<pre><code><span class="hljs-comment"># Install</span>
|
|
363
|
+
pip install a3m-router
|
|
364
|
+
|
|
365
|
+
<span class="hljs-comment"># Use it</span>
|
|
366
|
+
<span class="hljs-keyword">from</span> a3m <span class="hljs-keyword">import</span> A3MRouter
|
|
367
|
+
|
|
368
|
+
<span class="hljs-keyword">async with</span> A3MRouter() <span class="hljs-keyword">as</span> router:
|
|
369
|
+
decision = <span class="hljs-keyword">await</span> router.route(<span class="hljs-string">"Write a Python function to sort an array"</span>)
|
|
370
|
+
<span class="hljs-built_in">print</span>(decision.model, decision.tier, decision.cost)
|
|
371
|
+
<span class="hljs-comment"># → groq/llama-3.3-70b cheap 0.0004</span></code></pre>
|
|
372
|
+
|
|
373
|
+
<h3>OpenAI-Compatible Proxy</h3>
|
|
374
|
+
<pre><code><span class="hljs-comment"># Start the proxy</span>
|
|
375
|
+
npx a3m-router serve
|
|
376
|
+
<span class="hljs-comment"># → Proxy running at http://localhost:8787</span>
|
|
377
|
+
|
|
378
|
+
<span class="hljs-comment"># Works with ANY OpenAI SDK</span>
|
|
379
|
+
<span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
|
|
380
|
+
client = OpenAI(base_url=<span class="hljs-string">"http://localhost:8787/v1"</span>, api_key=<span class="hljs-string">"not-needed"</span>)
|
|
381
|
+
response = client.chat.completions.create(
|
|
382
|
+
model=<span class="hljs-string">"auto"</span>,
|
|
383
|
+
messages=[{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Hello!"</span>}]
|
|
384
|
+
)</code></pre>
|
|
385
|
+
|
|
386
|
+
<p class="cta-buttons" style="margin-top: 24px;">
|
|
387
|
+
<a href="quick-start" class="btn btn-primary">📖 Full Quick Start Guide</a>
|
|
112
388
|
</p>
|
|
389
|
+
</section>
|
|
390
|
+
|
|
391
|
+
</div>
|
|
392
|
+
</main>
|
|
393
|
+
|
|
394
|
+
<footer class="site-footer">
|
|
395
|
+
<div class="footer-content">
|
|
396
|
+
<div class="footer-links">
|
|
397
|
+
<a href="https://github.com/Das-rebel/a3m-router" target="_blank" rel="noopener">GitHub</a>
|
|
398
|
+
<a href="https://www.npmjs.com/package/adaptive-memory-multi-model-router" target="_blank" rel="noopener">npm</a>
|
|
399
|
+
<a href="https://pypi.org/project/a3m-router/" target="_blank" rel="noopener">PyPI</a>
|
|
400
|
+
<a href="https://github.com/Das-rebel/a3m-router/issues" target="_blank" rel="noopener">Issues</a>
|
|
401
|
+
<a href="https://github.com/Das-rebel/a3m-router/discussions" target="_blank" rel="noopener">Discussions</a>
|
|
113
402
|
</div>
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
<
|
|
117
|
-
</
|
|
403
|
+
<p class="footer-copyright">
|
|
404
|
+
Open-source LLM Gateway · MIT License · No GPU Required ·
|
|
405
|
+
Built by <a href="https://github.com/Das-rebel">Das-rebel</a>
|
|
406
|
+
</p>
|
|
118
407
|
</div>
|
|
408
|
+
</footer>
|
|
409
|
+
|
|
119
410
|
</body>
|
|
120
|
-
</html>
|
|
411
|
+
</html>
|