adaptive-memory-multi-model-router 2.11.0 → 2.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +56 -0
- package/.github/workflows/pages.yml +37 -0
- package/Awesome-LLM/LICENSE.md +121 -0
- package/Awesome-LLM/README.md +625 -0
- package/Awesome-LLM/contributing.md +24 -0
- package/Awesome-LLM/paper_list/RLHF.md +1 -0
- package/Awesome-LLM/paper_list/Retrieval_Augmented_Generation.md +6 -0
- package/Awesome-LLM/paper_list/acceleration.md +10 -0
- package/Awesome-LLM/paper_list/alignment.md +20 -0
- package/Awesome-LLM/paper_list/application.md +19 -0
- package/Awesome-LLM/paper_list/augmentation.md +14 -0
- package/Awesome-LLM/paper_list/chain_of_thougt.md +16 -0
- package/Awesome-LLM/paper_list/code_pretraining.md +0 -0
- package/Awesome-LLM/paper_list/detection.md +11 -0
- package/Awesome-LLM/paper_list/evaluation.md +64 -0
- package/Awesome-LLM/paper_list/in_context_learning.md +4 -0
- package/Awesome-LLM/paper_list/instruction-tuning.md +35 -0
- package/Awesome-LLM/paper_list/moe.md +0 -0
- package/Awesome-LLM/paper_list/prompt_learning.md +15 -0
- package/Awesome-LLM/resources/DeepSpeed_light.svg +27 -0
- package/Awesome-LLM/resources/alpa-logo-cropped.png +0 -0
- package/Awesome-LLM/resources/colossal-ai_logo_vertical.png +0 -0
- package/Awesome-LLM/resources/creepy_llm.jpeg +0 -0
- package/Awesome-LLM/resources/image8.gif +0 -0
- package/Awesome-LLM/resources/jax_logo_250px.png +0 -0
- package/Awesome-LLM/resources/logo.png +0 -0
- package/Awesome-LLM/resources/nvidia.png +0 -0
- package/Awesome-LLM/resources/tensorflow.png +0 -0
- package/Awesome-LLM/resources//345/244/247/345/236/213/350/257/255/350/250/200/346/250/241/345/236/213/347/232/204/350/203/275/345/212/233/345/210/206/346/236/220/344/270/216/345/272/224/347/224/250 - 30min.pdf +0 -0
- package/LANDING.md +46 -0
- package/MANIFESTO.md +54 -0
- package/README.md +3 -3
- package/SUBMISSIONS.md +43 -0
- package/assets/chart-cost-v2.svg +91 -0
- package/assets/chart-cost-v3.svg +143 -0
- package/assets/chart-features-v2.svg +132 -0
- package/assets/chart-features-v3.svg +211 -0
- package/assets/chart-growth-v2.svg +122 -0
- package/assets/chart-growth-v3.svg +189 -0
- package/assets/cost-simple.svg +64 -0
- package/assets/growth-simple.svg +69 -0
- package/assets/hero-diagram.svg +81 -0
- package/assets/logo-new.svg +21 -0
- package/assets/social-preview-new.svg +100 -0
- package/assets/social-v2.svg +130 -0
- package/assets/social-v3.svg +212 -0
- package/awesome-ai-gateways/CONTRIBUTING.md +49 -0
- package/awesome-ai-gateways/LICENSE +3 -0
- package/awesome-ai-gateways/README.md +190 -0
- package/awesome-selfhosted/.github/ISSUE_TEMPLATE/config.yml +5 -0
- package/awesome-selfhosted/.github/PULL_REQUEST_TEMPLATE.md +1 -0
- package/awesome-selfhosted/LICENSE +376 -0
- package/awesome-selfhosted/README.md +2303 -0
- package/awesome-selfhosted/_static/awesome.png +0 -0
- package/awesome-selfhosted/non-free.md +377 -0
- package/demo-new.tape +71 -0
- package/demo-real.sh +198 -0
- package/demo-simple.tape +205 -0
- package/demo.html +520 -0
- package/demo.sh +85 -0
- package/demo.tape +259 -0
- package/dist/cli/setupWizard.js +194 -0
- package/dist/cli/tui.js +152 -0
- package/dist/cli.js +9 -2
- package/dist/routing/providerRetry.d.ts +5 -0
- package/dist/routing/providerRetry.js +37 -0
- package/dist/routing/providerRetry.js.map +1 -1
- package/docs/CHINESE_PROVIDER_RELIABILITY.md +37 -0
- package/docs/CLAIMS_AND_EVIDENCE.md +58 -0
- package/docs/ENGINEERING_SPEC.md +55 -0
- package/docs/RELEASE_CHECKLIST.md +32 -0
- package/docs/REPRODUCIBILITY.md +63 -0
- package/docs/demo-auto.html +264 -0
- package/docs/demo.html +416 -0
- package/docs/index.html +112 -599
- package/eval/README.md +46 -0
- package/eval/baselines/main.json +12 -0
- package/eval/benchmark_dataset.jsonl +16 -0
- package/eval/check_golden_routes.js +64 -0
- package/eval/datasets/catalog.json +33 -0
- package/eval/datasets/slices/cn_provider_reliability_v1.jsonl +3 -0
- package/eval/datasets/slices/cost_pressure_v1.jsonl +3 -0
- package/eval/datasets/slices/safety_guardrails_v1.jsonl +3 -0
- package/eval/fault_injection_thresholds.json +3 -0
- package/eval/generate_report.js +128 -0
- package/eval/golden_routes.json +114 -0
- package/eval/lib/experiment_registry.js +24 -0
- package/eval/run_eval.js +197 -0
- package/eval/run_fault_injection.js +201 -0
- package/eval/run_shadow_eval.js +85 -0
- package/eval/thresholds.json +9 -0
- package/index.html +667 -0
- package/package.json +29 -168
- package/pytest.ini +2 -0
- package/src/cli/setupWizard.ts +194 -0
- package/src/routing/providerRetry.ts +41 -1
- package/docs/assets/cost-comparison.svg +0 -134
- package/docs/assets/growth-chart-animated.svg +0 -76
- package/docs/assets/og-banner.svg +0 -194
- package/docs/assets/social-preview.svg +0 -194
- package/python/a3m/__pycache__/__init__.cpython-312.pyc +0 -0
- package/python/a3m/__pycache__/client.cpython-312.pyc +0 -0
- package/python/a3m/__pycache__/models.cpython-312.pyc +0 -0
- package/python/a3m/__pycache__/sync_client.cpython-312.pyc +0 -0
package/demo.tape
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# A3M Router VHS Demo - 90 Second Terminal Demo
|
|
2
|
+
# Target: 60-90 seconds
|
|
3
|
+
# Output: a3m-demo.gif
|
|
4
|
+
|
|
5
|
+
# =============================================================================
|
|
6
|
+
# SETUP
|
|
7
|
+
# =============================================================================
|
|
8
|
+
|
|
9
|
+
Output demo.gif
|
|
10
|
+
Set FontSize 16
|
|
11
|
+
Set Width 1200
|
|
12
|
+
Set Height 675
|
|
13
|
+
Set TypingSpeed 30ms
|
|
14
|
+
Set Shell bash
|
|
15
|
+
Set FontFamily "JetBrains Mono"
|
|
16
|
+
|
|
17
|
+
# Theme - dark terminal
|
|
18
|
+
Set ThemeFlavor catppuccin-mocha
|
|
19
|
+
|
|
20
|
+
# =============================================================================
|
|
21
|
+
# SCENE 1: THE PROBLEM (0-15s)
|
|
22
|
+
# =============================================================================
|
|
23
|
+
|
|
24
|
+
Type "#!/bin/bash"
|
|
25
|
+
Enter
|
|
26
|
+
|
|
27
|
+
Sleep 500ms
|
|
28
|
+
|
|
29
|
+
Type "# Your LLM app today — costs $0.015 per request, no fallback"
|
|
30
|
+
Enter
|
|
31
|
+
|
|
32
|
+
Type "cat << 'EOF'"
|
|
33
|
+
Enter
|
|
34
|
+
Sleep 100ms
|
|
35
|
+
Type "// Without A3M: you manage everything manually"
|
|
36
|
+
Enter
|
|
37
|
+
Type "const response = await openai.chat.completions.create({"
|
|
38
|
+
Enter
|
|
39
|
+
Type " model: \"gpt-4o\", // Always expensive"
|
|
40
|
+
Enter
|
|
41
|
+
Type " messages: [{ role: \"user\", content: \"Explain quantum entanglement\" }]"
|
|
42
|
+
Enter
|
|
43
|
+
Type "});"
|
|
44
|
+
Enter
|
|
45
|
+
Type "// If OpenAI goes down → your app crashes"
|
|
46
|
+
Enter
|
|
47
|
+
Type "// You pay $0.015 per request forever"
|
|
48
|
+
Enter
|
|
49
|
+
Type "EOF"
|
|
50
|
+
Enter
|
|
51
|
+
|
|
52
|
+
Sleep 1s
|
|
53
|
+
|
|
54
|
+
# Red X marks
|
|
55
|
+
Type "echo \"❌ Problem: $0.015/req, no fallback, manual management\""
|
|
56
|
+
Enter
|
|
57
|
+
|
|
58
|
+
Sleep 2s
|
|
59
|
+
|
|
60
|
+
# =============================================================================
|
|
61
|
+
# SCENE 2: WITH A3M - ONE LINE CHANGE (15-35s)
|
|
62
|
+
# =============================================================================
|
|
63
|
+
|
|
64
|
+
Ctrl+L
|
|
65
|
+
Sleep 500ms
|
|
66
|
+
|
|
67
|
+
Type "# With A3M: one line change, infinite benefits"
|
|
68
|
+
Enter
|
|
69
|
+
|
|
70
|
+
Type "cat << 'EOF'"
|
|
71
|
+
Enter
|
|
72
|
+
Sleep 100ms
|
|
73
|
+
Type "// Same request with A3M router"
|
|
74
|
+
Enter
|
|
75
|
+
Type "const response = await fetch(\"http://localhost:8787/v1/chat/completions\", {"
|
|
76
|
+
Enter
|
|
77
|
+
Type " method: \"POST\","
|
|
78
|
+
Enter
|
|
79
|
+
Type " headers: { \"Content-Type\": \"application/json\" },"
|
|
80
|
+
Enter
|
|
81
|
+
Type " body: JSON.stringify({"
|
|
82
|
+
Enter
|
|
83
|
+
Type " model: \"auto\", // ← A3M picks the best model"
|
|
84
|
+
Enter
|
|
85
|
+
Type " messages: [{ role: \"user\", content: \"Explain quantum entanglement\" }]"
|
|
86
|
+
Enter
|
|
87
|
+
Type " })"
|
|
88
|
+
Enter
|
|
89
|
+
Type "});"
|
|
90
|
+
Enter
|
|
91
|
+
Type "EOF"
|
|
92
|
+
Enter
|
|
93
|
+
|
|
94
|
+
Sleep 1s
|
|
95
|
+
|
|
96
|
+
# Show routing decision
|
|
97
|
+
Enter
|
|
98
|
+
Type "# A3M analyzes and decides:"
|
|
99
|
+
Enter
|
|
100
|
+
Sleep 200ms
|
|
101
|
+
Type "echo \"📊 Task: explanation → Complexity: simple → Budget: minimize\""
|
|
102
|
+
Enter
|
|
103
|
+
Sleep 500ms
|
|
104
|
+
Type "echo \"\""
|
|
105
|
+
Enter
|
|
106
|
+
Sleep 200ms
|
|
107
|
+
Type "echo \"✅ Routed to: Groq (FREE, 847ms)\""
|
|
108
|
+
Enter
|
|
109
|
+
Sleep 500ms
|
|
110
|
+
Type "echo \" Cost: $0.00 | Quality: 94%\""
|
|
111
|
+
Enter
|
|
112
|
+
|
|
113
|
+
Sleep 2s
|
|
114
|
+
|
|
115
|
+
# =============================================================================
|
|
116
|
+
# SCENE 3: FAILURE RECOVERY (35-55s)
|
|
117
|
+
# =============================================================================
|
|
118
|
+
|
|
119
|
+
Ctrl+L
|
|
120
|
+
Sleep 500ms
|
|
121
|
+
|
|
122
|
+
Type "# But what happens when a provider fails?"
|
|
123
|
+
Enter
|
|
124
|
+
Sleep 1s
|
|
125
|
+
|
|
126
|
+
Type "# A3M gracefully handles failures..."
|
|
127
|
+
Enter
|
|
128
|
+
|
|
129
|
+
Sleep 1s
|
|
130
|
+
|
|
131
|
+
# Simulate the failure scenario
|
|
132
|
+
Type "echo \"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\""
|
|
133
|
+
Enter
|
|
134
|
+
Sleep 300ms
|
|
135
|
+
Type "echo \" [Groq] — Attempting connection...\""
|
|
136
|
+
Enter
|
|
137
|
+
Sleep 1s
|
|
138
|
+
Type "echo \" [Groq] — ✗ FAILED — 503 Service Unavailable\""
|
|
139
|
+
Enter
|
|
140
|
+
Sleep 500ms
|
|
141
|
+
Type "echo \" [Circuit Breaker] — Tripped after 3 failures\""
|
|
142
|
+
Enter
|
|
143
|
+
Sleep 500ms
|
|
144
|
+
Type "echo \" [DeepSeek] — HEALTHY — Switching...\""
|
|
145
|
+
Enter
|
|
146
|
+
Sleep 1s
|
|
147
|
+
Type "echo \"\""
|
|
148
|
+
Enter
|
|
149
|
+
Sleep 300ms
|
|
150
|
+
Type "echo \"✅ Response delivered via DeepSeek fallback\""
|
|
151
|
+
Enter
|
|
152
|
+
Sleep 500ms
|
|
153
|
+
Type "echo \" Your app: never knew there was a problem\""
|
|
154
|
+
Enter
|
|
155
|
+
Sleep 500ms
|
|
156
|
+
Type "echo \"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\""
|
|
157
|
+
Enter
|
|
158
|
+
|
|
159
|
+
Sleep 2s
|
|
160
|
+
|
|
161
|
+
# =============================================================================
|
|
162
|
+
# SCENE 4: COST SAVINGS (55-70s)
|
|
163
|
+
# =============================================================================
|
|
164
|
+
|
|
165
|
+
Ctrl+L
|
|
166
|
+
Sleep 500ms
|
|
167
|
+
|
|
168
|
+
Type "# Same query. Dramatically different costs."
|
|
169
|
+
Enter
|
|
170
|
+
Sleep 500ms
|
|
171
|
+
|
|
172
|
+
Type "echo \"\""
|
|
173
|
+
Enter
|
|
174
|
+
Sleep 200ms
|
|
175
|
+
Type "echo \" WITHOUT A3M WITH A3M\""
|
|
176
|
+
Enter
|
|
177
|
+
Sleep 200ms
|
|
178
|
+
Type "echo \" ───────────────── ────────────────\""
|
|
179
|
+
Enter
|
|
180
|
+
Sleep 200ms
|
|
181
|
+
Type "echo \" GPT-4o @ $3.00/1K Groq @ $0.00/1K\""
|
|
182
|
+
Enter
|
|
183
|
+
Sleep 200ms
|
|
184
|
+
Type "echo \" 500 tokens = $0.0015 500 tokens = $0.00\""
|
|
185
|
+
Enter
|
|
186
|
+
Sleep 1s
|
|
187
|
+
Type "echo \"\""
|
|
188
|
+
Enter
|
|
189
|
+
Sleep 200ms
|
|
190
|
+
Type "echo \"💰 SAVINGS: 100% per query\""
|
|
191
|
+
Enter
|
|
192
|
+
Sleep 300ms
|
|
193
|
+
Type "echo \" At 1000 queries/day → $1.50 saved daily\""
|
|
194
|
+
Enter
|
|
195
|
+
Sleep 300ms
|
|
196
|
+
Type "echo \" At 1000 queries/day → $547 saved yearly\""
|
|
197
|
+
Enter
|
|
198
|
+
|
|
199
|
+
Sleep 2s
|
|
200
|
+
|
|
201
|
+
# =============================================================================
|
|
202
|
+
# SCENE 5: ONE COMMAND (70-90s)
|
|
203
|
+
# =============================================================================
|
|
204
|
+
|
|
205
|
+
Ctrl+L
|
|
206
|
+
Sleep 500ms
|
|
207
|
+
|
|
208
|
+
Type "echo \"\""
|
|
209
|
+
Enter
|
|
210
|
+
Sleep 200ms
|
|
211
|
+
Type "echo \"╔═══════════════════════════════════════════════════╗\""
|
|
212
|
+
Enter
|
|
213
|
+
Sleep 100ms
|
|
214
|
+
Type "echo \"║ Get started in 10 seconds: ║\""
|
|
215
|
+
Enter
|
|
216
|
+
Sleep 100ms
|
|
217
|
+
Type "echo \"╚═══════════════════════════════════════════════════╝\""
|
|
218
|
+
Enter
|
|
219
|
+
Sleep 500ms
|
|
220
|
+
Type "echo \"\""
|
|
221
|
+
Enter
|
|
222
|
+
Sleep 200ms
|
|
223
|
+
Type "echo \" npm install -g adaptive-memory-multi-model-router\""
|
|
224
|
+
Enter
|
|
225
|
+
Sleep 300ms
|
|
226
|
+
Type "echo \"\""
|
|
227
|
+
Enter
|
|
228
|
+
Sleep 200ms
|
|
229
|
+
Type "echo \" # Auto-detects your API keys, zero config\""
|
|
230
|
+
Enter
|
|
231
|
+
Sleep 300ms
|
|
232
|
+
Type "echo \" npx a3m-router serve\""
|
|
233
|
+
Enter
|
|
234
|
+
Sleep 500ms
|
|
235
|
+
Type "echo \"\""
|
|
236
|
+
Enter
|
|
237
|
+
Sleep 200ms
|
|
238
|
+
Type "echo \" # Change: model: 'gpt-4o' → model: 'auto'\""
|
|
239
|
+
Enter
|
|
240
|
+
Sleep 500ms
|
|
241
|
+
Type "echo \"\""
|
|
242
|
+
Enter
|
|
243
|
+
Sleep 300ms
|
|
244
|
+
Type "echo \" GitHub: github.com/Das-rebel/adaptive-memory-multi-model-router\""
|
|
245
|
+
Enter
|
|
246
|
+
Sleep 200ms
|
|
247
|
+
Type "echo \" npm: npmjs.com/package/adaptive-memory-multi-model-router\""
|
|
248
|
+
Enter
|
|
249
|
+
|
|
250
|
+
Sleep 2s
|
|
251
|
+
|
|
252
|
+
# Final pulse effect
|
|
253
|
+
Type "echo \"\""
|
|
254
|
+
Enter
|
|
255
|
+
Sleep 200ms
|
|
256
|
+
Type "echo \"✨ One prompt in. The right model out. ✨\""
|
|
257
|
+
Enter
|
|
258
|
+
|
|
259
|
+
Sleep 1s
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A3M Router Setup Wizard
|
|
3
|
+
* Interactive configuration wizard
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const readline = require('readline');
|
|
9
|
+
|
|
10
|
+
const CONFIG_DIR = path.join(process.env.HOME || '/tmp', '.config', 'a3m-router');
|
|
11
|
+
const CONFIG_FILE = path.join(CONFIG_DIR, 'providers.json');
|
|
12
|
+
|
|
13
|
+
// API key environment variable mappings
|
|
14
|
+
const API_KEY_ENV_MAP = {
|
|
15
|
+
'GROQ_API_KEY': 'groq',
|
|
16
|
+
'OPENAI_API_KEY': 'openai',
|
|
17
|
+
'ANTHROPIC_API_KEY': 'anthropic',
|
|
18
|
+
'DEEPSEEK_API_KEY': 'deepseek',
|
|
19
|
+
'MISTRAL_API_KEY': 'mistral',
|
|
20
|
+
'GOOGLE_API_KEY': 'google',
|
|
21
|
+
'CEREBRAS_API_KEY': 'cerebras',
|
|
22
|
+
'TOGETHER_API_KEY': 'together',
|
|
23
|
+
'AI21_API_KEY': 'ai21',
|
|
24
|
+
'COHERE_API_KEY': 'cohere',
|
|
25
|
+
'MINIMAX_API_KEY': 'minimax',
|
|
26
|
+
'KIMI_API_KEY': 'kimi',
|
|
27
|
+
'MOONSHOT_API_KEY': 'moonshot',
|
|
28
|
+
'QWEN_API_KEY': 'qwen',
|
|
29
|
+
'ZHIPU_API_KEY': 'zhipu',
|
|
30
|
+
'YI_API_KEY': 'yi',
|
|
31
|
+
'BAICHUAN_API_KEY': 'baichuan',
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// Provider metadata
|
|
35
|
+
const PROVIDER_INFO = {
|
|
36
|
+
groq: { name: 'Groq', models: 'llama-3.3-70b-versatile', tier: 'free', strength: 'Fast, free tier' },
|
|
37
|
+
openai: { name: 'OpenAI', models: 'gpt-4o-mini', tier: 'paid', strength: 'GPT-4, most capable' },
|
|
38
|
+
anthropic: { name: 'Anthropic', models: 'claude-3.5-haiku', tier: 'paid', strength: 'Claude, best reasoning' },
|
|
39
|
+
deepseek: { name: 'DeepSeek', models: 'deepseek-chat-v3', tier: 'cheap', strength: 'Cheap, good code' },
|
|
40
|
+
mistral: { name: 'Mistral', models: 'mistral-small-latest', tier: 'cheap', strength: 'European, balanced' },
|
|
41
|
+
google: { name: 'Google AI', models: 'gemini-1.5-flash', tier: 'free', strength: 'Gemini, multimodal' },
|
|
42
|
+
cerebras: { name: 'Cerebras', models: 'llama-3.3-70b', tier: 'free', strength: 'Fastest inference' },
|
|
43
|
+
together: { name: 'Together AI', models: 'Llama-3.3-70B-Instruct', tier: 'cheap', strength: 'Managed, reliable' },
|
|
44
|
+
ai21: { name: 'AI21', models: 'jamba-1.5-medium', tier: 'paid', strength: 'Jamba, long context' },
|
|
45
|
+
cohere: { name: 'Cohere', models: 'command-r7b', tier: 'cheap', strength: 'Command series, fast' },
|
|
46
|
+
minimax: { name: 'MiniMax', models: 'abab6.5s-chat', tier: 'cheap', strength: 'Chinese, cheap' },
|
|
47
|
+
kimi: { name: 'Kimi/Moonshot', models: 'moonshot-v1-8k', tier: 'cheap', strength: 'Chinese, 128k context' },
|
|
48
|
+
moonshot: { name: 'Moonshot', models: 'moonshot-v1-8k', tier: 'cheap', strength: 'Chinese, good' },
|
|
49
|
+
qwen: { name: 'Qwen', models: 'qwen-turbo', tier: 'cheap', strength: 'Alibaba, multilingual' },
|
|
50
|
+
zhipu: { name: 'Zhipu GLM', models: 'glm-4', tier: 'cheap', strength: 'Chinese, smart' },
|
|
51
|
+
yi: { name: 'Yi', models: 'yi-large', tier: 'cheap', strength: 'Chinese, good reasoning' },
|
|
52
|
+
baichuan: { name: 'Baichuan', models: 'baichuan-4', tier: 'cheap', strength: 'Chinese, balanced' },
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
function createInterface() {
|
|
56
|
+
return readline.createInterface({
|
|
57
|
+
input: process.stdin,
|
|
58
|
+
output: process.stdout
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function question(rl, text) {
|
|
63
|
+
return new Promise((resolve) => {
|
|
64
|
+
rl.question(text, (answer) => resolve(answer));
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function detectApiKeys() {
|
|
69
|
+
const detected = [];
|
|
70
|
+
for (const [envVar, providerId] of Object.entries(API_KEY_ENV_MAP)) {
|
|
71
|
+
if (process.env[envVar]) {
|
|
72
|
+
detected.push({ envVar, providerId, info: PROVIDER_INFO[providerId] });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return detected;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async function runWizard() {
|
|
79
|
+
console.log('\n🔧 A3M Router Setup Wizard');
|
|
80
|
+
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━\n');
|
|
81
|
+
|
|
82
|
+
const rl = createInterface();
|
|
83
|
+
|
|
84
|
+
// Ensure config directory exists
|
|
85
|
+
if (!fs.existsSync(CONFIG_DIR)) {
|
|
86
|
+
fs.mkdirSync(CONFIG_DIR, { recursive: true });
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Check for existing config
|
|
90
|
+
let existingConfig = {};
|
|
91
|
+
if (fs.existsSync(CONFIG_FILE)) {
|
|
92
|
+
try {
|
|
93
|
+
existingConfig = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
|
|
94
|
+
console.log('✓ Found existing config at', CONFIG_FILE);
|
|
95
|
+
console.log(' Providers:', Object.keys(existingConfig.providers || {}).join(', '));
|
|
96
|
+
console.log('');
|
|
97
|
+
} catch (e) {
|
|
98
|
+
console.log('⚠ Could not read existing config, starting fresh\n');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Auto-detect API keys
|
|
103
|
+
console.log('🔍 Scanning for API keys in environment...');
|
|
104
|
+
const detected = await detectApiKeys();
|
|
105
|
+
|
|
106
|
+
if (detected.length === 0) {
|
|
107
|
+
console.log('⚠ No API keys detected in environment.');
|
|
108
|
+
console.log(' Set any of: GROQ_API_KEY, OPENAI_API_KEY, DEEPSEEK_API_KEY, etc.\n');
|
|
109
|
+
} else {
|
|
110
|
+
console.log('✓ Found', detected.length, 'API key(s):');
|
|
111
|
+
detected.forEach(({ envVar, providerId, info }) => {
|
|
112
|
+
console.log(' ✓', envVar, '→', info?.name || providerId);
|
|
113
|
+
});
|
|
114
|
+
console.log('');
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Provider selection
|
|
118
|
+
const allProviders = Object.keys(PROVIDER_INFO);
|
|
119
|
+
const selected = new Set();
|
|
120
|
+
|
|
121
|
+
// Pre-select providers with detected keys
|
|
122
|
+
detected.forEach(({ providerId }) => selected.add(providerId));
|
|
123
|
+
|
|
124
|
+
console.log('📡 Select providers to configure (comma-separated numbers, or "all"):');
|
|
125
|
+
console.log('');
|
|
126
|
+
|
|
127
|
+
const numbered = allProviders.map((id, i) => ({ id, i }));
|
|
128
|
+
numbered.forEach(({ id, i }) => {
|
|
129
|
+
const info = PROVIDER_INFO[id];
|
|
130
|
+
const selected_mark = selected.has(id) ? '[x]' : '[ ]';
|
|
131
|
+
const tier_mark = info?.tier === 'free' ? '(FREE)' : info?.tier === 'cheap' ? '(cheap)' : '(paid)';
|
|
132
|
+
console.log(` ${String(i + 1).padStart(2)}. ${selected_mark} ${id.padEnd(12)} ${tier_mark} - ${info?.strength || ''}`);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
console.log('');
|
|
136
|
+
const answer = await question(rl, ' Enter numbers or "all" [all with keys detected]: ');
|
|
137
|
+
|
|
138
|
+
if (answer.toLowerCase().trim() === 'all') {
|
|
139
|
+
allProviders.forEach(id => selected.add(id));
|
|
140
|
+
} else if (answer.trim()) {
|
|
141
|
+
const nums = answer.split(',').map(s => parseInt(s.trim())).filter(n => !isNaN(n));
|
|
142
|
+
nums.forEach(n => {
|
|
143
|
+
const idx = n - 1;
|
|
144
|
+
if (idx >= 0 && idx < allProviders.length) {
|
|
145
|
+
selected.add(allProviders[idx]);
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
console.log('\n✓ Selected providers:', Array.from(selected).join(', '));
|
|
151
|
+
|
|
152
|
+
// Build config
|
|
153
|
+
const config = {
|
|
154
|
+
version: '1.0',
|
|
155
|
+
providers: {}
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
selected.forEach(providerId => {
|
|
159
|
+
const info = PROVIDER_INFO[providerId];
|
|
160
|
+
const envKey = Object.entries(API_KEY_ENV_MAP).find(([k, v]) => v === providerId)?.[0];
|
|
161
|
+
|
|
162
|
+
config.providers[providerId] = {
|
|
163
|
+
name: info?.name || providerId,
|
|
164
|
+
apiKey: envKey ? process.env[envKey] : '',
|
|
165
|
+
models: [info?.models || 'default'],
|
|
166
|
+
type: 'api',
|
|
167
|
+
enabled: true
|
|
168
|
+
};
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// Save config
|
|
172
|
+
fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2));
|
|
173
|
+
console.log('\n✓ Config saved to', CONFIG_FILE);
|
|
174
|
+
|
|
175
|
+
// Test connections
|
|
176
|
+
console.log('\n🧪 Testing connections...');
|
|
177
|
+
console.log(' (Skipped in wizard mode - run "npx a3m-router test" to verify)\n');
|
|
178
|
+
|
|
179
|
+
// Ready message
|
|
180
|
+
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━');
|
|
181
|
+
console.log('✅ A3M Router is ready!');
|
|
182
|
+
console.log('');
|
|
183
|
+
console.log(' Next steps:');
|
|
184
|
+
console.log(' 1. npx a3m-router serve # Start proxy server');
|
|
185
|
+
console.log(' 2. npx a3m-router test # Test provider connections');
|
|
186
|
+
console.log(' 3. npx a3m-router route "hi" # Try routing a query');
|
|
187
|
+
console.log('');
|
|
188
|
+
console.log(' Docs: https://github.com/Das-rebel/adaptive-memory-multi-model-router');
|
|
189
|
+
console.log('');
|
|
190
|
+
|
|
191
|
+
rl.close();
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
module.exports = { runWizard };
|
package/dist/cli/tui.js
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* A3M Router Quick TUI
|
|
4
|
+
* Zero-config experience - no API key needed for free tier
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const args = process.argv.slice(2);
|
|
8
|
+
|
|
9
|
+
if (args.length === 0) {
|
|
10
|
+
console.log(`
|
|
11
|
+
╔══════════════════════════════════════════════╗
|
|
12
|
+
║ 🔀 A3M Router - Zero Config ║
|
|
13
|
+
╠══════════════════════════════════════════════╣
|
|
14
|
+
║ ║
|
|
15
|
+
║ One prompt in. The right model out. ║
|
|
16
|
+
║ ║
|
|
17
|
+
║ Usage: ║
|
|
18
|
+
║ a3m hello world # Route a query ║
|
|
19
|
+
║ a3m --serve # Start proxy ║
|
|
20
|
+
║ a3m --setup # Configure API keys ║
|
|
21
|
+
║ a3m --demo # Try without keys ║
|
|
22
|
+
║ ║
|
|
23
|
+
╚══════════════════════════════════════════════╝
|
|
24
|
+
`);
|
|
25
|
+
process.exit(0);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const command = args[0];
|
|
29
|
+
|
|
30
|
+
// Demo mode - works without API key
|
|
31
|
+
if (command === '--demo') {
|
|
32
|
+
console.log(`
|
|
33
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
34
|
+
🎮 A3M Router Demo Mode
|
|
35
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
36
|
+
|
|
37
|
+
Try these queries:
|
|
38
|
+
• "Write Python hello world"
|
|
39
|
+
• "Explain quantum physics"
|
|
40
|
+
• "Translate to Japanese: Hello"
|
|
41
|
+
• "Write a haiku about coding"
|
|
42
|
+
`);
|
|
43
|
+
|
|
44
|
+
const readline = require('readline');
|
|
45
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
46
|
+
|
|
47
|
+
const demoResults = [
|
|
48
|
+
{ provider: 'groq', model: 'llama-3.3-70b-versatile', tier: 'free', cost: '$0.00' },
|
|
49
|
+
{ provider: 'deepseek', model: 'deepseek-chat-v3', tier: 'cheap', cost: '$0.00' },
|
|
50
|
+
{ provider: 'kimi', model: 'moonshot-v1-8k', tier: 'cheap', cost: '$0.00' },
|
|
51
|
+
{ provider: 'openai', model: 'gpt-4o-mini', tier: 'paid', cost: '$0.00' },
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
let count = 0;
|
|
55
|
+
function ask() {
|
|
56
|
+
rl.question('\n🔀 Query: ', (q) => {
|
|
57
|
+
if (!q || q.toLowerCase() === 'exit') {
|
|
58
|
+
console.log('\n👋 Ready for the real thing? Run: npm install adaptive-memory-multi-model-router\n');
|
|
59
|
+
rl.close();
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const r = demoResults[count++ % demoResults.length];
|
|
63
|
+
console.log(`\n 🏆 Best: ${r.provider} / ${r.model}`);
|
|
64
|
+
console.log(` 💰 Cost: ${r.cost} (${r.tier})`);
|
|
65
|
+
ask();
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
ask();
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Serve mode
|
|
73
|
+
if (command === '--serve') {
|
|
74
|
+
console.log(`
|
|
75
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
76
|
+
🚀 A3M Router Proxy Server
|
|
77
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
78
|
+
|
|
79
|
+
Server: http://localhost:8787/v1/chat/completions
|
|
80
|
+
API: OpenAI-compatible
|
|
81
|
+
|
|
82
|
+
No API keys configured? Add free tier:
|
|
83
|
+
GROQ_API_KEY=your_key npx a3m-router serve
|
|
84
|
+
|
|
85
|
+
Or run setup: a3m --setup
|
|
86
|
+
`);
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const server = require('./server/proxyServer.js');
|
|
90
|
+
const port = parseInt(args[1]) || 8787;
|
|
91
|
+
server.createProxyServer({ port });
|
|
92
|
+
console.log(`\n✅ Server running at http://localhost:${port}`);
|
|
93
|
+
} catch (e) {
|
|
94
|
+
console.log('\n⚠️ Run "npm install" first, then "npm run build"\n');
|
|
95
|
+
}
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Setup mode
|
|
100
|
+
if (command === '--setup') {
|
|
101
|
+
const wizard = require('./cli/setupWizard.js');
|
|
102
|
+
wizard.runWizard();
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Zero-config mode - route with free tier defaults
|
|
107
|
+
const query = args.join(' ');
|
|
108
|
+
|
|
109
|
+
// Check for GROQ_API_KEY first (free tier)
|
|
110
|
+
if (process.env.GROQ_API_KEY) {
|
|
111
|
+
console.log('\n🔀 Routing: "' + query + '"');
|
|
112
|
+
console.log('📡 Using free tier (Groq)...\n');
|
|
113
|
+
|
|
114
|
+
// Quick route via groq
|
|
115
|
+
const { routeQuery } = require('./index.js');
|
|
116
|
+
routeQuery(query).then(r => {
|
|
117
|
+
console.log(` 🏆 Provider: ${r.provider || 'groq'}`);
|
|
118
|
+
console.log(` 🤖 Model: ${r.model || 'llama-3.3-70b-versatile'}`);
|
|
119
|
+
console.log(` 💰 Est: $${(r.estimated_cost || 0.0001).toFixed(6)}`);
|
|
120
|
+
}).catch(e => {
|
|
121
|
+
console.log(' ⚠️ Error:', e.message);
|
|
122
|
+
console.log(' Tip: Get free key at https://console.groq.com\n');
|
|
123
|
+
});
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// No API key - show helpful message
|
|
128
|
+
console.log(`
|
|
129
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
130
|
+
⚠️ No API key detected
|
|
131
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
132
|
+
|
|
133
|
+
A3M Router needs at least one API key to route queries.
|
|
134
|
+
|
|
135
|
+
🔥 FREE TIER OPTIONS:
|
|
136
|
+
1. Groq (fastest, 1000+ req/day free)
|
|
137
|
+
→ https://console.groq.com/apikeys
|
|
138
|
+
→ Set: export GROQ_API_KEY=your_key
|
|
139
|
+
|
|
140
|
+
2. Cerebras (fastest inference, free)
|
|
141
|
+
→ https://cerebras.ai/labs
|
|
142
|
+
→ Set: export CEREBRAS_API_KEY=your_key
|
|
143
|
+
|
|
144
|
+
3. Google AI (Gemini, free tier)
|
|
145
|
+
→ https://aistudio.google.com/app/apikey
|
|
146
|
+
→ Set: export GOOGLE_API_KEY=your_key
|
|
147
|
+
|
|
148
|
+
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
149
|
+
|
|
150
|
+
Quick setup: npx a3m --setup
|
|
151
|
+
Quick start: GROQ_API_KEY=your_key npx a3m-router serve
|
|
152
|
+
`);
|
package/dist/cli.js
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
* A3M Router CLI - Adaptive Memory Multi-Model Router
|
|
4
4
|
*
|
|
5
5
|
* Commands:
|
|
6
|
-
* npx a3m-router serve [--port 8787]
|
|
6
|
+
* npx a3m-router serve [--port 8787] Start OpenAI-compatible proxy server
|
|
7
7
|
* npx a3m-router route <query> Route query to best provider
|
|
8
|
-
* npx a3m-router
|
|
8
|
+
* npx a3m-router setup Interactive setup wizard (auto-detect API keys)
|
|
9
|
+
* npx a3m-router batch <q1> <q2>.. Route multiple queries
|
|
9
10
|
* npx a3m-router providers List all configured providers
|
|
10
11
|
* npx a3m-router test Test all providers
|
|
11
12
|
* npx a3m-router compare <query> Compare providers side by side
|
|
@@ -155,6 +156,12 @@ async function main() {
|
|
|
155
156
|
const router = createA3MRouter({ memory: { maxSize: 1000 } });
|
|
156
157
|
|
|
157
158
|
switch (command) {
|
|
159
|
+
case 'setup': {
|
|
160
|
+
const { runWizard } = require('./cli/setupWizard.js');
|
|
161
|
+
runWizard();
|
|
162
|
+
break;
|
|
163
|
+
}
|
|
164
|
+
|
|
158
165
|
case 'providers': {
|
|
159
166
|
const providers = providerConfig.getAvailableProviders();
|
|
160
167
|
const allProviders = providerConfig._providers;
|
|
@@ -72,6 +72,11 @@ export declare class ProviderRetryHandler {
|
|
|
72
72
|
* Check if an error should trigger a retry
|
|
73
73
|
*/
|
|
74
74
|
isRetryableError(error: any): boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Detect hard non-retryable account/policy states.
|
|
77
|
+
* These should fail fast instead of wasting retries.
|
|
78
|
+
*/
|
|
79
|
+
private isPermanentProviderStateError;
|
|
75
80
|
/**
|
|
76
81
|
* Check if error is a rate limit (429)
|
|
77
82
|
*/
|
|
@@ -283,6 +283,10 @@ class ProviderRetryHandler {
|
|
|
283
283
|
isRetryableError(error) {
|
|
284
284
|
if (!error)
|
|
285
285
|
return false;
|
|
286
|
+
// Known non-retryable account/quota/policy states seen in Chinese provider APIs.
|
|
287
|
+
if (this.isPermanentProviderStateError(error)) {
|
|
288
|
+
return false;
|
|
289
|
+
}
|
|
286
290
|
const config = this.configs.get('default').retry;
|
|
287
291
|
const retryableErrors = config.retryableErrors || DEFAULT_RETRY_CONFIG.retryableErrors;
|
|
288
292
|
// Check error code/message
|
|
@@ -304,6 +308,39 @@ class ProviderRetryHandler {
|
|
|
304
308
|
return true;
|
|
305
309
|
return false;
|
|
306
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Detect hard non-retryable account/policy states.
|
|
313
|
+
* These should fail fast instead of wasting retries.
|
|
314
|
+
*/
|
|
315
|
+
isPermanentProviderStateError(error) {
|
|
316
|
+
const status = Number(error?.status ?? error?.statusCode ?? 0);
|
|
317
|
+
const message = String(error?.message || '').toLowerCase();
|
|
318
|
+
const code = String(error?.code || '').toLowerCase();
|
|
319
|
+
// Payment/balance/account states are generally non-retryable without human action.
|
|
320
|
+
if (status === 402)
|
|
321
|
+
return true;
|
|
322
|
+
if (status === 401 || status === 403)
|
|
323
|
+
return true;
|
|
324
|
+
const hardPatterns = [
|
|
325
|
+
'insufficient',
|
|
326
|
+
'balance',
|
|
327
|
+
'invalid api key',
|
|
328
|
+
'incorrect api key',
|
|
329
|
+
'access terminated',
|
|
330
|
+
'account abnormal',
|
|
331
|
+
'违规',
|
|
332
|
+
'quota exhausted',
|
|
333
|
+
'hour allocated quota exceeded',
|
|
334
|
+
'week allocated quota exceeded',
|
|
335
|
+
'month allocated quota exceeded',
|
|
336
|
+
'tpd rate limit'
|
|
337
|
+
];
|
|
338
|
+
for (const p of hardPatterns) {
|
|
339
|
+
if (message.includes(p) || code.includes(p))
|
|
340
|
+
return true;
|
|
341
|
+
}
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
307
344
|
/**
|
|
308
345
|
* Check if error is a rate limit (429)
|
|
309
346
|
*/
|