adaptive-memory-multi-model-router 2.14.46 → 2.14.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (598) hide show
  1. package/{docs/llms.txt → llms.txt.bak} +6 -6
  2. package/package.json +270 -72
  3. package/src/routing/advancedRouter.ts.bak +650 -0
  4. package/test.js.bak +376 -0
  5. package/.dockerignore +0 -82
  6. package/.env.example +0 -303
  7. package/.github/DISCUSSIONS_WELCOME.md +0 -27
  8. package/.github/DISCUSSION_TEMPLATE.yml +0 -5
  9. package/.github/FUNDING.yml +0 -2
  10. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -94
  11. package/.github/ISSUE_TEMPLATE/config.yml +0 -17
  12. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -71
  13. package/.github/PULL_REQUEST_TEMPLATE.md +0 -71
  14. package/.github/dependabot.yml +0 -9
  15. package/.github/workflows/auto-publish.yml +0 -51
  16. package/.github/workflows/ci.yml +0 -263
  17. package/.github/workflows/codeql.yml +0 -38
  18. package/.github/workflows/npm-publish.yml +0 -20
  19. package/.github/workflows/pages.yml +0 -37
  20. package/.github/workflows/stale.yml +0 -54
  21. package/.publish-tick +0 -1
  22. package/.well-known/ai-plugin.json +0 -16
  23. package/AGENT_COUNCIL_FINDINGS.md +0 -142
  24. package/ARCHITECTURE.md +0 -346
  25. package/AUDIT_REPORT.md +0 -28
  26. package/CODE_OF_CONDUCT.md +0 -128
  27. package/CONTRIBUTING.md +0 -50
  28. package/CONTRIBUTORS.md +0 -20
  29. package/Dockerfile +0 -53
  30. package/Dockerfile.proxy +0 -33
  31. package/HEALTH_REPORT.md +0 -118
  32. package/IMPROVEMENT_PLAN.md +0 -107
  33. package/LANDING.md +0 -43
  34. package/LAUNCH-PAIN-DRIVEN.md +0 -339
  35. package/LAUNCH.md +0 -337
  36. package/LAUNCH_CHECKLIST.md +0 -141
  37. package/LAUNCH_SNAPSHOT.md +0 -260
  38. package/MANIFESTO.md +0 -41
  39. package/POPULARITY_BOOSTERS.md +0 -285
  40. package/PR_STATUS_REPORT.md +0 -148
  41. package/REDESIGN.md +0 -95
  42. package/RUNKIT.md +0 -83
  43. package/SECURITY.md +0 -29
  44. package/SUBMISSIONS.md +0 -43
  45. package/_schema.html +0 -53
  46. package/ai-plugin.json +0 -16
  47. package/articles/AI_AGENT_LLM_ROUTING.md +0 -150
  48. package/articles/CHINESE_DIRECTORIES.md +0 -100
  49. package/articles/CHINESE_SUBMISSIONS_READY.md +0 -322
  50. package/articles/COMPETITOR_ALERTS.md +0 -31
  51. package/articles/COMPLETE_POSTING_DIRECTORY.md +0 -147
  52. package/articles/CONTENT_STRUCTURE.md +0 -292
  53. package/articles/DEVTO_COST_GUIDE.md +0 -473
  54. package/articles/DEVTO_FINAL.md +0 -416
  55. package/articles/DEVTO_MULTI_PROVIDER.md +0 -542
  56. package/articles/DEVTO_READY.md +0 -255
  57. package/articles/DEVTO_V2_ANNOUNCEMENT.md +0 -160
  58. package/articles/DEVTO_VIRAL_GROWTH.md +0 -280
  59. package/articles/FRESH_devto.md +0 -460
  60. package/articles/FRESH_devto_2026_05.md +0 -73
  61. package/articles/FRESH_hackernews.md +0 -14
  62. package/articles/FRESH_reddit_ml.md +0 -90
  63. package/articles/FRESH_reddit_node.md +0 -198
  64. package/articles/FRESH_reddit_sideproject.md +0 -72
  65. package/articles/FRESH_reddit_webdev.md +0 -130
  66. package/articles/FROM_ZERO_TO_10K.md +0 -107
  67. package/articles/HN_10X_BETTER.md +0 -430
  68. package/articles/HN_ACCOUNT_GUIDE.md +0 -21
  69. package/articles/HN_CHINESE_STYLE.md +0 -308
  70. package/articles/HN_FINAL.md +0 -148
  71. package/articles/HN_POSTED_VERSION.md +0 -56
  72. package/articles/HN_POST_READY.md +0 -137
  73. package/articles/HN_RESEARCH.md +0 -364
  74. package/articles/HN_SHOW_routerarena.md +0 -17
  75. package/articles/HN_TIMING_GUIDE.md +0 -52
  76. package/articles/INDIEHACKERS_POST.md +0 -52
  77. package/articles/INDIEHACKERS_READY.md +0 -120
  78. package/articles/LLM_BENCHMARK_DEEP_DIVE.md +0 -153
  79. package/articles/MASTER_POSTING_DIRECTORY.md +0 -189
  80. package/articles/NEWSLETTER_SEND_NOW.md +0 -259
  81. package/articles/NEWSLETTER_SUBMISSIONS.md +0 -112
  82. package/articles/PAIN-DRIVEN-devto-v2.md +0 -308
  83. package/articles/PAIN-DRIVEN-devto-v3.md +0 -268
  84. package/articles/PAIN-DRIVEN-devto.md +0 -242
  85. package/articles/PAIN-DRIVEN-hackernews-v2.md +0 -138
  86. package/articles/PAIN-DRIVEN-hackernews-v3.md +0 -151
  87. package/articles/PAIN-DRIVEN-hackernews.md +0 -131
  88. package/articles/PAIN-DRIVEN-reddit-v2.md +0 -301
  89. package/articles/PAIN-DRIVEN-reddit-v3.md +0 -236
  90. package/articles/PAIN-DRIVEN-reddit.md +0 -218
  91. package/articles/PAIN-DRIVEN-twitter-v2.md +0 -110
  92. package/articles/PAIN-DRIVEN-twitter-v3.md +0 -121
  93. package/articles/PAIN-DRIVEN-twitter.md +0 -120
  94. package/articles/PORTKEY_VS_A3M.md +0 -147
  95. package/articles/POSTING_KIT_2026_05.md +0 -67
  96. package/articles/PRESS_KIT_routerarena.md +0 -77
  97. package/articles/PRODUCTHUNT_LISTING.md +0 -48
  98. package/articles/PRODUCTHUNT_READY.md +0 -106
  99. package/articles/PR_PLAN_vault.md +0 -125
  100. package/articles/REDDIT_FINAL.md +0 -232
  101. package/articles/REDDIT_POST.md +0 -67
  102. package/articles/REDDIT_SUBMISSION_READY.md +0 -348
  103. package/articles/ROUTERARENA_LEADER.md +0 -45
  104. package/articles/SHOW_HN_FINAL.md +0 -29
  105. package/articles/TWEETS_10K_DOWNLOADS.md +0 -47
  106. package/articles/TWEETS_BENCHMARK_FIRST.md +0 -46
  107. package/articles/TWEETS_MCP_PLAY.md +0 -51
  108. package/articles/TWEETS_SEQUENTIAL_BROKEN.md +0 -49
  109. package/articles/TWEETS_WHY_BUILD.md +0 -54
  110. package/articles/TWEETS_routerarena_leader.md +0 -53
  111. package/articles/TWEET_STORM_READY.md +0 -165
  112. package/articles/TWITTER_FINAL.md +0 -167
  113. package/articles/WHY_10X_BETTER.md +0 -261
  114. package/articles/WHY_CHINESE_STYLE_BETTER.md +0 -323
  115. package/articles/ai-discoverability-llm-routing.md +0 -210
  116. package/articles/devto-llm-routing.md +0 -138
  117. package/articles/hackernews-show-hn.md +0 -54
  118. package/articles/hashnode-llm-cost-optimization.md +0 -125
  119. package/articles/hn_show_2026_05.md +0 -11
  120. package/articles/medium-building-llm-router.md +0 -205
  121. package/articles/reddit-ml.md +0 -76
  122. package/articles/twitter-thread-cost-savings.md +0 -50
  123. package/articles/youtube-tutorial-script.md +0 -262
  124. package/assets/a3m_3blue1brown.mp4 +0 -0
  125. package/assets/banner.svg +0 -109
  126. package/assets/chart-cost-v2.svg +0 -91
  127. package/assets/chart-cost-v3.svg +0 -143
  128. package/assets/chart-features-v2.svg +0 -132
  129. package/assets/chart-features-v3.svg +0 -211
  130. package/assets/chart-growth-v2.svg +0 -122
  131. package/assets/chart-growth-v3.svg +0 -189
  132. package/assets/cost-comparison.svg +0 -134
  133. package/assets/cost-simple.svg +0 -64
  134. package/assets/demo-hn.gif +0 -0
  135. package/assets/feature-matrix.svg +0 -136
  136. package/assets/growth-chart-animated.svg +0 -76
  137. package/assets/growth-chart.svg +0 -82
  138. package/assets/growth-simple.svg +0 -69
  139. package/assets/hero-diagram.svg +0 -81
  140. package/assets/logo-new.svg +0 -21
  141. package/assets/logo.svg +0 -68
  142. package/assets/provider-comparison.svg +0 -121
  143. package/assets/social-preview-new.svg +0 -100
  144. package/assets/social-preview.svg +0 -194
  145. package/assets/social-v2.svg +0 -130
  146. package/assets/social-v3.svg +0 -212
  147. package/benchmark-provider-results.json +0 -245
  148. package/benchmark-results.json +0 -54
  149. package/council-votes/architecture-vote.md +0 -121
  150. package/council-votes/coverage-vote.md +0 -93
  151. package/data/adaptive-benchmark.json +0 -92
  152. package/data/benchmark-results.json +0 -47
  153. package/data/labeled-benchmark.json +0 -88
  154. package/demo/3blue1brown_video.py +0 -285
  155. package/demo/3blue1brown_video_v2.py +0 -310
  156. package/demo/IMPROVED_PROMPTS.md +0 -229
  157. package/demo/VEO3_PROMPTS.md +0 -269
  158. package/demo/VIDEO_PRODUCTION_GUIDE.md +0 -333
  159. package/demo/a3m_3blue1brown.mp4 +0 -0
  160. package/demo/asciinema-demo.sh +0 -195
  161. package/demo/demo-hn.tape +0 -74
  162. package/demo/demo-script.md +0 -53
  163. package/demo/demo-script.sh +0 -62
  164. package/demo/demo.svg +0 -75
  165. package/demo/frame1_ai_data_center.png +0 -0
  166. package/demo/frame1_sunset_video.mp4 +0 -0
  167. package/demo/frame2_cost_comparison.png +0 -0
  168. package/demo/frame2_cost_comparison_fallback.png +0 -0
  169. package/demo/frame3_parallel_execution.png +0 -0
  170. package/demo/frame3_parallel_execution_fallback.png +0 -0
  171. package/demo/frame4_providers.png +0 -0
  172. package/demo/frame4_providers_fallback.png +0 -0
  173. package/demo/frame5_endcard.png +0 -0
  174. package/demo/frame5_endcard_fallback.png +0 -0
  175. package/demo/new_frame1_hook.png +0 -0
  176. package/demo/new_frame2_proof.png +0 -0
  177. package/demo/new_frame3_wow.png +0 -0
  178. package/demo/new_frame4_social.png +0 -0
  179. package/demo/new_frame5_cta.png +0 -0
  180. package/demo/package.json +0 -13
  181. package/demo/product-video-final.mp4 +0 -0
  182. package/demo/product-video-hype-v1.mp4 +0 -0
  183. package/demo/product-video-v1.mp4 +0 -0
  184. package/demo/public/index.html +0 -762
  185. package/demo/recording.cast +0 -55
  186. package/demo/server.js +0 -405
  187. package/demo-new.tape +0 -71
  188. package/demo-real.sh +0 -198
  189. package/demo-simple.tape +0 -205
  190. package/demo.html +0 -520
  191. package/demo.sh +0 -85
  192. package/demo.tape +0 -259
  193. package/dist/analytics/costAnalytics.d.ts.map +0 -1
  194. package/dist/analytics/costAnalytics.js.map +0 -1
  195. package/dist/benchmark/comprehensive.js.map +0 -1
  196. package/dist/benchmark/reproducible.d.ts.map +0 -1
  197. package/dist/benchmark/reproducible.js.map +0 -1
  198. package/dist/cache/prefixCache.d.ts.map +0 -1
  199. package/dist/cache/prefixCache.js.map +0 -1
  200. package/dist/cache/responseCache.d.ts.map +0 -1
  201. package/dist/cache/responseCache.js.map +0 -1
  202. package/dist/cache/semanticCache.d.ts.map +0 -1
  203. package/dist/cache/semanticCache.js.map +0 -1
  204. package/dist/cli/setupWizard.d.ts.map +0 -1
  205. package/dist/cli/setupWizard.js.map +0 -1
  206. package/dist/cost/budgetEnforcer.d.ts.map +0 -1
  207. package/dist/cost/budgetEnforcer.js.map +0 -1
  208. package/dist/cost/costTracker.d.ts.map +0 -1
  209. package/dist/cost/costTracker.js.map +0 -1
  210. package/dist/ensemble/multiRoundDialog.js.map +0 -1
  211. package/dist/ensemble/shapleyValue.js.map +0 -1
  212. package/dist/integrations/langchainAdapter.d.ts.map +0 -1
  213. package/dist/integrations/langchainAdapter.js.map +0 -1
  214. package/dist/integrations/oauth.d.ts.map +0 -1
  215. package/dist/integrations/oauth.js.map +0 -1
  216. package/dist/integrations/scienceAdapter.js.map +0 -1
  217. package/dist/memory/autoFetch.d.ts.map +0 -1
  218. package/dist/memory/autoFetch.js.map +0 -1
  219. package/dist/memory/episodicMemory.d.ts.map +0 -1
  220. package/dist/memory/episodicMemory.js.map +0 -1
  221. package/dist/memory/hybridMemory.js.map +0 -1
  222. package/dist/memory/memoryTree.d.ts.map +0 -1
  223. package/dist/memory/memoryTree.js.map +0 -1
  224. package/dist/memory/obsidianVault.d.ts.map +0 -1
  225. package/dist/memory/obsidianVault.js.map +0 -1
  226. package/dist/memory/reasoningBank.js.map +0 -1
  227. package/dist/observability/changeWatch.d.ts.map +0 -1
  228. package/dist/observability/changeWatch.js.map +0 -1
  229. package/dist/observability/fatigueDetector.d.ts.map +0 -1
  230. package/dist/observability/fatigueDetector.js.map +0 -1
  231. package/dist/observability/index.d.ts.map +0 -1
  232. package/dist/observability/index.js.map +0 -1
  233. package/dist/observability/metrics.d.ts.map +0 -1
  234. package/dist/observability/metrics.js.map +0 -1
  235. package/dist/observability/middleware.d.ts.map +0 -1
  236. package/dist/observability/middleware.js.map +0 -1
  237. package/dist/observability/tracer.d.ts.map +0 -1
  238. package/dist/observability/tracer.js.map +0 -1
  239. package/dist/observability/types.d.ts.map +0 -1
  240. package/dist/observability/types.js.map +0 -1
  241. package/dist/orchestration/haloOrchestrator.d.ts.map +0 -1
  242. package/dist/orchestration/haloOrchestrator.js.map +0 -1
  243. package/dist/orchestration/mctsWorkflow.d.ts.map +0 -1
  244. package/dist/orchestration/mctsWorkflow.js.map +0 -1
  245. package/dist/providers/localProvider.d.ts.map +0 -1
  246. package/dist/providers/localProvider.js.map +0 -1
  247. package/dist/providers/providerConfig.d.ts.map +0 -1
  248. package/dist/providers/providerConfig.js.map +0 -1
  249. package/dist/providers/registry.d.ts.map +0 -1
  250. package/dist/providers/registry.js.map +0 -1
  251. package/dist/routing/advancedRouter.d.ts.map +0 -1
  252. package/dist/routing/advancedRouter.js.map +0 -1
  253. package/dist/routing/crossModelValidation.d.ts.map +0 -1
  254. package/dist/routing/crossModelValidation.js.map +0 -1
  255. package/dist/routing/providerHealth.d.ts.map +0 -1
  256. package/dist/routing/providerHealth.js.map +0 -1
  257. package/dist/routing/providerRetry.d.ts.map +0 -1
  258. package/dist/routing/providerRetry.js.map +0 -1
  259. package/dist/scripts/banner.js +0 -29
  260. package/dist/security/guardrails.d.ts.map +0 -1
  261. package/dist/security/guardrails.js.map +0 -1
  262. package/dist/server/dashboard.d.ts.map +0 -1
  263. package/dist/server/dashboard.js.map +0 -1
  264. package/dist/server/modelMapper.d.ts.map +0 -1
  265. package/dist/server/modelMapper.js.map +0 -1
  266. package/dist/server/proxyServer.d.ts.map +0 -1
  267. package/dist/server/proxyServer.js.map +0 -1
  268. package/dist/skills/__tests__/skill_manager.test.d.ts +0 -2
  269. package/dist/skills/__tests__/skill_manager.test.d.ts.map +0 -1
  270. package/dist/skills/__tests__/skill_manager.test.js +0 -268
  271. package/dist/skills/__tests__/skill_manager.test.js.map +0 -1
  272. package/dist/tools/tmlpdTools.d.ts.map +0 -1
  273. package/dist/tools/tmlpdTools.js.map +0 -1
  274. package/dist/tui/dashboard.d.ts.map +0 -1
  275. package/dist/tui/dashboard.js.map +0 -1
  276. package/dist/tui/index.d.ts.map +0 -1
  277. package/dist/tui/index.js.map +0 -1
  278. package/dist/utils/batchProcessor.d.ts.map +0 -1
  279. package/dist/utils/batchProcessor.js.map +0 -1
  280. package/dist/utils/compression.d.ts.map +0 -1
  281. package/dist/utils/compression.js.map +0 -1
  282. package/dist/utils/costUtils.d.ts.map +0 -1
  283. package/dist/utils/costUtils.js.map +0 -1
  284. package/dist/utils/reliability.d.ts.map +0 -1
  285. package/dist/utils/reliability.js.map +0 -1
  286. package/dist/utils/sorting.d.ts.map +0 -1
  287. package/dist/utils/sorting.js.map +0 -1
  288. package/dist/utils/speculativeDecoding.d.ts.map +0 -1
  289. package/dist/utils/speculativeDecoding.js.map +0 -1
  290. package/dist/utils/tokenUtils.d.ts.map +0 -1
  291. package/dist/utils/tokenUtils.js.map +0 -1
  292. package/docs/.nojekyll +0 -0
  293. package/docs/ANALYSIS_PRINCIPLES.md +0 -162
  294. package/docs/API.md +0 -855
  295. package/docs/ARCHITECTURAL-IMPROVEMENTS-2025.md +0 -1391
  296. package/docs/ARCHITECTURAL-IMPROVEMENTS-REVISED-2025.md +0 -1051
  297. package/docs/BENCHMARK.md +0 -170
  298. package/docs/CHINESE_PROVIDER_RELIABILITY.md +0 -37
  299. package/docs/CITATIONS.md +0 -74
  300. package/docs/CLAIMS_AND_EVIDENCE.md +0 -58
  301. package/docs/CONFIGURATION.md +0 -476
  302. package/docs/COUNCIL_DECISION.json +0 -816
  303. package/docs/COUNCIL_SUMMARY.md +0 -319
  304. package/docs/COUNCIL_V2.2_DECISION.md +0 -416
  305. package/docs/ENGINEERING_SPEC.md +0 -55
  306. package/docs/FACTORY_RESET.md +0 -34
  307. package/docs/GEO.md +0 -66
  308. package/docs/GEO_OPTIMIZATION.md +0 -30
  309. package/docs/GEO_ROOT_CAUSE.md +0 -136
  310. package/docs/GEO_STATUS.md +0 -85
  311. package/docs/GEO_TEST_RESULTS.md +0 -176
  312. package/docs/HN_CHECKLIST.md +0 -38
  313. package/docs/HN_FOUNDER_COMMENT.md +0 -17
  314. package/docs/HN_SUBMISSION_FINAL.md +0 -180
  315. package/docs/HN_SUBMISSION_V3.md +0 -56
  316. package/docs/IMPROVEMENT_ROADMAP.md +0 -515
  317. package/docs/INTEGRATIONS.md +0 -420
  318. package/docs/LANGCHAIN_INTEGRATION.md +0 -147
  319. package/docs/LLM_COUNCIL_DECISION.md +0 -508
  320. package/docs/MIDDLEWARE_CHAIN.md +0 -35
  321. package/docs/PROMO_CHECKLIST.md +0 -200
  322. package/docs/QUICKSTART.md +0 -271
  323. package/docs/QUICK_START.md +0 -43
  324. package/docs/QUICK_START_VISIBILITY.md +0 -782
  325. package/docs/REDDIT_GAP_ANALYSIS.md +0 -299
  326. package/docs/RELEASE_CHECKLIST.md +0 -32
  327. package/docs/REPRODUCIBILITY.md +0 -63
  328. package/docs/RESEARCH_BACKED_IMPROVEMENTS.md +0 -1180
  329. package/docs/ROUTING_RUBRIC.md +0 -197
  330. package/docs/SEO_AUDIT.md +0 -186
  331. package/docs/SOCIAL_LISTENING.md +0 -219
  332. package/docs/TMLPD_QNA.md +0 -751
  333. package/docs/TMLPD_V2.1_COMPLETE.md +0 -763
  334. package/docs/TMLPD_V2.2_RESEARCH_ROADMAP.md +0 -754
  335. package/docs/UPDATE_TOPICS.md +0 -15
  336. package/docs/USE_CASES.md +0 -59
  337. package/docs/V2.2_IMPLEMENTATION_COMPLETE.md +0 -446
  338. package/docs/V2_IMPLEMENTATION_GUIDE.md +0 -388
  339. package/docs/VERCEL_AI_SDK.md +0 -209
  340. package/docs/VISIBILITY_ADOPTION_PLAN.md +0 -1005
  341. package/docs/_config.yml +0 -49
  342. package/docs/ai-plugin.json +0 -16
  343. package/docs/api.html +0 -513
  344. package/docs/architecture-diagram.md +0 -40
  345. package/docs/benchmark-chart.png +0 -0
  346. package/docs/benchmark.html +0 -387
  347. package/docs/blog/routerarena-number-one.html +0 -73
  348. package/docs/cli-cheatsheet.md +0 -339
  349. package/docs/compare.md +0 -109
  350. package/docs/comparison-litellm.md +0 -88
  351. package/docs/comparison.md +0 -108
  352. package/docs/cost-chart-ascii.md +0 -42
  353. package/docs/cost-comparison-chart.svg +0 -88
  354. package/docs/curl-examples.md +0 -247
  355. package/docs/demo-auto.html +0 -264
  356. package/docs/demo.html +0 -416
  357. package/docs/geo/GENERATIVE_ENGINE_OPTIMIZATION.md +0 -232
  358. package/docs/index.html +0 -507
  359. package/docs/launch-content/LAUNCH_EXECUTION_CHECKLIST.md +0 -421
  360. package/docs/launch-content/README.md +0 -457
  361. package/docs/launch-content/assets/cost_comparison_100_tasks.png +0 -0
  362. package/docs/launch-content/assets/cumulative_savings.png +0 -0
  363. package/docs/launch-content/assets/parallel_speedup.png +0 -0
  364. package/docs/launch-content/assets/provider_pricing_comparison.png +0 -0
  365. package/docs/launch-content/assets/task_breakdown_comparison.png +0 -0
  366. package/docs/launch-content/generate_charts.py +0 -313
  367. package/docs/launch-content/hn_show_post.md +0 -139
  368. package/docs/launch-content/partner_outreach_templates.md +0 -745
  369. package/docs/launch-content/reddit_posts.md +0 -467
  370. package/docs/launch-content/twitter_thread.txt +0 -460
  371. package/docs/npm-downloads-chart.svg +0 -43
  372. package/docs/openapi.json +0 -139
  373. package/docs/openapi.yaml +0 -1318
  374. package/docs/quick-start.html +0 -366
  375. package/docs/robots.txt +0 -52
  376. package/docs/sitemap.xml +0 -57
  377. package/docs/styles.css +0 -682
  378. package/docs/well-known/ai-plugin.json +0 -16
  379. package/docs/wellknown/ai-plugin.json +0 -16
  380. package/docs-site/assets/og-banner.svg +0 -194
  381. package/docs-site/index.html +0 -632
  382. package/eval/README.md +0 -46
  383. package/eval/baselines/main.json +0 -12
  384. package/eval/benchmark_dataset.jsonl +0 -16
  385. package/eval/check_golden_routes.js +0 -64
  386. package/eval/datasets/catalog.json +0 -33
  387. package/eval/datasets/slices/cn_provider_reliability_v1.jsonl +0 -3
  388. package/eval/datasets/slices/cost_pressure_v1.jsonl +0 -3
  389. package/eval/datasets/slices/safety_guardrails_v1.jsonl +0 -3
  390. package/eval/evals.json +0 -199
  391. package/eval/fault_injection_thresholds.json +0 -3
  392. package/eval/generate_report.js +0 -128
  393. package/eval/golden_routes.json +0 -114
  394. package/eval/lib/experiment_registry.js +0 -24
  395. package/eval/run_eval.js +0 -197
  396. package/eval/run_fault_injection.js +0 -201
  397. package/eval/run_shadow_eval.js +0 -85
  398. package/eval/thresholds.json +0 -9
  399. package/examples/QUICKSTART.md +0 -183
  400. package/examples/README.md +0 -61
  401. package/examples/a3m-sdk.js +0 -124
  402. package/examples/basic-route.js +0 -54
  403. package/examples/chat-loop.js +0 -202
  404. package/examples/classify-then-route.js +0 -102
  405. package/examples/cost-compare.js +0 -120
  406. package/examples/ensemble.js +0 -160
  407. package/examples/whatsapp-telegram-bridge-demo.js +0 -302
  408. package/examples/whatsapp-telegram-bridge.js +0 -269
  409. package/hf-space/README.md +0 -23
  410. package/hf-space/app.py +0 -240
  411. package/hf-space/requirements.txt +0 -1
  412. package/huggingface_space/README.md +0 -35
  413. package/huggingface_space/app.py +0 -126
  414. package/huggingface_space/create_space.py +0 -208
  415. package/huggingface_space/requirements.txt +0 -1
  416. package/mcp-server/README.md +0 -188
  417. package/mcp-server/package.json +0 -29
  418. package/mcp-server/src/index.ts +0 -744
  419. package/mcp-server/tsconfig.json +0 -19
  420. package/openclaw-alexa-bridge/ALL_REMAINING_FIXES_PLAN.md +0 -313
  421. package/openclaw-alexa-bridge/REMAINING_FIXES_SUMMARY.md +0 -277
  422. package/openclaw-alexa-bridge/src/alexa_handler_no_tmlpd.js +0 -1234
  423. package/openclaw-alexa-bridge/test_fixes.js +0 -77
  424. package/playground/README.md +0 -51
  425. package/playground/codesandbox.json +0 -12
  426. package/playground/index.js +0 -39
  427. package/proxy/README.md +0 -227
  428. package/proxy/package-lock.json +0 -831
  429. package/proxy/package.json +0 -17
  430. package/proxy/rate-limit.js +0 -145
  431. package/proxy/rate-limit.test.js +0 -311
  432. package/proxy/server.js +0 -970
  433. package/python/README.md +0 -102
  434. package/python/a3m/__init__.py +0 -6
  435. package/python/a3m/client.py +0 -190
  436. package/python/a3m/models.py +0 -40
  437. package/python/a3m/sync_client.py +0 -61
  438. package/python/examples.py +0 -53
  439. package/python/integrations.py +0 -330
  440. package/python/pyproject.toml +0 -23
  441. package/python/setup.py +0 -28
  442. package/python/tmlpd.py +0 -369
  443. package/qna/REDDIT_GAP_ANALYSIS.md +0 -299
  444. package/qna/TMLPD_QNA.md +0 -751
  445. package/research/FINDING_001_safety.md +0 -28
  446. package/research/FINDING_002_error_diversity.md +0 -32
  447. package/research/FINDING_003_confidence_weighted_voting.md +0 -32
  448. package/research/FINDING_004_cross_model_semantic_detection.md +0 -37
  449. package/research/FINDING_005_knowledge_gap_orthogonality.md +0 -34
  450. package/research/HALLUCINATION_RESEARCH.md +0 -27
  451. package/research/ensemble-voting.md +0 -324
  452. package/research/loss-functions.md +0 -545
  453. package/research-log.md +0 -49
  454. package/scripts/banner.js +0 -29
  455. package/scripts/benchmark-local-routerarena.ts +0 -176
  456. package/scripts/benchmark.js +0 -145
  457. package/scripts/benchmark.sh +0 -61
  458. package/scripts/compare-providers.sh +0 -230
  459. package/scripts/content-planner.js +0 -25
  460. package/scripts/create-labeled-benchmark.ts +0 -105
  461. package/scripts/cross_post.py +0 -443
  462. package/scripts/local-router-benchmark.ts +0 -154
  463. package/scripts/post-all.sh +0 -41
  464. package/scripts/publish_fcc.py +0 -106
  465. package/scripts/push-to-gitee.sh +0 -25
  466. package/scripts/routerarena_ensemble.js +0 -144
  467. package/scripts/routing-benchmark-v2.js +0 -373
  468. package/scripts/routing-benchmark-v3.js +0 -118
  469. package/scripts/routing-benchmark.js +0 -462
  470. package/scripts/run-labeled-benchmark.mjs +0 -104
  471. package/scripts/run-mmlu-benchmark.js +0 -176
  472. package/scripts/run-provider-benchmark.js +0 -244
  473. package/scripts/update-npm-badges.js +0 -158
  474. package/skill/SKILL.md +0 -238
  475. package/src/__tests__/integration/tmpld_integration.test.py +0 -540
  476. package/src/skills/__tests__/skill_manager.test.ts +0 -328
  477. package/submissions/benchmarks/ALL_PLATFORMS_SUBMISSION.md +0 -94
  478. package/submissions/benchmarks/LLMROUTERBENCH_SUBMISSION.md +0 -121
  479. package/submissions/benchmarks/MMRBENCH_SUBMISSION.md +0 -94
  480. package/submissions/benchmarks/ROUTERARENA_UPDATE.md +0 -83
  481. package/submissions/benchmarks/ROUTERBENCH_SUBMISSION.md +0 -225
  482. package/test-council/1-structure-tests.test.js +0 -353
  483. package/test-council/1-structure-tests.test.ts +0 -353
  484. package/test-council/2-edge-case-tests.test.ts +0 -361
  485. package/test-council/3-performance-tests.test.ts +0 -669
  486. package/test-council/4-integration-tests.test.ts +0 -391
  487. package/test-council/5-agent-council-eval.test.ts +0 -413
  488. package/test-council/AGENT_COUNCIL_ARCHITECTURE.md +0 -349
  489. package/test-council/TEST_COUNCIL_REPORT.md +0 -201
  490. package/test-council/agents/edge-case-agent.ts +0 -363
  491. package/test-council/agents/performance-agent.ts +0 -426
  492. package/test-council/agents/structure-agent.ts +0 -227
  493. package/test-council/council.md +0 -183
  494. package/tests/__mocks__/tokenUtils.ts +0 -8
  495. package/tests/memory/episodicMemory.test.ts +0 -227
  496. package/tests/package-lock.json +0 -1628
  497. package/tests/package.json +0 -18
  498. package/tests/routing/ensembleVoting.test.ts +0 -236
  499. package/tests/routing/providerRetry.test.ts +0 -360
  500. package/tests/routing/queryTypePresets.test.ts +0 -208
  501. package/tests/security/guardrailEngine.test.ts +0 -700
  502. package/tests/tsconfig.json +0 -21
  503. package/tests/vitest.config.ts +0 -18
  504. package/tmlpd-pi-extension/README.md +0 -66
  505. package/tmlpd-pi-extension/dist/cache/prefixCache.d.ts +0 -114
  506. package/tmlpd-pi-extension/dist/cache/prefixCache.d.ts.map +0 -1
  507. package/tmlpd-pi-extension/dist/cache/prefixCache.js +0 -285
  508. package/tmlpd-pi-extension/dist/cache/prefixCache.js.map +0 -1
  509. package/tmlpd-pi-extension/dist/cache/responseCache.d.ts +0 -58
  510. package/tmlpd-pi-extension/dist/cache/responseCache.d.ts.map +0 -1
  511. package/tmlpd-pi-extension/dist/cache/responseCache.js +0 -153
  512. package/tmlpd-pi-extension/dist/cache/responseCache.js.map +0 -1
  513. package/tmlpd-pi-extension/dist/cli.js +0 -59
  514. package/tmlpd-pi-extension/dist/cost/costTracker.d.ts +0 -95
  515. package/tmlpd-pi-extension/dist/cost/costTracker.d.ts.map +0 -1
  516. package/tmlpd-pi-extension/dist/cost/costTracker.js +0 -240
  517. package/tmlpd-pi-extension/dist/cost/costTracker.js.map +0 -1
  518. package/tmlpd-pi-extension/dist/index.d.ts +0 -723
  519. package/tmlpd-pi-extension/dist/index.d.ts.map +0 -1
  520. package/tmlpd-pi-extension/dist/index.js +0 -239
  521. package/tmlpd-pi-extension/dist/index.js.map +0 -1
  522. package/tmlpd-pi-extension/dist/memory/episodicMemory.d.ts +0 -82
  523. package/tmlpd-pi-extension/dist/memory/episodicMemory.d.ts.map +0 -1
  524. package/tmlpd-pi-extension/dist/memory/episodicMemory.js +0 -145
  525. package/tmlpd-pi-extension/dist/memory/episodicMemory.js.map +0 -1
  526. package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.d.ts +0 -102
  527. package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.d.ts.map +0 -1
  528. package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.js +0 -207
  529. package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.js.map +0 -1
  530. package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.d.ts +0 -85
  531. package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.d.ts.map +0 -1
  532. package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.js +0 -210
  533. package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.js.map +0 -1
  534. package/tmlpd-pi-extension/dist/providers/localProvider.d.ts +0 -102
  535. package/tmlpd-pi-extension/dist/providers/localProvider.d.ts.map +0 -1
  536. package/tmlpd-pi-extension/dist/providers/localProvider.js +0 -338
  537. package/tmlpd-pi-extension/dist/providers/localProvider.js.map +0 -1
  538. package/tmlpd-pi-extension/dist/providers/registry.d.ts +0 -55
  539. package/tmlpd-pi-extension/dist/providers/registry.d.ts.map +0 -1
  540. package/tmlpd-pi-extension/dist/providers/registry.js +0 -138
  541. package/tmlpd-pi-extension/dist/providers/registry.js.map +0 -1
  542. package/tmlpd-pi-extension/dist/routing/advancedRouter.d.ts +0 -68
  543. package/tmlpd-pi-extension/dist/routing/advancedRouter.d.ts.map +0 -1
  544. package/tmlpd-pi-extension/dist/routing/advancedRouter.js +0 -332
  545. package/tmlpd-pi-extension/dist/routing/advancedRouter.js.map +0 -1
  546. package/tmlpd-pi-extension/dist/tools/tmlpdTools.d.ts +0 -101
  547. package/tmlpd-pi-extension/dist/tools/tmlpdTools.d.ts.map +0 -1
  548. package/tmlpd-pi-extension/dist/tools/tmlpdTools.js +0 -368
  549. package/tmlpd-pi-extension/dist/tools/tmlpdTools.js.map +0 -1
  550. package/tmlpd-pi-extension/dist/utils/batchProcessor.d.ts +0 -96
  551. package/tmlpd-pi-extension/dist/utils/batchProcessor.d.ts.map +0 -1
  552. package/tmlpd-pi-extension/dist/utils/batchProcessor.js +0 -170
  553. package/tmlpd-pi-extension/dist/utils/batchProcessor.js.map +0 -1
  554. package/tmlpd-pi-extension/dist/utils/compression.d.ts +0 -61
  555. package/tmlpd-pi-extension/dist/utils/compression.d.ts.map +0 -1
  556. package/tmlpd-pi-extension/dist/utils/compression.js +0 -281
  557. package/tmlpd-pi-extension/dist/utils/compression.js.map +0 -1
  558. package/tmlpd-pi-extension/dist/utils/reliability.d.ts +0 -74
  559. package/tmlpd-pi-extension/dist/utils/reliability.d.ts.map +0 -1
  560. package/tmlpd-pi-extension/dist/utils/reliability.js +0 -177
  561. package/tmlpd-pi-extension/dist/utils/reliability.js.map +0 -1
  562. package/tmlpd-pi-extension/dist/utils/speculativeDecoding.d.ts +0 -117
  563. package/tmlpd-pi-extension/dist/utils/speculativeDecoding.d.ts.map +0 -1
  564. package/tmlpd-pi-extension/dist/utils/speculativeDecoding.js +0 -246
  565. package/tmlpd-pi-extension/dist/utils/speculativeDecoding.js.map +0 -1
  566. package/tmlpd-pi-extension/dist/utils/tokenUtils.d.ts +0 -50
  567. package/tmlpd-pi-extension/dist/utils/tokenUtils.d.ts.map +0 -1
  568. package/tmlpd-pi-extension/dist/utils/tokenUtils.js +0 -124
  569. package/tmlpd-pi-extension/dist/utils/tokenUtils.js.map +0 -1
  570. package/tmlpd-pi-extension/examples/QUICKSTART.md +0 -183
  571. package/tmlpd-pi-extension/package-lock.json +0 -79
  572. package/tmlpd-pi-extension/package.json +0 -172
  573. package/tmlpd-pi-extension/python/examples.py +0 -53
  574. package/tmlpd-pi-extension/python/integrations.py +0 -330
  575. package/tmlpd-pi-extension/python/setup.py +0 -28
  576. package/tmlpd-pi-extension/python/tmlpd.py +0 -369
  577. package/tmlpd-pi-extension/qna/REDDIT_GAP_ANALYSIS.md +0 -299
  578. package/tmlpd-pi-extension/qna/TMLPD_QNA.md +0 -751
  579. package/tmlpd-pi-extension/skill/SKILL.md +0 -238
  580. package/tmlpd-pi-extension/src/cache/responseCache.ts +0 -147
  581. package/tmlpd-pi-extension/src/cost/costTracker.ts +0 -302
  582. package/tmlpd-pi-extension/src/index.ts +0 -232
  583. package/tmlpd-pi-extension/src/memory/episodicMemory.ts +0 -257
  584. package/tmlpd-pi-extension/src/orchestration/haloOrchestrator.ts +0 -266
  585. package/tmlpd-pi-extension/src/orchestration/mctsWorkflow.ts +0 -262
  586. package/tmlpd-pi-extension/src/providers/localProvider.ts +0 -406
  587. package/tmlpd-pi-extension/src/providers/registry.ts +0 -164
  588. package/tmlpd-pi-extension/src/routing/ensembleVoting.ts +0 -159
  589. package/tmlpd-pi-extension/src/routing/queryTypePresets.ts +0 -136
  590. package/tmlpd-pi-extension/src/tools/tmlpdTools.ts +0 -433
  591. package/tmlpd-pi-extension/src/utils/batchProcessor.ts +0 -232
  592. package/tmlpd-pi-extension/src/utils/compression.ts +0 -325
  593. package/tmlpd-pi-extension/src/utils/reliability.ts +0 -221
  594. package/tmlpd-pi-extension/src/utils/tokenUtils.ts +0 -145
  595. package/tmlpd-pi-extension/tsconfig.json +0 -18
  596. package/tsconfig.build.json +0 -29
  597. package/tsconfig.json +0 -18
  598. /package/{docs/llms-full.txt → llms-full.txt.bak} +0 -0
@@ -1,176 +0,0 @@
1
- /**
2
- * Local benchmark using RouterArena cached queries.
3
- *
4
- * We use the cached results to determine ground truth:
5
- * - A query's "actual tier" is based on which model produced the BEST answer
6
- * - We compare our routing decision against this ground truth
7
- */
8
-
9
- import * as fs from 'fs';
10
- import * as path from 'path';
11
- import { routeQuery, createRouterContext } from '../src/index';
12
-
13
- // Load cached results from RouterArena
14
- const CACHE_DIR = '/tmp/routerarena/cached_results';
15
-
16
- interface CachedResult {
17
- global_index: string;
18
- generated_answer: string;
19
- success: boolean;
20
- token_usage: { input_tokens: number; output_tokens: number };
21
- provider: string;
22
- }
23
-
24
- interface GroundTruthTier {
25
- query: string;
26
- global_index: string;
27
- actualTier: 'free' | 'cheap' | 'mid' | 'premium';
28
- groundTruthModel: string;
29
- }
30
-
31
- interface BenchmarkResult {
32
- timestamp: string;
33
- version: string;
34
- total: number;
35
- correct: number;
36
- accuracy: number;
37
- confusion: { [key: string]: { [key: string]: number } };
38
- tierAccuracy: { [tier: string]: { correct: number; total: number } };
39
- sampleQueries: { query: string; actual: string; predicted: string; correct: boolean }[];
40
- }
41
-
42
- async function loadCachedResults(model: string): Promise<{ [key: string]: CachedResult }> {
43
- const results: { [key: string]: CachedResult } = {};
44
- const filePath = path.join(CACHE_DIR, `${model}.jsonl`);
45
-
46
- if (!fs.existsSync(filePath)) {
47
- console.log(`Cache not found: ${filePath}`);
48
- return results;
49
- }
50
-
51
- const content = fs.readFileSync(filePath, 'utf-8');
52
- for (const line of content.split('\n')) {
53
- if (line.trim()) {
54
- const entry = JSON.parse(line);
55
- results[entry.global_index] = entry;
56
- }
57
- }
58
-
59
- return results;
60
- }
61
-
62
- function inferTierFromModel(model: string): 'free' | 'cheap' | 'mid' | 'premium' {
63
- const m = model.toLowerCase();
64
- if (m.includes('deepseek')) return 'free';
65
- if (m.includes('gpt') && m.includes('mini')) return 'cheap';
66
- if (m.includes('mistral') || m.includes('ministral')) return 'mid';
67
- if (m.includes('gemini') || m.includes('claude') || m.includes('gpt-4')) return 'premium';
68
- return 'mid';
69
- }
70
-
71
- async function runBenchmark(): Promise<void> {
72
- console.log('Loading cached results...');
73
-
74
- // Load cached results for all models
75
- const deepseekResults = await loadCachedResults('deepseek-chat');
76
- const gptResults = await loadCachedResults('gpt-4o-mini');
77
- const haikuResults = await loadCachedResults('claude-3-haiku-20240307');
78
- const geminiResults = await loadCachedResults('gemini-2.0-flash-001');
79
-
80
- console.log(`Loaded: deepseek=${Object.keys(deepseekResults).length}, gpt=${Object.keys(gptResults).length}, haiku=${Object.keys(haikuResults).length}, gemini=${Object.keys(geminiResults).length}`);
81
-
82
- // Create ground truth: for each query, which model is best?
83
- // We need a sample query text - we'll use the global_index as identifier
84
- const queries = Object.keys(deepseekResults).slice(0, 500); // Sample 500 for speed
85
-
86
- console.log(`\nBenchmarking ${queries.length} queries...`);
87
-
88
- // Create router context
89
- const context = await createRouterContext();
90
-
91
- const confusion: { [key: string]: { [key: string]: number } } = {
92
- 'free': {}, 'cheap': {}, 'mid': {}, 'premium': {}
93
- };
94
-
95
- const tierAccuracy: { [key: string]: { correct: number; total: number } } = {
96
- 'free': { correct: 0, total: 0 },
97
- 'cheap': { correct: 0, total: 0 },
98
- 'mid': { correct: 0, total: 0 },
99
- 'premium': { correct: 0, total: 0 }
100
- };
101
-
102
- let correct = 0;
103
- const sampleQueries: BenchmarkResult['sampleQueries'] = [];
104
-
105
- for (const idx of queries) {
106
- const deepseekResult = deepseekResults[idx];
107
-
108
- // Infer the "actual" tier based on which model would be best
109
- // For simplicity, use deepseek as reference (it's the most capable free model)
110
- const actualTier = 'free'; // Default assumption
111
-
112
- // Route using our router
113
- const query = `[${idx}] ${deepseekResult?.generated_answer?.substring(0, 100) || 'query'}`;
114
-
115
- try {
116
- const result = await routeQuery(idx, context);
117
- const predictedTier = inferTierFromModel(result.model);
118
-
119
- const isCorrect = predictedTier === actualTier;
120
- if (isCorrect) correct++;
121
-
122
- // Update confusion
123
- if (!confusion[actualTier][predictedTier]) confusion[actualTier][predictedTier] = 0;
124
- confusion[actualTier][predictedTier]++;
125
-
126
- tierAccuracy[actualTier].total++;
127
- if (isCorrect) tierAccuracy[actualTier].correct++;
128
-
129
- if (sampleQueries.length < 50) {
130
- sampleQueries.push({
131
- query: idx,
132
- actual: actualTier,
133
- predicted: predictedTier,
134
- correct: isCorrect
135
- });
136
- }
137
- } catch (e) {
138
- console.error(`Error routing ${idx}: ${e}`);
139
- }
140
- }
141
-
142
- const result: BenchmarkResult = {
143
- timestamp: new Date().toISOString(),
144
- version: '2.14.26',
145
- total: queries.length,
146
- correct,
147
- accuracy: correct / queries.length,
148
- confusion,
149
- tierAccuracy,
150
- sampleQueries
151
- };
152
-
153
- console.log('\n=== Local Benchmark Results ===');
154
- console.log(`Total: ${result.total}, Correct: ${result.correct}`);
155
- console.log(`Accuracy: ${(result.accuracy * 100).toFixed(1)}%`);
156
- console.log('\nTier Accuracy:');
157
- for (const [tier, stats] of Object.entries(result.tierAccuracy)) {
158
- const pct = stats.total > 0 ? (100 * stats.correct / stats.total).toFixed(0) : 'N/A';
159
- console.log(` ${tier}: ${stats.correct}/${stats.total} (${pct}%)`);
160
- }
161
- console.log('\nConfusion Matrix (rows=actual, cols=predicted):');
162
- console.log(' free cheap mid prem');
163
- for (const actual of ['free', 'cheap', 'mid', 'premium']) {
164
- const row = confusion[actual];
165
- const free = row['free'] || 0;
166
- const cheap = row['cheap'] || 0;
167
- const mid = row['mid'] || 0;
168
- const prem = row['premium'] || 0;
169
- console.log(`${actual.padEnd(8)}${free.toString().padStart(5)} ${cheap.toString().padStart(5)} ${mid.toString().padStart(5)} ${prem.toString().padStart(5)}`);
170
- }
171
-
172
- fs.writeFileSync('./benchmark-local-results.json', JSON.stringify(result, null, 2));
173
- console.log('\nSaved to benchmark-local-results.json');
174
- }
175
-
176
- runBenchmark().catch(console.error);
@@ -1,145 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * A3M Router Real Benchmark
4
- * Runs actual queries through the router and measures cost savings
5
- */
6
-
7
- const QUERY_SET = {
8
- simple: [
9
- "What is 2+2?",
10
- "What is the capital of France?",
11
- "How do you say hello in Spanish?",
12
- "What day is it today?",
13
- "Convert 100 Celsius to Fahrenheit",
14
- "What is the largest planet in the solar system?",
15
- "How many ounces in a pound?",
16
- "What is the speed of light?",
17
- "Who wrote Romeo and Juliet?",
18
- "What is photosynthesis?",
19
- "What is the square root of 144?",
20
- "Name three primary colors",
21
- "What is the chemical symbol for gold?",
22
- "How many continents are there?",
23
- "What is gravity?",
24
- ],
25
- medium: [
26
- "Summarize this article about climate change in 3 bullet points",
27
- "Translate this paragraph from English to French",
28
- "Write a Python function to sort a list",
29
- "Explain the difference between TCP and UDP",
30
- "Write a SQL query to find the top 10 customers by revenue",
31
- "Summarize the key points of the Paris Agreement",
32
- "Create a REST API endpoint in Express.js",
33
- "Explain Docker containers vs virtual machines",
34
- "Write a regex to validate email addresses",
35
- "Describe the water cycle in simple terms",
36
- ],
37
- complex: [
38
- "Analyze the economic implications of AI automation on developing countries",
39
- "Write a detailed technical design document for a microservices architecture",
40
- "Compare and contrast the philosophical frameworks of Kant and Hume on causation",
41
- "Design a distributed caching strategy for a social media platform at scale",
42
- "Critically evaluate the evidence for and against universal basic income",
43
- "Write a comprehensive literature review on transformer architecture improvements since 2020",
44
- "Propose a novel approach to reducing bias in large language model training",
45
- "Architect a real-time collaboration system similar to Google Docs",
46
- "Analyze the geopolitical implications of rare earth mineral supply chains",
47
- "Design an experiment to test the effectiveness of retrieval-augmented generation",
48
- ],
49
- };
50
-
51
- // Expand to 100 queries
52
- const allSimple = Array(47).fill(null).map((_, i) => QUERY_SET.simple[i % QUERY_SET.simple.length]);
53
- const allMedium = Array(33).fill(null).map((_, i) => QUERY_SET.medium[i % QUERY_SET.medium.length]);
54
- const allComplex = Array(20).fill(null).map((_, i) => QUERY_SET.complex[i % QUERY_SET.complex.length]);
55
-
56
- console.log("=== A3M Router Benchmark ===\n");
57
- console.log(`Running ${allSimple.length + allMedium.length + allComplex.length} queries...`);
58
-
59
- try {
60
- const { createA3MRouter } = require("../src/index.js");
61
- const router = createA3MRouter();
62
-
63
- const results = { simple: [], medium: [], complex: [] };
64
- let gpt4Total = 0;
65
- let smartTotal = 0;
66
-
67
- // Cost model per query by type and provider
68
- const costModel = {
69
- simple: { gpt4: 0.0045, groq: 0.00009 },
70
- medium: { gpt4: 0.015, gpt4mini: 0.000075 },
71
- complex: { gpt4: 0.036 },
72
- };
73
-
74
- // Route simple queries
75
- console.log("\nRouting 47 simple queries...");
76
- for (const q of allSimple) {
77
- try {
78
- const result = router.route(q);
79
- results.simple.push(result);
80
- } catch {
81
- results.simple.push({ provider: "groq", simulated: true });
82
- }
83
- gpt4Total += costModel.simple.gpt4;
84
- smartTotal += costModel.simple.groq;
85
- }
86
-
87
- // Route medium queries
88
- console.log("Routing 33 medium queries...");
89
- for (const q of allMedium) {
90
- try {
91
- const result = router.route(q);
92
- results.medium.push(result);
93
- } catch {
94
- results.medium.push({ provider: "gpt4mini", simulated: true });
95
- }
96
- gpt4Total += costModel.medium.gpt4;
97
- smartTotal += costModel.medium.gpt4mini;
98
- }
99
-
100
- // Route complex queries
101
- console.log("Routing 20 complex queries...");
102
- for (const q of allComplex) {
103
- try {
104
- const result = router.route(q);
105
- results.complex.push(result);
106
- } catch {
107
- results.complex.push({ provider: "gpt4", simulated: true });
108
- }
109
- gpt4Total += costModel.complex.gpt4;
110
- smartTotal += costModel.complex.gpt4;
111
- }
112
-
113
- console.log(`\nAll GPT-4o: $${gpt4Total.toFixed(4)}`);
114
- console.log(`A3M Router: $${smartTotal.toFixed(4)}`);
115
- console.log(`Savings: ${((1 - smartTotal / gpt4Total) * 100).toFixed(1)}%`);
116
- } catch (e) {
117
- console.log("Router not available, running simulation mode.\n");
118
-
119
- // Simulate with cost model
120
- const costs = {
121
- simple: { gpt4: 0.0045, groq: 0.00009 },
122
- medium: { gpt4: 0.015, gpt4mini: 0.000075 },
123
- complex: { gpt4: 0.036 },
124
- };
125
-
126
- let gpt4Total = 0;
127
- let smartTotal = 0;
128
-
129
- allSimple.forEach(() => {
130
- gpt4Total += costs.simple.gpt4;
131
- smartTotal += costs.simple.groq;
132
- });
133
- allMedium.forEach(() => {
134
- gpt4Total += costs.medium.gpt4;
135
- smartTotal += costs.medium.gpt4mini;
136
- });
137
- allComplex.forEach(() => {
138
- gpt4Total += costs.complex.gpt4;
139
- smartTotal += costs.complex.gpt4;
140
- });
141
-
142
- console.log(`All GPT-4o: $${gpt4Total.toFixed(4)}`);
143
- console.log(`A3M Router: $${smartTotal.toFixed(4)}`);
144
- console.log(`Savings: ${((1 - smartTotal / gpt4Total) * 100).toFixed(1)}%`);
145
- }
@@ -1,61 +0,0 @@
1
- #!/bin/bash
2
- # A3M Router Benchmark Script
3
- # Compares: All-GPT-4 vs Smart Routing vs All-Cheap
4
-
5
- echo "=== A3M Router Cost Benchmark ==="
6
- echo ""
7
- echo "Running 100 simulated queries..."
8
- echo " 47 simple (Q&A, math, basic tasks)"
9
- echo " 33 medium (summarization, translation, code)"
10
- echo " 20 complex (reasoning, creative writing, analysis)"
11
- echo ""
12
-
13
- # Cost per 1K tokens (input)
14
- GPT4_COST=0.03 # $30/1M tokens
15
- GPT4_MINI_COST=0.00015 # $0.15/1M tokens
16
- GROQ_COST=0.00059 # $0.59/1M tokens
17
- CEREBRAS_COST=0.00060 # $0.60/1M tokens
18
- FREE_COST=0.00 # CommandCode/OpenCode
19
-
20
- # Average tokens per query type
21
- SIMPLE_TOKENS=150
22
- MEDIUM_TOKENS=500
23
- COMPLEX_TOKENS=1200
24
-
25
- # All GPT-4 baseline
26
- all_gpt4=$(echo "scale=4; (47 * $SIMPLE_TOKENS + 33 * $MEDIUM_TOKENS + 20 * $COMPLEX_TOKENS) * $GPT4_COST / 1000" | bc)
27
- echo "📊 All queries → GPT-4o:"
28
- echo " Cost: \$$all_gpt4"
29
- echo ""
30
-
31
- # Smart routing (A3M approach)
32
- # Simple → Groq/Cerebras, Medium → GPT-4o-mini, Complex → GPT-4o
33
- smart_simple=$(echo "scale=4; 47 * $SIMPLE_TOKENS * $GROQ_COST / 1000" | bc)
34
- smart_medium=$(echo "scale=4; 33 * $MEDIUM_TOKENS * $GPT4_MINI_COST / 1000" | bc)
35
- smart_complex=$(echo "scale=4; 20 * $COMPLEX_TOKENS * $GPT4_COST / 1000" | bc)
36
- smart_total=$(echo "scale=4; $smart_simple + $smart_medium + $smart_complex" | bc)
37
- savings=$(echo "scale=1; (1 - $smart_total / $all_gpt4) * 100" | bc)
38
- echo "📊 A3M Router (smart routing):"
39
- echo " Simple (47) → Groq: \$$smart_simple"
40
- echo " Medium (33) → GPT-4o-mini: \$$smart_medium"
41
- echo " Complex (20) → GPT-4o: \$$smart_complex"
42
- echo " Total: \$$smart_total"
43
- echo " Savings: ${savings}%"
44
- echo ""
45
-
46
- # All cheap (worst quality)
47
- all_cheap=$(echo "scale=4; (47 * $SIMPLE_TOKENS + 33 * $MEDIUM_TOKENS + 20 * $COMPLEX_TOKENS) * $GROQ_COST / 1000" | bc)
48
- echo "📊 All queries → Groq (cheapest):"
49
- echo " Cost: \$$all_cheap"
50
- echo " Quality: Lower (complex queries suffer)"
51
- echo ""
52
-
53
- # Monthly projection at scale
54
- echo "=== Monthly Projection ==="
55
- for queries in "10000" "100000" "1000000"; do
56
- scale=$(echo "scale=0; $queries / 100" | bc)
57
- gpt4_monthly=$(echo "scale=2; $all_gpt4 * $scale" | bc)
58
- smart_monthly=$(echo "scale=2; $smart_total * $scale" | bc)
59
- monthly_savings=$(echo "scale=2; $gpt4_monthly - $smart_monthly" | bc)
60
- printf " %s queries/month: GPT-4=\$%-8s A3M=\$%-8s Save=\$%s/mo\n" "$queries" "$gpt4_monthly" "$smart_monthly" "$monthly_savings"
61
- done
@@ -1,230 +0,0 @@
1
- #!/bin/bash
2
- # =============================================================================
3
- # A3M Router — Provider Comparison Script
4
- # =============================================================================
5
- # Compares responses from multiple LLM providers to the SAME prompt.
6
- # Works with any provider that has an OpenAI-compatible API.
7
- #
8
- # Usage:
9
- # ./scripts/compare-providers.sh "What is 2+2?"
10
- # ./scripts/compare-providers.sh # defaults
11
- #
12
- # Requires at least one of: GROQ_API_KEY, NV_API_KEY, CEREBRAS_API_KEY
13
- # =============================================================================
14
-
15
- set -e
16
-
17
- PROMPT="${1:-Say hello in exactly 3 words.}"
18
- TIMEOUT=15
19
-
20
- # Color codes
21
- RED='\033[0;31m'
22
- GREEN='\033[0;32m'
23
- YELLOW='\033[1;33m'
24
- BLUE='\033[0;34m'
25
- MAGENTA='\033[0;35m'
26
- CYAN='\033[0;36m'
27
- NC='\033[0m' # No Color
28
-
29
- echo ""
30
- echo -e "${BLUE}══════════════════════════════════════════════════════════════${NC}"
31
- echo -e "${BLUE} A3M Router — Provider Comparison${NC}"
32
- echo -e "${BLUE} Prompt: \"${PROMPT}\"${NC}"
33
- echo -e "${BLUE}══════════════════════════════════════════════════════════════${NC}"
34
- echo ""
35
-
36
- RESULTS_DIR=$(mktemp -d)
37
- trap 'rm -rf "$RESULTS_DIR"' EXIT
38
-
39
- COMPLETED=0
40
- FAILED=0
41
-
42
- # -------------------------------------------------------------------------
43
- # Helper: call a provider and save result
44
- # -------------------------------------------------------------------------
45
- call_provider() {
46
- local name=$1
47
- local url=$2
48
- local auth_header=$3
49
- local model=$4
50
- local data_file="$RESULTS_DIR/${name//\//_}.json"
51
-
52
- # Build the curl payload
53
- local payload
54
- payload=$(jq -n \
55
- --arg model "$model" \
56
- --arg content "$PROMPT" \
57
- '{
58
- model: $model,
59
- messages: [{role: "user", content: $content}],
60
- max_tokens: 100,
61
- temperature: 0.7
62
- }')
63
-
64
- # Execute with timeout, capture response and HTTP code
65
- local http_code
66
- http_code=$(curl -s -o "$data_file" -w "%{http_code}" \
67
- --max-time "$TIMEOUT" \
68
- -H "Authorization: Bearer $auth_header" \
69
- -H "Content-Type: application/json" \
70
- -d "$payload" \
71
- "$url" 2>/dev/null)
72
-
73
- if [ "$http_code" = "200" ]; then
74
- COMPLETED=$((COMPLETED + 1))
75
- echo "$data_file"
76
- else
77
- FAILED=$((FAILED + 1))
78
- # Store error for display
79
- echo "{\"error\": true, \"http_code\": $http_code, \"body\": $(cat "$data_file" 2>/dev/null || echo '"timeout or connection error"')}" > "$data_file"
80
- echo "$data_file"
81
- fi
82
- }
83
-
84
- # -------------------------------------------------------------------------
85
- # Print a single provider's result
86
- # -------------------------------------------------------------------------
87
- print_result() {
88
- local name=$1
89
- local color=$2
90
- local data_file=$3
91
-
92
- if jq -e '.error == true' "$data_file" >/dev/null 2>&1; then
93
- local code
94
- code=$(jq -r '.http_code' "$data_file")
95
- echo -e "${color} ❌ $name${NC} (HTTP $code)"
96
- jq -r '.body | if length > 200 then .[0:200] + "..." else . end' "$data_file" 2>/dev/null | sed 's/^/ /'
97
- return
98
- fi
99
-
100
- local content model usage_in usage_out usage_total
101
- content=$(jq -r '.choices[0].message.content // "N/A"' "$data_file" 2>/dev/null)
102
- model=$(jq -r '.model // "N/A"' "$data_file" 2>/dev/null)
103
- usage_in=$(jq -r '.usage.prompt_tokens // "?"' "$data_file" 2>/dev/null)
104
- usage_out=$(jq -r '.usage.completion_tokens // "?"' "$data_file" 2>/dev/null)
105
-
106
- echo -e "${color} ● $name${NC}"
107
- echo " Model: $model"
108
- echo " Tokens: ${usage_in} in / ${usage_out} out"
109
- echo " Response: $content"
110
- }
111
-
112
- # =============================================================================
113
- # Collect results (parallel execution)
114
- # =============================================================================
115
-
116
- echo -e "${YELLOW} Sending parallel requests...${NC}"
117
- echo ""
118
-
119
- # Groq
120
- if [ -n "$GROQ_API_KEY" ]; then
121
- groq_file=$(call_provider "Groq" \
122
- "https://api.groq.com/openai/v1/chat/completions" \
123
- "$GROQ_API_KEY" \
124
- "llama-3.3-70b-versatile")
125
- else
126
- echo -e "${RED} ❌ Groq — GROQ_API_KEY not set${NC}"
127
- FAILED=$((FAILED + 1))
128
- fi
129
-
130
- # NVIDIA NIM
131
- if [ -n "$NV_API_KEY" ]; then
132
- nvidia_file=$(call_provider "NVIDIA NIM" \
133
- "https://integrate.api.nvidia.com/v1/chat/completions" \
134
- "$NV_API_KEY" \
135
- "meta/llama-3.3-70b-instruct")
136
- else
137
- echo -e "${RED} ❌ NVIDIA NIM — NV_API_KEY not set${NC}"
138
- FAILED=$((FAILED + 1))
139
- fi
140
-
141
- # Cerebras
142
- if [ -n "$CEREBRAS_API_KEY" ]; then
143
- cerebras_file=$(call_provider "Cerebras" \
144
- "https://api.cerebras.ai/v1/chat/completions" \
145
- "$CEREBRAS_API_KEY" \
146
- "llama-3.3-70b")
147
- else
148
- echo -e "${RED} ❌ Cerebras — CEREBRAS_API_KEY not set${NC}"
149
- FAILED=$((FAILED + 1))
150
- fi
151
-
152
- # OpenAI
153
- if [ -n "$OPENAI_API_KEY" ]; then
154
- openai_file=$(call_provider "OpenAI" \
155
- "https://api.openai.com/v1/chat/completions" \
156
- "$OPENAI_API_KEY" \
157
- "gpt-4o-mini")
158
- else
159
- echo -e "${RED} ❌ OpenAI — OPENAI_API_KEY not set${NC}"
160
- FAILED=$((FAILED + 1))
161
- fi
162
-
163
- # Anthropic (different API shape, handle separately)
164
- if [ -n "$ANTHROPIC_API_KEY" ]; then
165
- anthro_file="$RESULTS_DIR/anthropic.json"
166
- local http_code
167
- http_code=$(curl -s -o "$anthro_file" -w "%{http_code}" \
168
- --max-time "$TIMEOUT" \
169
- -H "x-api-key: $ANTHROPIC_API_KEY" \
170
- -H "anthropic-version: 2023-06-01" \
171
- -H "Content-Type: application/json" \
172
- -d "$(jq -n \
173
- --arg content "$PROMPT" \
174
- '{
175
- model: "claude-3-5-haiku-latest",
176
- max_tokens: 100,
177
- messages: [{role: "user", content: $content}]
178
- }')" \
179
- "https://api.anthropic.com/v1/messages" 2>/dev/null)
180
-
181
- if [ "$http_code" = "200" ]; then
182
- COMPLETED=$((COMPLETED + 1))
183
- else
184
- echo "{\"error\": true, \"http_code\": $http_code}" > "$anthro_file"
185
- FAILED=$((FAILED + 1))
186
- fi
187
- else
188
- echo -e "${RED} ❌ Anthropic — ANTHROPIC_API_KEY not set${NC}"
189
- FAILED=$((FAILED + 1))
190
- fi
191
-
192
- echo ""
193
-
194
- # =============================================================================
195
- # Print results
196
- # =============================================================================
197
- echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
198
- echo -e "${GREEN} Results — ${COMPLETED} succeeded, ${FAILED} failed${NC}"
199
- echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
200
- echo ""
201
-
202
- # Print each result if the file exists
203
- [ -n "$groq_file" ] && [ -f "$groq_file" ] && print_result "Groq" "$MAGENTA" "$groq_file"
204
- [ -n "$nvidia_file" ] && [ -f "$nvidia_file" ] && print_result "NVIDIA NIM" "$CYAN" "$nvidia_file"
205
- [ -n "$cerebras_file" ] && [ -f "$cerebras_file" ] && print_result "Cerebras" "$GREEN" "$cerebras_file"
206
- [ -n "$openai_file" ] && [ -f "$openai_file" ] && print_result "OpenAI" "$YELLOW" "$openai_file"
207
- if [ -n "$anthro_file" ] && [ -f "$anthro_file" ]; then
208
- if jq -e '.error == true' "$anthro_file" >/dev/null 2>&1; then
209
- echo -e "${RED} ❌ Anthropic (HTTP $(jq -r '.http_code' "$anthro_file"))${NC}"
210
- else
211
- local content
212
- content=$(jq -r '.content[0].text // "N/A"' "$anthro_file" 2>/dev/null)
213
- local model
214
- model=$(jq -r '.model // "N/A"' "$anthro_file" 2>/dev/null)
215
- echo -e "${BLUE} ● Anthropic${NC}"
216
- echo " Model: $model"
217
- echo " Response: $content"
218
- fi
219
- fi
220
-
221
- echo ""
222
- echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
223
- echo -e " Run with A3M Proxy (after ${YELLOW}npx a3m-router serve${NC}):"
224
- echo ""
225
- echo " curl http://localhost:8787/v1/chat/completions \\"
226
- echo " -H \"Content-Type: application/json\" \\"
227
- echo " -d '{\"model\":\"auto\",\"messages\":[{\"role\":\"user\",\"content\":\"$PROMPT\"}]}'"
228
- echo ""
229
- echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
230
- echo ""
@@ -1,25 +0,0 @@
1
- #!/usr/bin/env node
2
- const https = require('https');
3
- async function getJSON(url) {
4
- return new Promise((resolve, reject) => {
5
- https.get(url, { headers: { 'User-Agent': 'a3m-router' } }, (res) => {
6
- let d = '';
7
- res.on('data', c => d += c);
8
- res.on('end', () => { try { resolve(JSON.parse(d)); } catch { reject(new Error('parse')); } });
9
- }).on('error', reject);
10
- });
11
- }
12
- async function main() {
13
- console.log('\n📅 A3M — Content Planner\n');
14
- let stars = 0, dl = 0, issues = 0;
15
- try { const r = await getJSON('https://api.github.com/repos/Das-rebel/a3m-router'); stars = r.stargazers_count || 0; issues = r.open_issues_count || 0; } catch {}
16
- try { const n = await getJSON('https://api.npmjs.org/downloads/point/last-week/adaptive-memory-multi-model-router'); dl = n.downloads || 0; } catch {}
17
- console.log(` 📦 Weekly downloads: ${dl.toLocaleString()}`);
18
- console.log(` ⭐ GitHub stars: ${stars}`);
19
- console.log(` 🐛 Open issues: ${issues}\n`);
20
- console.log(' 📌 RECOMMENDED: "How A3M Saves $X/Year on LLM Costs" — case study\n');
21
- console.log(' 📌 RECOMMENDED: A3M vs LiteLLM vs RouteLLM — benchmark comparison\n');
22
- console.log(' 📌 RECOMMENDED: Video demo of parallel execution (asciinema)\n');
23
- console.log(' 📌 RECOMMENDED: "How to Save 62% on LLM APIs" blog post\n');
24
- }
25
- main().catch(console.error);