mlx-bun 0.0.10 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +107 -32
  2. package/THIRD_PARTY_LICENSES.md +10 -0
  3. package/docs/archive/mlx-bun-lab-report.html +275 -0
  4. package/docs/archive/wwdc-mlx-bun.html +848 -0
  5. package/docs/design/audio-input-plan.md +356 -0
  6. package/docs/design/batching-perf-path.md +58 -35
  7. package/docs/design/batching-v2-plan.md +23 -9
  8. package/docs/design/bucketing-stage.md +2 -1
  9. package/docs/design/colibri-glm52-port.md +1479 -0
  10. package/docs/design/compat-cli-surface-design.md +4 -3
  11. package/docs/design/decode-speed-program.md +172 -0
  12. package/docs/design/docs-reorg-plan.md +1 -1
  13. package/docs/design/dspark-serving-program.md +371 -0
  14. package/docs/design/dspark-speculative-decoding.md +167 -5
  15. package/docs/design/faithful-l1-consolidation.md +310 -0
  16. package/docs/design/generic-model-support.md +8 -5
  17. package/docs/design/grammar-spec-batching-integration.md +301 -0
  18. package/docs/design/minicpm5-decode-megakernel.md +16 -11
  19. package/docs/design/mlx-lm-tool-parity-plan.md +11 -1
  20. package/docs/design/omlx-adoption-map.md +14 -6
  21. package/docs/design/orpo-training.md +34 -30
  22. package/docs/design/paged-kv-cache.md +214 -0
  23. package/docs/design/parallel-slots.md +82 -49
  24. package/docs/design/parity-tier-dag.md +43 -33
  25. package/docs/design/pre-colibri-stabilization.md +329 -0
  26. package/docs/design/repo-cleanup-plan.md +189 -0
  27. package/docs/design/runtime-isolation.md +19 -3
  28. package/docs/design/segmented-backward-training.md +3 -2
  29. package/docs/design/ssd-kv-cold-tier.md +130 -0
  30. package/docs/design/structured-output.md +484 -0
  31. package/docs/design/the-dreaming-master-plan.md +2 -3
  32. package/docs/design/turboquant-kv.md +235 -0
  33. package/docs/design/unified-engine-frontier-plan.md +774 -0
  34. package/docs/design/web-chat-beat-matrix.md +315 -0
  35. package/docs/design/web-chat-redesign.md +1112 -0
  36. package/docs/investigations/colibri-oracle-pin.md +1284 -0
  37. package/docs/investigations/dspark-handoff.md +182 -16
  38. package/docs/investigations/indexshare-performance-spike.md +357 -0
  39. package/docs/investigations/lab-build-journal.md +2 -2
  40. package/docs/investigations/openwiki-evaluation.md +249 -0
  41. package/docs/planning/IDEAS.md +14 -0
  42. package/docs/planning/PRODUCT_ROADMAP.md +2 -3
  43. package/docs/planning/chat-ui-vision-memory-first.md +199 -0
  44. package/docs/planning/chat-ui-vision.md +50 -0
  45. package/docs/planning/memory-docs-and-dag-plan.md +15 -6
  46. package/docs/planning/release-notes-v0.0.11.md +239 -0
  47. package/docs/planning/release-notes-v0.0.12.md +80 -0
  48. package/docs/reference/cli.md +63 -18
  49. package/docs/reference/distribution.md +37 -6
  50. package/docs/reference/embedding.md +40 -16
  51. package/docs/reference/features-matrix.md +99 -0
  52. package/docs/reference/library-api.md +69 -20
  53. package/docs/reference/memory.md +26 -6
  54. package/docs/reference/models.md +60 -3
  55. package/docs/reference/orpo-quickstart.md +3 -4
  56. package/docs/reference/server-api.md +717 -24
  57. package/docs/reference/server-config.md +265 -129
  58. package/docs/reference/training.md +24 -16
  59. package/package.json +5 -3
  60. package/src/anthropic.ts +67 -44
  61. package/src/audio/conformer.ts +647 -0
  62. package/src/audio/decode.ts +133 -0
  63. package/src/audio/features.ts +202 -0
  64. package/src/audio/transcode.ts +67 -0
  65. package/src/chat-template.ts +155 -10
  66. package/src/cli.ts +483 -113
  67. package/src/config.ts +116 -24
  68. package/src/diffusion/diffusion-generate.ts +14 -8
  69. package/src/download.ts +143 -12
  70. package/src/eval/README.md +7 -6
  71. package/src/eval/ifeval.ts +63 -120
  72. package/src/eval/kl.ts +59 -3
  73. package/src/eval/quality-db.ts +2 -2
  74. package/src/eval/runner.ts +17 -8
  75. package/src/eval/tasks/ifeval.ts +167 -31
  76. package/src/expert-io.ts +391 -0
  77. package/src/expert-offload.ts +15 -2
  78. package/src/expert-residency.ts +1134 -0
  79. package/src/expert-usage.ts +415 -0
  80. package/src/flags.ts +13 -0
  81. package/src/generate.ts +438 -35
  82. package/src/grammar.ts +585 -0
  83. package/src/hf-push.ts +1 -1
  84. package/src/hub-rest.ts +242 -0
  85. package/src/index.ts +14 -2
  86. package/src/jobs/runner.ts +37 -18
  87. package/src/kv-store.ts +586 -93
  88. package/src/lora.ts +85 -14
  89. package/src/md-modules.d.ts +42 -0
  90. package/src/media-fetch.ts +426 -0
  91. package/src/memory/rest.ts +245 -0
  92. package/src/memory/vault.ts +74 -0
  93. package/src/mlx/array.ts +149 -19
  94. package/src/mlx/autograd.ts +53 -16
  95. package/src/mlx/checkpoint.ts +46 -13
  96. package/src/mlx/custom-vjp.ts +83 -35
  97. package/src/mlx/ffi.ts +27 -3
  98. package/src/mlx/metal-kernel.ts +7 -4
  99. package/src/mlx/ops.ts +153 -23
  100. package/src/mlx/turboquant-ops.ts +831 -0
  101. package/src/mlx/turboquant-tables.ts +237 -0
  102. package/src/model/batched-mask.ts +62 -1
  103. package/src/model/batched-quant.ts +206 -0
  104. package/src/model/batched-rotating-quant.ts +346 -0
  105. package/src/model/batched-rotating.ts +46 -1
  106. package/src/model/compiled-decode.ts +4 -15
  107. package/src/model/factory.ts +103 -6
  108. package/src/model/gemma4-base.ts +395 -39
  109. package/src/model/gemma4.ts +122 -33
  110. package/src/model/generated/gemma4-12b.ts +9 -13
  111. package/src/model/generated/gemma4-26b.ts +10 -5
  112. package/src/model/generated/gemma4-e4b.ts +14 -9
  113. package/src/model/glm52-atlas.ts +511 -0
  114. package/src/model/glm52-cache.ts +706 -0
  115. package/src/model/glm52-config.ts +262 -0
  116. package/src/model/glm52-container.ts +305 -0
  117. package/src/model/glm52-coupling.ts +388 -0
  118. package/src/model/glm52-dsa.ts +602 -0
  119. package/src/model/glm52-expert-layout.ts +253 -0
  120. package/src/model/glm52-layout.ts +152 -0
  121. package/src/model/glm52-memory.ts +473 -0
  122. package/src/model/glm52-mla.ts +786 -0
  123. package/src/model/glm52-moe.ts +223 -0
  124. package/src/model/glm52-pilot.ts +537 -0
  125. package/src/model/glm52-reference.ts +445 -0
  126. package/src/model/glm52-residency.ts +557 -0
  127. package/src/model/glm52-resident-weights.ts +135 -0
  128. package/src/model/glm52-streamed-experts.ts +453 -0
  129. package/src/model/glm52-streamed-metal.ts +610 -0
  130. package/src/model/glm52-weights.ts +503 -0
  131. package/src/model/glm52.ts +1153 -0
  132. package/src/model/minicpm5.ts +58 -102
  133. package/src/model/paged-kv.ts +253 -0
  134. package/src/model/qwen3-delta.ts +76 -15
  135. package/src/model/qwen3-moe.ts +404 -0
  136. package/src/model/qwen3.ts +9 -8
  137. package/src/model/qwen3_5.ts +88 -23
  138. package/src/model/support.ts +19 -1
  139. package/src/model/universal/dense.ts +40 -5
  140. package/src/model/universal/rope.ts +36 -21
  141. package/src/native/expert_io.c +523 -0
  142. package/src/native-pack.ts +6 -4
  143. package/src/pi-web.ts +1299 -82
  144. package/src/preflight.ts +27 -8
  145. package/src/prompt-cache.ts +293 -33
  146. package/src/quantize/quantizer.ts +16 -1
  147. package/src/registry.ts +57 -2
  148. package/src/responses.ts +62 -49
  149. package/src/serve/batch-scheduler.ts +1050 -78
  150. package/src/serve/generation-gateway.ts +252 -24
  151. package/src/serve/isolate.ts +417 -0
  152. package/src/serve/lane-registry.ts +44 -0
  153. package/src/serve/session-search.ts +326 -0
  154. package/src/server.ts +1701 -175
  155. package/src/spec/assistant-source.ts +150 -0
  156. package/src/spec/deepspec-source.ts +143 -0
  157. package/src/spec/dflash-source.ts +124 -0
  158. package/src/spec/dspark/ab-stats.ts +136 -0
  159. package/src/spec/dspark/calibration.ts +112 -0
  160. package/src/spec/dspark/deepspec-module.ts +801 -0
  161. package/src/spec/dspark/generate-dflash.ts +72 -20
  162. package/src/spec/dspark/loader.ts +23 -0
  163. package/src/spec/dspark/module-dflash.ts +278 -36
  164. package/src/spec/dspark/quantize-drafter.ts +91 -0
  165. package/src/spec/generate.ts +59 -25
  166. package/src/spec/glm52-mtp-source.ts +392 -0
  167. package/src/spec/ngram-source.ts +132 -0
  168. package/src/spec/serve-loop.ts +617 -0
  169. package/src/spec/source.ts +118 -0
  170. package/src/spec/two-model.ts +171 -0
  171. package/src/ssd-cache.ts +111 -35
  172. package/src/tokenizer.ts +288 -2
  173. package/src/tool-approvals.ts +131 -0
  174. package/src/tool-call.ts +438 -22
  175. package/src/train/flash-cce.ts +2 -2
  176. package/src/train/job.ts +0 -12
  177. package/src/train/lora-params.ts +17 -6
  178. package/src/train/trainer.ts +60 -30
  179. package/src/vision/prompt.ts +217 -48
  180. package/src/web/app.html +1197 -1791
  181. package/src/web/app.js +5694 -0
  182. package/src/web/icon.svg +13 -0
  183. package/src/web/manifest.webmanifest +24 -0
  184. package/src/web/src/adapters-panel.ts +288 -0
  185. package/src/web/src/api.ts +57 -0
  186. package/src/web/src/assistant.ts +485 -0
  187. package/src/web/src/chat.ts +1050 -0
  188. package/src/web/src/composer.ts +959 -0
  189. package/src/web/src/dataset.ts +132 -0
  190. package/src/web/src/finetune.ts +267 -0
  191. package/src/web/src/hub.ts +428 -0
  192. package/src/web/src/main.ts +98 -0
  193. package/src/web/src/markdown.ts +0 -0
  194. package/src/web/src/memory-panel.ts +572 -0
  195. package/src/web/src/model-picker.ts +159 -0
  196. package/src/web/src/palette.ts +346 -0
  197. package/src/web/src/protocol.ts +62 -0
  198. package/src/web/src/quantize.ts +167 -0
  199. package/src/web/src/rag.ts +274 -0
  200. package/src/web/src/sessions.ts +469 -0
  201. package/src/web/src/shell.ts +804 -0
  202. package/src/web/src/status.ts +300 -0
  203. package/src/web/src/ui-catalog.ts +92 -0
  204. package/src/web/sw.js +71 -0
  205. package/src/web/vendor/README.md +61 -0
  206. package/src/web/vendor/hljs-LICENSE +29 -0
  207. package/src/web/vendor/hljs-theme.css +59 -0
  208. package/src/web/vendor/hljs.js +26 -0
  209. package/src/web-tools.ts +54 -2
  210. package/src/weights.ts +82 -19
  211. package/src/model/fused-decode-kernel.ts +0 -312
  212. package/src/model/fused-geglu-kernel.ts +0 -198
  213. package/src/model/fused-mlp-kernel.ts +0 -169
  214. package/src/model/fused-swiglu-kernel.ts +0 -240
  215. package/src/model/steel-linear-kernel.ts +0 -96
  216. package/src/status-page.html +0 -497
package/src/status-page.html DELETED
@@ -1,497 +0,0 @@
1
- <!doctype html>
2
- <html lang="en">
3
- <head>
4
- <meta charset="utf-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1">
6
- <title>mlx-bun · status</title>
7
- <style>
8
- /* ————————————————————————————————————————————————
9
- mlx-bun · status — Apple keynote grammar:
10
- black stage, SF Pro, gradient hero type, hairline cards,
11
- ambient color blooms, numbers that land.
12
- ———————————————————————————————————————————————— */
13
- :root{
14
- --bg:#000;
15
- --ink:#f5f5f7;
16
- --dim:#86868b;
17
- --dimmer:#515154;
18
- --hairline:rgba(255,255,255,.14);
19
- --card:rgba(255,255,255,.045);
20
- --card-hover:rgba(255,255,255,.08);
21
- --blue:#0a84ff; --cyan:#64d2ff; --indigo:#5e5ce6; --purple:#bf5af2;
22
- --pink:#ff375f; --orange:#ff9f0a; --yellow:#ffd60a; --green:#30d158;
23
- --red:#ff453a;
24
- --ease:cubic-bezier(.22,.61,.21,1);
25
- }
26
- *{box-sizing:border-box;margin:0;padding:0}
27
- body{
28
- background:var(--bg);color:var(--ink);
29
- font-family:"SF Pro Display","SF Pro Text",-apple-system,BlinkMacSystemFont,"Helvetica Neue",Helvetica,Arial,sans-serif;
30
- -webkit-font-smoothing:antialiased;
31
- }
32
- ::selection{background:rgba(10,132,255,.4)}
33
- .num{font-variant-numeric:tabular-nums}
34
- code{font-family:"SF Mono",ui-monospace,Menlo,monospace;font-size:.86em;
35
- background:rgba(255,255,255,.08);padding:2px 8px;border-radius:6px}
36
-
37
- /* ambient blooms */
38
- .bloom{position:fixed;inset:-25%;pointer-events:none;z-index:0;
39
- background:
40
- radial-gradient(55vw 42vh at 12% 8%, rgba(255,159,10,.16), transparent 68%),
41
- radial-gradient(60vw 48vh at 88% 30%, rgba(94,92,230,.18), transparent 68%),
42
- radial-gradient(50vw 40vh at 30% 95%, rgba(10,132,255,.12), transparent 68%);
43
- filter:blur(70px);
44
- }
45
- main{position:relative;z-index:1;max-width:1180px;margin:0 auto;padding:0 28px 120px}
46
-
47
- /* reveals */
48
- [data-reveal]{opacity:0;transform:translateY(26px);
49
- transition:opacity .9s var(--ease),transform .9s var(--ease);
50
- transition-delay:calc(var(--d,0)*1ms)}
51
- .lit [data-reveal]{opacity:1;transform:none}
52
- @media (prefers-reduced-motion:reduce){[data-reveal]{transition:none;opacity:1;transform:none}}
53
-
54
- /* ——— hero ——— */
55
- header{padding:96px 0 64px;text-align:center}
56
- .eyebrow{font-size:14px;font-weight:600;letter-spacing:.28em;text-transform:uppercase;
57
- color:var(--dim);margin-bottom:22px}
58
- h1{font-size:clamp(64px,10vw,150px);font-weight:700;letter-spacing:-.035em;line-height:1}
59
- .grad{background-clip:text;-webkit-background-clip:text;color:transparent}
60
- .g-hero{background-image:linear-gradient(96deg,var(--orange) 0%,var(--pink) 30%,var(--purple) 62%,var(--blue) 100%)}
61
- .g-blue{background-image:linear-gradient(94deg,var(--cyan),var(--blue) 55%,var(--indigo))}
62
- .g-green{background-image:linear-gradient(94deg,#a8ff9e,var(--green) 45%,var(--cyan))}
63
- .g-pink{background-image:linear-gradient(94deg,var(--orange),var(--pink) 55%,var(--purple))}
64
- header .sub{margin-top:24px;font-size:clamp(17px,2vw,24px);color:var(--dim);font-weight:500}
65
- header .sub b{color:var(--ink);font-weight:600}
66
- .pill{display:inline-flex;align-items:center;gap:9px;margin-top:34px;
67
- border:1px solid var(--hairline);border-radius:999px;padding:9px 20px;
68
- font-size:14px;font-weight:600;color:var(--dim);letter-spacing:.02em}
69
- .pill .dot{width:9px;height:9px;border-radius:50%;background:var(--red);
70
- box-shadow:0 0 12px var(--red);transition:all .4s}
71
- .pill.ok .dot{background:var(--green);box-shadow:0 0 12px var(--green)}
72
- .pill.ok{color:var(--ink)}
73
-
74
- /* ——— sections ——— */
75
- section{margin-top:92px}
76
- h2{font-size:clamp(32px,4.6vw,58px);font-weight:700;letter-spacing:-.028em;line-height:1.05;margin-bottom:10px}
77
- h2 .soft{color:var(--dimmer)}
78
- .lede{font-size:clamp(15px,1.4vw,19px);line-height:1.55;color:var(--dim);max-width:64ch;margin-bottom:34px}
79
- .lede strong{color:var(--ink);font-weight:600}
80
-
81
- /* cards */
82
- .grid{display:grid;gap:14px}
83
- .g4{grid-template-columns:repeat(auto-fit,minmax(240px,1fr))}
84
- .card{background:var(--card);border:1px solid var(--hairline);border-radius:20px;
85
- padding:24px 26px;transition:background .3s var(--ease)}
86
- .card:hover{background:var(--card-hover)}
87
- .card h3{font-size:12.5px;font-weight:600;letter-spacing:.14em;text-transform:uppercase;
88
- color:var(--dim);margin-bottom:14px}
89
- .stat{font-size:clamp(30px,3.4vw,44px);font-weight:700;letter-spacing:-.025em;line-height:1.05}
90
- .stat small{font-size:.42em;font-weight:600;color:var(--dim);letter-spacing:0;margin-left:6px}
91
- .card .cap{margin-top:8px;font-size:13.5px;color:var(--dim);line-height:1.5}
92
- .kv{display:flex;justify-content:space-between;gap:10px;padding:5px 0;font-size:14.5px;
93
- border-bottom:1px solid rgba(255,255,255,.06)}
94
- .kv:last-child{border-bottom:none}
95
- .kv b{font-weight:500;color:var(--dim)}
96
- .kv span{font-weight:600}
97
- .meter{height:5px;border-radius:3px;background:rgba(255,255,255,.09);margin-top:16px;overflow:hidden}
98
- .meter i{display:block;height:100%;border-radius:3px;width:0%;
99
- background:linear-gradient(90deg,var(--cyan),var(--blue));transition:width .8s var(--ease)}
100
- .meter.warm i{background:linear-gradient(90deg,var(--orange),var(--pink))}
101
-
102
- /* model banner */
103
- #model-card{grid-column:1 / -1;display:flex;flex-wrap:wrap;align-items:baseline;gap:14px 26px}
104
- #model-id{font-size:clamp(20px,2.6vw,32px);font-weight:700;letter-spacing:-.02em;word-break:break-all}
105
- #model-meta{font-size:14.5px;color:var(--dim);font-weight:500}
106
-
107
- /* sku table */
108
- table{width:100%;border-collapse:collapse;font-size:14.5px}
109
- th,td{text-align:right;padding:13px 16px;border-bottom:1px solid rgba(255,255,255,.07)}
110
- th:first-child,td:first-child{text-align:left}
111
- td:not(:first-child),th:not(:first-child){white-space:nowrap}
112
- th{font-size:11.5px;font-weight:600;letter-spacing:.13em;text-transform:uppercase;color:var(--dim)}
113
- tr:last-child td{border-bottom:none}
114
- tbody tr{transition:background .2s}
115
- tbody tr:hover{background:rgba(255,255,255,.04)}
116
- td.fit-yes{color:var(--green);font-weight:600}
117
- td.fit-no{color:var(--dimmer)}
118
- tr.you td{background:rgba(10,132,255,.10)}
119
- tr.you td:first-child{border-left:3px solid var(--blue);border-radius:2px}
120
- .tablecard{padding:8px 6px}
121
-
122
- /* about */
123
- .pillars{display:grid;gap:14px;grid-template-columns:repeat(auto-fit,minmax(250px,1fr))}
124
- .pillar h4{font-size:17px;font-weight:700;letter-spacing:-.01em;margin-bottom:8px}
125
- .pillar p{font-size:14px;line-height:1.6;color:var(--dim)}
126
- .endpoints{display:flex;flex-wrap:wrap;gap:10px;margin-top:26px}
127
- .ep{border:1px solid var(--hairline);border-radius:12px;padding:9px 14px;font-size:13px;
128
- color:var(--dim);transition:all .25s}
129
- .ep:hover{color:var(--ink);background:var(--card-hover)}
130
- .ep b{color:var(--ink);font-weight:600;font-family:"SF Mono",ui-monospace,Menlo,monospace;font-size:12.5px}
131
-
132
- footer{margin-top:96px;border-top:1px solid var(--hairline);padding-top:26px;
133
- display:flex;flex-wrap:wrap;justify-content:space-between;gap:12px;
134
- font-size:13px;color:var(--dimmer)}
135
- footer a{color:var(--dim);text-decoration:none}
136
- footer a:hover{color:var(--ink)}
137
- </style>
138
- </head>
139
- <body>
140
- <div class="bloom"></div>
141
- <main id="page">
142
-
143
- <header>
144
- <div class="eyebrow" data-reveal>local ai · apple silicon · one binary</div>
145
- <h1 class="grad g-hero" data-reveal style="--d:80">mlx-bun</h1>
146
- <div class="sub" data-reveal style="--d:160">The Python-free local AI engine. <b>Bun + MLX, nothing else.</b></div>
147
- <div class="pill" id="conn" data-reveal style="--d:240"><span class="dot"></span><span id="conn-text">connecting…</span></div>
148
- <div data-reveal style="--d:300;margin-top:18px">
149
- <a href="/chat" style="display:inline-block;padding:12px 28px;border-radius:999px;font-weight:600;font-size:15px;color:#fff;text-decoration:none;background:linear-gradient(120deg,var(--blue),var(--purple))">Open chat →</a>
150
- </div>
151
- </header>
152
-
153
- <section data-reveal style="--d:320">
154
- <h2>Now serving<span class="soft">.</span></h2>
155
- <div class="grid g4">
156
- <div class="card" id="model-card">
157
- <div>
158
- <h3>Model</h3>
159
- <div id="model-id">—</div>
160
- </div>
161
- <div id="model-meta">—</div>
162
- </div>
163
-
164
- <div class="card">
165
- <h3>Decode · predicted</h3>
166
- <div class="stat num grad g-green"><span id="tps">—</span><small>tok/s</small></div>
167
- <div class="cap" id="tps-cap">from weights ÷ bandwidth on this machine</div>
168
- </div>
169
-
170
- <div class="card">
171
- <h3>Max safe context</h3>
172
- <div class="stat num grad g-blue"><span id="safe-ctx">—</span><small>tokens</small></div>
173
- <div class="cap">solved from KV growth + prefill transient, never guessed</div>
174
- </div>
175
-
176
- <div class="card">
177
- <h3>Memory</h3>
178
- <div class="kv"><b>weights</b><span class="num" id="weights">—</span></div>
179
- <div class="kv"><b>usable</b><span class="num" id="usable">—</span></div>
180
- <div class="kv"><b>budget</b><span class="num" id="budget">—</span></div>
181
- <div class="meter warm"><i id="mem-bar"></i></div>
182
- </div>
183
-
184
- <div class="card">
185
- <h3>Prompt cache</h3>
186
- <div class="kv"><b>hit rate</b><span class="num" id="pc-hits">—</span></div>
187
- <div class="kv"><b>entries</b><span class="num" id="pc-entries">—</span></div>
188
- <div class="kv"><b>bytes</b><span class="num" id="pc-bytes">—</span></div>
189
- <div class="meter"><i id="pc-bar"></i></div>
190
- </div>
191
-
192
- <div class="card">
193
- <h3>Response store</h3>
194
- <div class="kv"><b>entries</b><span class="num" id="rs-entries">—</span></div>
195
- <div class="kv"><b>bytes</b><span class="num" id="rs-bytes">—</span></div>
196
- <div class="kv"><b>ttl</b><span class="num" id="rs-ttl">—</span></div>
197
- <div class="meter"><i id="rs-bar"></i></div>
198
- </div>
199
-
200
- <div class="card">
201
- <h3>KV quantization</h3>
202
- <div class="kv"><b>mode</b><span id="kv-mode">—</span></div>
203
- <div id="kv-layers"></div>
204
- </div>
205
-
206
- <div class="card" id="dl-card" style="grid-column: 1 / -1; display:none">
207
- <h3>Downloads</h3>
208
- <div id="dl-list"></div>
209
- </div>
210
- </div>
211
- </section>
212
-
213
- <section data-reveal style="--d:80">
214
- <h2>Library<span class="soft">.</span> <span class="grad g-green">Every model, assessed.</span></h2>
215
- <p class="lede">Everything in the local cache, each judged by the fit solver for
216
- <strong>this machine</strong> — supported, fits, usable context, predicted speed.</p>
217
- <div class="card tablecard">
218
- <table>
219
- <thead><tr><th>Model</th><th>Size</th><th>Status</th><th>Max context</th><th>Decode</th></tr></thead>
220
- <tbody id="lib-body"><tr><td colspan="5" style="text-align:center;color:var(--dimmer)">waiting for server…</td></tr></tbody>
221
- </table>
222
- </div>
223
- </section>
224
-
225
- <section data-reveal style="--d:80">
226
- <h2>Will it fit<span class="soft">?</span> <span class="grad g-blue">Solved, not guessed.</span></h2>
227
- <p class="lede">mlx-bun's fit model solves the memory equation — <strong>weights + KV growth + prefill
228
- transient</strong> against wired memory — and predicts decode speed from memory bandwidth.
229
- This is the same assessment that gates admission on every request.</p>
230
- <div class="grid g4" style="margin-bottom:14px">
231
- <div class="card">
232
- <h3>This machine</h3>
233
- <div class="kv"><b>RAM</b><span class="num" id="m-ram">—</span></div>
234
- <div class="kv"><b>bandwidth</b><span class="num" id="m-bw">—</span></div>
235
- <div class="kv"><b>verdict</b><span id="m-fits">—</span></div>
236
- </div>
237
- <div class="card">
238
- <h3>At max context · <span class="num" id="fit-ctx">—</span> tokens</h3>
239
- <div class="kv"><b>weights</b><span class="num" id="f-weights">—</span></div>
240
- <div class="kv"><b>kv cache</b><span class="num" id="f-kv">—</span></div>
241
- <div class="kv"><b>transient</b><span class="num" id="f-transient">—</span></div>
242
- <div class="kv"><b>total / usable</b><span class="num" id="f-total">—</span></div>
243
- <div class="meter"><i id="fit-bar"></i></div>
244
- </div>
245
- </div>
246
- <div class="card tablecard">
247
- <table>
248
- <thead><tr><th>Apple silicon · this model @ <span class="num" id="sku-ctx">32,768</span> ctx</th>
249
- <th>RAM</th><th>Fits</th><th>Max context</th><th>Decode</th></tr></thead>
250
- <tbody id="sku-body"><tr><td colspan="5" style="text-align:center;color:var(--dimmer)">waiting for server…</td></tr></tbody>
251
- </table>
252
- </div>
253
- </section>
254
-
255
- <section data-reveal style="--d:80">
256
- <h2>The whole engine<span class="soft">,</span> <span class="grad g-pink">one binary.</span></h2>
257
- <p class="lede">Born from one bad evening with a Python stack: download stalls, segfaults,
258
- missing deps, an OOM footgun. <strong>The layer with all the bugs is the layer that doesn't
259
- need Python</strong> — so mlx-bun replaces it. TypeScript on Bun, talking straight to MLX.</p>
260
- <div class="pillars">
261
- <div class="card pillar">
262
- <h4 class="grad g-blue">Bit-exact by oracle</h4>
263
- <p>Logit parity with mlx-lm is the correctness contract. Every model ships with
264
- tier-a token-identical parity tests against the Python reference.</p>
265
- </div>
266
- <div class="card pillar">
267
- <h4 class="grad g-green">OptiQ quants, first-class</h4>
268
- <p>Sensitivity-aware 4-bit quantization, mixed KV-cache quantization, hot-swap LoRA
269
- adapters on a quantized base — never reload the weights.</p>
270
- </div>
271
- <div class="card pillar">
272
- <h4 class="grad g-pink">Every API you already use</h4>
273
- <p>OpenAI chat + Responses, Anthropic messages, SSE streaming, tool calling,
274
- vision. Point your SDK at localhost and go.</p>
275
- </div>
276
- <div class="card pillar">
277
- <h4 class="grad g-hero">Agents, built in</h4>
278
- <p><code>mlx-bun pi</code> drops you into a pi coding-agent session on this server —
279
- local model, local tools, zero cloud.</p>
280
- </div>
281
- </div>
282
- <div class="grid" style="grid-template-columns:1fr;margin-top:26px">
283
- <div class="card tablecard">
284
- <table>
285
- <thead><tr><th>Command</th><th style="text-align:left">What it does</th></tr></thead>
286
- <tbody>
287
- <tr><td><code>mlx-bun pi</code></td><td style="text-align:left">Agent session on a local model — picks/downloads the right model, serves, drops into pi. Flags: <code>--query</code>, <code>--port</code>; everything else passes to pi (<code>-p</code>, <code>--continue</code>, <code>@file</code>…)</td></tr>
288
- <tr><td><code>mlx-bun serve &lt;query&gt;</code></td><td style="text-align:left">Persistent server + this page. <code>--port 8080</code>, <code>--memory-budget GB</code></td></tr>
289
- <tr><td><code>mlx-bun get &lt;org/repo&gt;</code></td><td style="text-align:left">Resumable, sha256-verified model download</td></tr>
290
- <tr><td><code>mlx-bun ls</code> / <code>fit</code></td><td style="text-align:left">List downloaded models / will-it-fit assessment (<code>--ctx</code>, <code>--skus</code>)</td></tr>
291
- <tr><td><code>mlx-bun harness pi</code></td><td style="text-align:left">Register this server as a provider in your own pi install (<code>--remove</code> undoes)</td></tr>
292
- <tr><td><code>mlx-bun help &lt;cmd&gt;</code></td><td style="text-align:left">Full options for any command (also <code>--help</code> anywhere)</td></tr>
293
- </tbody>
294
- </table>
295
- </div>
296
- </div>
297
- <div class="endpoints">
298
- <span class="ep"><b>POST /v1/chat/completions</b> · OpenAI</span>
299
- <span class="ep"><b>POST /v1/messages</b> · Anthropic</span>
300
- <span class="ep"><b>POST /v1/responses</b> · Responses API</span>
301
- <span class="ep"><b>GET /v1/models</b></span>
302
- <span class="ep"><b>GET/POST /v1/adapters</b> · LoRA</span>
303
- <span class="ep"><b>GET /stats</b></span>
304
- <span class="ep"><b>GET /fit</b></span>
305
- </div>
306
- </section>
307
-
308
- <footer>
309
- <span>mlx-bun · M-series Macs · MIT-licensed deps · no Python anywhere in the serving path</span>
310
- <span><a href="https://github.com/joshuarossi/mlx-bun">github.com/joshuarossi/mlx-bun</a> · server up <span id="uptime">—</span> · updated <span id="updated">never</span></span>
311
- </footer>
312
- </main>
313
-
314
- <script>
315
- const $ = (id) => document.getElementById(id);
316
- const gb = (b) => b == null ? "—" : (b / 2 ** 30).toFixed(2) + " GB";
317
- const mb = (b) => b == null ? "—" : (b / 2 ** 20).toFixed(1) + " MB";
318
- const num = (n) => n == null ? "—" : Math.round(n).toLocaleString();
319
- requestAnimationFrame(() => document.getElementById("page").classList.add("lit"));
320
- document.getElementById("page").classList.add("lit");
321
-
322
- let fitLoaded = false;
323
- async function tick() {
324
- try {
325
- const [stats, models] = await Promise.all([
326
- fetch("/stats").then((r) => r.json()),
327
- fetch("/v1/models").then((r) => r.json()),
328
- ]);
329
- const m = models.data?.[0];
330
- $("model-id").textContent = m?.id ?? "—";
331
- const a = stats.admission ?? {};
332
- $("model-meta").textContent =
333
- num(m?.context_window) + " token window · weights " + gb(a.weights_bytes);
334
- $("safe-ctx").textContent = num(a.max_safe_context);
335
-
336
- $("weights").textContent = gb(a.weights_bytes);
337
- $("usable").textContent = gb(a.usable_bytes);
338
- $("budget").textContent = a.memory_budget_bytes ? gb(a.memory_budget_bytes) : "off";
339
- $("mem-bar").style.width = a.usable_bytes ? Math.min(100, 100 * a.weights_bytes / a.usable_bytes) + "%" : "0%";
340
-
341
- const pc = stats.prompt_cache ?? {};
342
- const lookups = (pc.hits ?? 0) + (pc.misses ?? 0);
343
- $("pc-hits").textContent = lookups ? Math.round(100 * pc.hits / lookups) + "%" : "no lookups";
344
- $("pc-entries").textContent = num(pc.entries);
345
- $("pc-bytes").textContent = mb(pc.bytes) + " / " + mb(pc.max_bytes);
346
- $("pc-bar").style.width = pc.max_bytes ? Math.min(100, 100 * pc.bytes / pc.max_bytes) + "%" : "0%";
347
-
348
- const rs = stats.response_store ?? {};
349
- $("rs-entries").textContent = num(rs.entries);
350
- $("rs-bytes").textContent = mb(rs.bytes) + " / " + mb(rs.max_bytes);
351
- $("rs-ttl").textContent = rs.ttl_ms ? (rs.ttl_ms / 60000) + " min" : "—";
352
- $("rs-bar").style.width = rs.max_bytes ? Math.min(100, 100 * rs.bytes / rs.max_bytes) + "%" : "0%";
353
-
354
- const kv = stats.kv_quant ?? {};
355
- $("kv-mode").innerHTML = "<code>" + (kv.mode ?? "—") + "</code>";
356
- const att = kv.attention;
357
- $("kv-layers").innerHTML = Object.entries(kv.layers ?? {})
358
- .map(([k, v]) => '<div class="kv"><b>' + k + " layers</b><span class=\"num\">" + v + "</span></div>").join("") +
359
- (att ? '<div class="kv"><b>attention</b><span class="num">' + att.global + " global · " + att.sliding_window + " sliding</span></div>" : "");
360
-
361
- await loadDownloads();
362
- if (!fitLoaded) { await loadFit(); await loadLibrary(); }
363
- if (stats.server?.started_at) {
364
- const up = (Date.now() - stats.server.started_at) / 1000;
365
- $("uptime").textContent = up < 90 ? Math.round(up) + "s" :
366
- up < 5400 ? Math.round(up / 60) + "m" : (up / 3600).toFixed(1) + "h";
367
- }
368
- $("conn").className = "pill ok";
369
- $("conn-text").textContent = "live · localhost";
370
- $("updated").textContent = new Date().toLocaleTimeString();
371
- } catch {
372
- $("conn").className = "pill";
373
- $("conn-text").textContent = "server unreachable — retrying";
374
- }
375
- }
376
-
377
- const dlRates = {}; // repoId -> { bytes, t } for client-side throughput
378
- async function loadDownloads() {
379
- const { downloads } = await fetch("/downloads").then((r) => r.json());
380
- const card = $("dl-card");
381
- if (!downloads || downloads.length === 0) { card.style.display = "none"; return; }
382
- card.style.display = "";
383
- const order = downloads.slice().sort((a, b) =>
384
- (a.state === "active" ? 0 : 1) - (b.state === "active" ? 0 : 1) || b.startedAt - a.startedAt);
385
- $("dl-list").innerHTML = order.map((d) => {
386
- const pct = d.totalBytes ? Math.min(100, 100 * d.receivedBytes / d.totalBytes) : 0;
387
- let detail, barCls = "";
388
- if (d.state === "active") {
389
- // Prefer the server-measured rolling rate; fall back to poll deltas.
390
- let bps = d.bytesPerSec ?? 0;
391
- if (!bps) {
392
- const prev = dlRates[d.repoId];
393
- const now = performance.now();
394
- if (prev && now > prev.t) bps = (d.receivedBytes - prev.bytes) / ((now - prev.t) / 1000);
395
- dlRates[d.repoId] = { bytes: d.receivedBytes, t: now };
396
- }
397
- const speed = bps > 0
398
- ? " · " + mb(bps) + "/s · ~" + Math.max(1, Math.round((d.totalBytes - d.receivedBytes) / bps / 60)) + " min left"
399
- : "";
400
- detail = gb(d.receivedBytes) + " / " + gb(d.totalBytes) + " · " + pct.toFixed(0) + "%" + speed +
401
- (d.currentFile ? " · <code>" + d.currentFile + "</code>" : "");
402
- barCls = "warm";
403
- } else if (d.state === "done") {
404
- detail = gb(d.totalBytes) + ' · <span style="color:var(--green);font-weight:600">complete</span>';
405
- } else {
406
- detail = '<span style="color:var(--red);font-weight:600">error</span> · ' + (d.error ?? "");
407
- }
408
- return '<div style="padding:10px 0">' +
409
- '<div class="kv" style="border:none"><b>' + d.repoId + "</b>" +
410
- '<span class="num">' + d.filesDone + "/" + d.filesTotal + " files</span></div>" +
411
- '<div style="font-size:13.5px;color:var(--dim)" class="num">' + detail + "</div>" +
412
- '<div class="meter ' + barCls + '"><i style="width:' + pct + '%"></i></div></div>';
413
- }).join("");
414
- }
415
-
416
- async function loadLibrary() {
417
- const { models } = await fetch("/library").then((r) => r.json());
418
- if (!models) return;
419
- $("lib-body").innerHTML = models
420
- .slice()
421
- .sort((a, b) => (b.serving ? 1 : 0) - (a.serving ? 1 : 0) || b.size_bytes - a.size_bytes)
422
- .map((m) => {
423
- const a = m.assessment;
424
- let status;
425
- if (m.serving) status = '<span style="color:var(--blue);font-weight:700">SERVING</span>';
426
- else if (!m.supported) status = '<span style="color:var(--dimmer)">unsupported (' + m.model_type + ")</span>";
427
- else if (a && a.fits) status = '<span style="color:var(--green);font-weight:600">fits</span>';
428
- else status = '<span style="color:var(--red);font-weight:600">too big</span>';
429
- return "<tr" + (m.serving ? ' class="you"' : "") + "><td>" + m.repo_id +
430
- (m.vision ? ' · <span style="color:var(--dim)">vision</span>' : "") +
431
- '</td><td class="num">' + gb(m.size_bytes) + "</td><td>" + status + "</td>" +
432
- '<td class="num">' + (m.supported && a && a.fits ? num(a.max_safe_context) : "—") + "</td>" +
433
- '<td class="num">' + (m.supported && a && a.fits ? a.predicted_decode_tps.toFixed(0) + " tok/s" : "—") + "</td></tr>";
434
- }).join("");
435
- }
436
-
437
- async function loadFit() {
438
- const f = await fetch("/fit").then((r) => r.json());
439
- fitLoaded = true;
440
- const r = f.report ?? {};
441
- if (f.measured_decode_tps) {
442
- // A real benchmark of this model on this machine beats any model.
443
- $("tps").textContent = f.measured_decode_tps.toFixed(1);
444
- document.querySelector("#tps").closest(".card").querySelector("h3").textContent = "Decode · measured";
445
- $("tps-cap").textContent = "mlx-bun benchmark on this machine · predicted " +
446
- (f.typical_decode_tps ?? 0).toFixed(1) + " at " + num(f.typical_context_tokens) + " ctx";
447
- } else if (f.typical_decode_tps) {
448
- $("tps").textContent = f.typical_decode_tps.toFixed(1);
449
- $("tps-cap").textContent = "at " + num(f.typical_context_tokens) +
450
- " context · " + (r.predicted_decode_tps ?? 0).toFixed(1) + " tok/s at the " +
451
- num(f.context_tokens) + " max (full-KV reads)";
452
- } else {
453
- $("tps").textContent = (r.predicted_decode_tps ?? 0).toFixed(1);
454
- }
455
- $("m-ram").textContent = gb(f.machine?.ram_bytes);
456
- $("m-bw").textContent = (f.machine?.bandwidth_gbs ?? "—") + " GB/s";
457
- $("m-fits").innerHTML = r.fits
458
- ? '<span style="color:var(--green);font-weight:700">FITS</span>'
459
- : '<span style="color:var(--red);font-weight:700">DOES NOT FIT</span>';
460
- $("fit-ctx").textContent = num(f.context_tokens);
461
- $("f-weights").textContent = gb(r.weights_bytes);
462
- $("f-kv").textContent = gb(r.kv_bytes);
463
- $("f-transient").textContent = gb(r.transient_bytes);
464
- $("f-total").textContent = gb(r.total_bytes) + " / " + gb(r.usable_bytes);
465
- $("fit-bar").style.width = r.usable_bytes ? Math.min(100, 100 * r.total_bytes / r.usable_bytes) + "%" : "0%";
466
-
467
- $("sku-ctx").textContent = num(f.sku_matrix_ctx);
468
- const myRam = Math.round((f.machine?.ram_bytes ?? 0) / 2 ** 30);
469
- const myChip = f.machine?.chip ?? null;
470
- // Exactly one highlighted row: chip+RAM exact → same chip, nearest
471
- // RAM → RAM-only (unknown chip). Never highlight a different chip
472
- // when ours is in the table.
473
- const rows = f.sku_matrix ?? [];
474
- let youIdx = rows.findIndex((r) => myChip && r.sku === myChip && r.ram_gb === myRam);
475
- if (youIdx < 0 && myChip) {
476
- let best = Infinity;
477
- rows.forEach((r, i) => {
478
- if (r.sku === myChip && Math.abs(r.ram_gb - myRam) < best) { best = Math.abs(r.ram_gb - myRam); youIdx = i; }
479
- });
480
- }
481
- if (youIdx < 0) youIdx = rows.findIndex((r) => r.ram_gb === myRam);
482
- $("sku-body").innerHTML = rows.map((row, i) => {
483
- const you = i === youIdx;
484
- const dimCls = row.fits ? "num" : "num fit-no";
485
- return "<tr" + (you ? ' class="you"' : "") + '><td>' + row.sku + (you ? " · <b>this machine</b>" : "") +
486
- '</td><td class="num">' + row.ram_gb + ' GB</td>' +
487
- '<td class="' + (row.fits ? "fit-yes" : "fit-no") + '">' + (row.fits ? "fits" : "no") + "</td>" +
488
- '<td class="' + dimCls + '">' + (row.max_context > 0 ? num(row.max_context) : "0 — weights alone don’t fit") + "</td>" +
489
- '<td class="' + dimCls + '">' + (row.decode_tps > 0 ? "~" + row.decode_tps.toFixed(0) + " tok/s" : "—") + "</td></tr>";
490
- }).join("");
491
- }
492
-
493
- tick();
494
- setInterval(tick, 2000);
495
- </script>
496
- </body>
497
- </html>