mlx-bun 0.5.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +1449 -109
  2. package/THIRD_PARTY_NOTICES.md +882 -0
  3. package/bin/mlx-bun.convert.mjs +4 -0
  4. package/bin/mlx-bun.fuse.mjs +4 -0
  5. package/bin/mlx-bun.generate.mjs +4 -0
  6. package/bin/mlx-bun.lora.mjs +4 -0
  7. package/bin/mlx-bun.mjs +13 -21
  8. package/bin/mlx-bun.server.mjs +4 -0
  9. package/bin/mlx-bun.upload.mjs +4 -0
  10. package/dist/web/app.js +9584 -0
  11. package/package.json +50 -53
  12. package/src/cli/ane-install.ts +76 -0
  13. package/src/cli/args.ts +179 -0
  14. package/src/cli/engine-entry.ts +68 -0
  15. package/src/cli/inference.ts +214 -0
  16. package/src/cli/job-entry.ts +97 -0
  17. package/src/cli/library-host.ts +117 -0
  18. package/src/cli/main.ts +57 -0
  19. package/src/cli/memory-engine.ts +163 -0
  20. package/src/cli/memory.ts +36 -0
  21. package/src/cli/mlx-lm-aliases.ts +309 -0
  22. package/src/cli/model-choice.ts +33 -0
  23. package/src/cli/model-selection.ts +84 -0
  24. package/src/cli/module-host.ts +11 -0
  25. package/src/cli/module-verbs.ts +50 -0
  26. package/src/cli/numerical-policy.ts +12 -0
  27. package/src/cli/perplexity.ts +89 -0
  28. package/src/cli/serve-host.ts +423 -0
  29. package/src/cli/serve-isolated.ts +217 -0
  30. package/src/cli/serve-options.ts +106 -0
  31. package/src/cli/serve-state.ts +205 -0
  32. package/src/cli/serve.ts +500 -0
  33. package/src/cli/served-model-host.ts +102 -0
  34. package/src/cli/server-entry.ts +148 -0
  35. package/src/cli/serving-unit.ts +177 -0
  36. package/src/{tui.ts → cli/terminal.ts} +13 -8
  37. package/src/cli/worker-adapters.ts +34 -0
  38. package/src/cli/worker-entry.ts +272 -0
  39. package/src/cli/worker-unit.ts +179 -0
  40. package/src/engine/adapter-operation.ts +26 -0
  41. package/src/engine/cache-services.ts +114 -0
  42. package/src/engine/completion.ts +39 -0
  43. package/src/engine/generation-gateway.ts +358 -0
  44. package/src/engine/index.ts +59 -0
  45. package/src/engine/kv-budget.ts +120 -0
  46. package/src/engine/media-preparation.ts +232 -0
  47. package/src/engine/model-binding.ts +24 -0
  48. package/src/engine/model-host.ts +513 -0
  49. package/src/engine/model-serving.ts +89 -0
  50. package/src/{serve → engine}/preparation.ts +7 -5
  51. package/src/{serve → engine}/session-completion-engine.ts +6 -6
  52. package/src/engine/telemetry.ts +109 -0
  53. package/src/hub/downloads.ts +116 -0
  54. package/src/jobs/db.ts +16 -7
  55. package/src/jobs/events.ts +29 -0
  56. package/src/jobs/executable.ts +5 -0
  57. package/src/jobs/host.ts +51 -0
  58. package/src/jobs/in-process.ts +55 -0
  59. package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
  60. package/src/jobs/runner.ts +150 -231
  61. package/src/jobs/service.ts +75 -0
  62. package/src/jobs/sse.ts +59 -23
  63. package/src/jobs/worker-memory.ts +46 -0
  64. package/src/jobs/worker-process.ts +163 -0
  65. package/src/jobs/worker-supervisor.ts +258 -0
  66. package/src/modules.ts +58 -0
  67. package/src/publishing/catalog-hub.ts +18 -0
  68. package/src/publishing/credentials.ts +31 -0
  69. package/src/publishing/upload.ts +37 -0
  70. package/src/residency/leased-adapters.ts +21 -0
  71. package/src/residency/local-records.ts +16 -0
  72. package/src/residency/model-residency.ts +405 -0
  73. package/src/residency/record-index.ts +28 -0
  74. package/src/residency/resident-estimate.ts +50 -0
  75. package/src/{anthropic.ts → server/anthropic.ts} +3 -81
  76. package/src/server/cache-routes.ts +36 -0
  77. package/src/{serve → server}/chat-request.ts +10 -10
  78. package/src/{serve → server}/chat-stage.ts +14 -14
  79. package/src/{client.ts → server/client.ts} +28 -2
  80. package/src/{serve → server}/completion-executor.ts +33 -41
  81. package/src/server/decision-body.ts +55 -0
  82. package/src/server/decision-routes.ts +134 -0
  83. package/src/server/decision-wire.ts +228 -0
  84. package/src/{serve → server}/discovery-routes.ts +25 -82
  85. package/src/{serve → server}/http.ts +25 -13
  86. package/src/{serve → server}/inference-request.ts +5 -5
  87. package/src/server/job-routes.ts +20 -0
  88. package/src/server/media-prompt.ts +63 -0
  89. package/src/server/memory-completion-client.ts +46 -0
  90. package/src/server/model-routes.ts +192 -0
  91. package/src/{serve → server}/openai-wire.ts +3 -2
  92. package/src/{serve → server}/pipeline.ts +1 -1
  93. package/src/server/prompt-builder.ts +13 -0
  94. package/src/server/prompt-contracts.ts +17 -0
  95. package/src/server/proxy-routes.ts +187 -0
  96. package/src/server/publishing-routes.ts +41 -0
  97. package/src/{serve → server}/request-plan.ts +6 -8
  98. package/src/{serve → server}/request-prep.ts +47 -50
  99. package/src/{serve → server}/responses-client.ts +30 -13
  100. package/src/{responses.ts → server/responses.ts} +12 -93
  101. package/src/server/routes.ts +205 -0
  102. package/src/server/start.ts +72 -0
  103. package/src/server/status-routes.ts +108 -0
  104. package/src/{serve → server}/text-completion-stage.ts +25 -4
  105. package/src/server/text-prompt.ts +12 -0
  106. package/src/{serve → server}/token-streams.ts +36 -38
  107. package/src/server/worker-routes.ts +311 -0
  108. package/src/storage/paths.ts +61 -0
  109. package/src/web/assets.ts +37 -0
  110. package/src/web/{src → browser}/api.ts +5 -23
  111. package/src/web/browser/chat-host.ts +17 -0
  112. package/src/web/browser/installed-panels.ts +6 -0
  113. package/src/web/browser/main.ts +55 -0
  114. package/src/web/browser/markdown.ts +279 -0
  115. package/src/web/browser/memory-panel.ts +9 -0
  116. package/src/web/browser/palette.ts +64 -0
  117. package/src/web/browser/protocol.ts +13 -0
  118. package/src/web/browser/shell.ts +467 -0
  119. package/src/web/{src → browser}/status.ts +32 -36
  120. package/src/web/build.ts +51 -0
  121. package/src/web/public/app.html +1054 -0
  122. package/src/web/{sw.js → public/sw.js} +4 -3
  123. package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
  124. package/src/web/{vendor → public/vendor}/hljs.js +2 -2
  125. package/THIRD_PARTY_LICENSES.md +0 -51
  126. package/docs/README.md +0 -107
  127. package/docs/archive/hlg-sampling.md +0 -350
  128. package/docs/archive/investigations/audio-input-plan.md +0 -356
  129. package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +0 -3617
  130. package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
  131. package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
  132. package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
  133. package/docs/archive/investigations/curve-bisector-routing.md +0 -123
  134. package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
  135. package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
  136. package/docs/archive/investigations/dspark-handoff.md +0 -224
  137. package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
  138. package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
  139. package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
  140. package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
  141. package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
  142. package/docs/archive/investigations/lab-build-journal.md +0 -157
  143. package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
  144. package/docs/archive/investigations/openjarvis-fit.md +0 -198
  145. package/docs/archive/investigations/openwiki-evaluation.md +0 -249
  146. package/docs/archive/investigations/optimization_plan.md +0 -346
  147. package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
  148. package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
  149. package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
  150. package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
  151. package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
  152. package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
  153. package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
  154. package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
  155. package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
  156. package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
  157. package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
  158. package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
  159. package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
  160. package/docs/archive/planning/chat-ui-vision.md +0 -50
  161. package/docs/archive/planning/chunk-task-roadmap.md +0 -77
  162. package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
  163. package/docs/archive/planning/journal.md +0 -316
  164. package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
  165. package/docs/archive/planning/pi-integration-review.md +0 -178
  166. package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
  167. package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
  168. package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
  169. package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
  170. package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
  171. package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
  172. package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
  173. package/docs/archive/planning/release-notes-v0.4.0.md +0 -45
  174. package/docs/archive/planning/release-notes-v0.5.0.md +0 -37
  175. package/docs/archive/planning/web-ui-pass-plan.md +0 -213
  176. package/docs/archive/planning/website-readme-pass-plan.md +0 -224
  177. package/docs/design/batching.md +0 -2054
  178. package/docs/design/decode-speed-program.md +0 -2404
  179. package/docs/design/dreaming-nightly-pipeline.md +0 -684
  180. package/docs/design/generic-model-support.md +0 -821
  181. package/docs/design/kv-cache.md +0 -993
  182. package/docs/design/orpo-training.md +0 -680
  183. package/docs/design/speculative-decoding.md +0 -2129
  184. package/docs/design/turboquant.md +0 -2225
  185. package/docs/design/unified-engine-frontier-plan.md +0 -1987
  186. package/docs/design/web-chat-redesign.md +0 -695
  187. package/docs/planning/PRODUCT_ROADMAP.md +0 -593
  188. package/docs/planning/ResearchTopics.md +0 -119
  189. package/docs/planning/public-readiness-handoff.md +0 -244
  190. package/docs/planning/public-readiness.md +0 -149
  191. package/docs/planning/video-series.md +0 -421
  192. package/docs/reference/benchmarks.md +0 -3587
  193. package/docs/reference/cli.md +0 -646
  194. package/docs/reference/distribution.md +0 -380
  195. package/docs/reference/environment.md +0 -494
  196. package/docs/reference/glossary.md +0 -39
  197. package/docs/reference/library-api.md +0 -688
  198. package/docs/reference/memory.md +0 -260
  199. package/docs/reference/models.md +0 -346
  200. package/docs/reference/server-api.md +0 -1945
  201. package/docs/reference/server-config.md +0 -973
  202. package/docs/reference/training.md +0 -462
  203. package/docs/reference/troubleshooting.md +0 -158
  204. package/src/admission-plan.ts +0 -18
  205. package/src/audio/audiotoolbox.ts +0 -112
  206. package/src/audio/conformer.ts +0 -655
  207. package/src/audio/decode.ts +0 -133
  208. package/src/audio/features.ts +0 -202
  209. package/src/audio/mic-capture.ts +0 -98
  210. package/src/audio/silero-vad.ts +0 -379
  211. package/src/audio/transcode.ts +0 -67
  212. package/src/audio/whisper-decode.ts +0 -774
  213. package/src/audio/whisper-fast.ts +0 -732
  214. package/src/audio/whisper-format.ts +0 -73
  215. package/src/audio/whisper-mel.ts +0 -185
  216. package/src/audio/whisper-timing.ts +0 -274
  217. package/src/audio/whisper-tokenizer.ts +0 -239
  218. package/src/audio/whisper-transcribe.ts +0 -410
  219. package/src/backends/mlx/assistant-target.ts +0 -46
  220. package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
  221. package/src/backends/mlx/autoregressive.ts +0 -129
  222. package/src/backends/mlx/batch-group.ts +0 -1874
  223. package/src/backends/mlx/cache-attachment.ts +0 -67
  224. package/src/backends/mlx/cache-layout.ts +0 -31
  225. package/src/backends/mlx/checkpoint-state.ts +0 -53
  226. package/src/backends/mlx/continuation-persistence.ts +0 -107
  227. package/src/backends/mlx/continuation-request.ts +0 -43
  228. package/src/backends/mlx/continuation.ts +0 -77
  229. package/src/backends/mlx/diffusion.ts +0 -39
  230. package/src/backends/mlx/draft-target.ts +0 -32
  231. package/src/backends/mlx/encoder-cache.ts +0 -25
  232. package/src/backends/mlx/evaluation.ts +0 -105
  233. package/src/backends/mlx/fill-append.ts +0 -58
  234. package/src/backends/mlx/fill-group.ts +0 -497
  235. package/src/backends/mlx/full-kv-row-append.ts +0 -32
  236. package/src/backends/mlx/full-kv-row-donor.ts +0 -54
  237. package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
  238. package/src/backends/mlx/gateway-binding.ts +0 -174
  239. package/src/backends/mlx/graph.ts +0 -38
  240. package/src/backends/mlx/kv-codec.ts +0 -9
  241. package/src/backends/mlx/kv-maintenance.ts +0 -161
  242. package/src/backends/mlx/kv-tensor-rows.ts +0 -212
  243. package/src/backends/mlx/memory-client.ts +0 -189
  244. package/src/backends/mlx/methods.ts +0 -90
  245. package/src/backends/mlx/mixed-iteration.ts +0 -62
  246. package/src/backends/mlx/model-prompt.ts +0 -178
  247. package/src/backends/mlx/model-serving.ts +0 -80
  248. package/src/backends/mlx/paged-cache-codec.ts +0 -27
  249. package/src/backends/mlx/prefill-cohort.ts +0 -100
  250. package/src/backends/mlx/prefill-policy.ts +0 -26
  251. package/src/backends/mlx/prefill-rows.ts +0 -244
  252. package/src/backends/mlx/prefill.ts +0 -33
  253. package/src/backends/mlx/projected-context-rows.ts +0 -72
  254. package/src/backends/mlx/prompt-input.ts +0 -29
  255. package/src/backends/mlx/qwen-prompt-input.ts +0 -40
  256. package/src/backends/mlx/request-policy.ts +0 -20
  257. package/src/backends/mlx/request-state-policy.ts +0 -54
  258. package/src/backends/mlx/rollback.ts +0 -92
  259. package/src/backends/mlx/row-sampling.ts +0 -49
  260. package/src/backends/mlx/serial-executor.ts +0 -251
  261. package/src/backends/mlx/speculative-group.ts +0 -438
  262. package/src/backends/mlx/speculative-round.ts +0 -161
  263. package/src/backends/mlx/speculative.ts +0 -131
  264. package/src/backends/mlx/state-rows.ts +0 -104
  265. package/src/backends/mlx/state-views.ts +0 -46
  266. package/src/backends/mlx/target-layout-capability.ts +0 -35
  267. package/src/backends/mlx/text-engine.ts +0 -73
  268. package/src/backends/mlx/transitioning-kv-rows.ts +0 -173
  269. package/src/chat-template.ts +0 -382
  270. package/src/cli.ts +0 -3490
  271. package/src/config.ts +0 -399
  272. package/src/contracts/completion.ts +0 -16
  273. package/src/contracts/execution.ts +0 -70
  274. package/src/contracts/generation.ts +0 -103
  275. package/src/contracts/host.ts +0 -7
  276. package/src/contracts/object-cache.ts +0 -8
  277. package/src/contracts/pi-provider.ts +0 -30
  278. package/src/contracts/pi.ts +0 -261
  279. package/src/contracts/prefix-cache.ts +0 -33
  280. package/src/contracts/resources.ts +0 -25
  281. package/src/contracts/sampling.ts +0 -23
  282. package/src/contracts/scheduling.ts +0 -54
  283. package/src/contracts/tsconfig.json +0 -14
  284. package/src/dataset/generators.ts +0 -1087
  285. package/src/dataset/index.ts +0 -12
  286. package/src/dataset/job.ts +0 -81
  287. package/src/dataset/llm.ts +0 -71
  288. package/src/dataset/registry.ts +0 -528
  289. package/src/diffusion/diffusion-generate.ts +0 -507
  290. package/src/download.ts +0 -505
  291. package/src/embed.ts +0 -77
  292. package/src/engine/admission.ts +0 -59
  293. package/src/engine/cancellation.ts +0 -38
  294. package/src/engine/engine.ts +0 -56
  295. package/src/engine/execution-coordinator.ts +0 -70
  296. package/src/engine/execution-plan.ts +0 -64
  297. package/src/engine/execution-tasks.ts +0 -44
  298. package/src/engine/namespaced-cache.ts +0 -13
  299. package/src/engine/resources.ts +0 -53
  300. package/src/engine/scheduler.ts +0 -92
  301. package/src/engine/session.ts +0 -247
  302. package/src/eval/README.md +0 -130
  303. package/src/eval/ifeval.ts +0 -81
  304. package/src/eval/kl-prompts.ts +0 -29
  305. package/src/eval/kl.ts +0 -590
  306. package/src/eval/perplexity.ts +0 -164
  307. package/src/eval/quality-db.ts +0 -92
  308. package/src/eval/reference-logits.ts +0 -207
  309. package/src/eval/runner.ts +0 -124
  310. package/src/eval/score.ts +0 -49
  311. package/src/eval/tasks/bfcl.ts +0 -535
  312. package/src/eval/tasks/gsm8k.ts +0 -98
  313. package/src/eval/tasks/hashhop.ts +0 -139
  314. package/src/eval/tasks/humaneval.ts +0 -275
  315. package/src/eval/tasks/ifeval.ts +0 -525
  316. package/src/eval/tasks/mmlu.ts +0 -157
  317. package/src/eval/ultrafeedback.ts +0 -60
  318. package/src/evaldb.ts +0 -199
  319. package/src/expert-io.ts +0 -393
  320. package/src/expert-offload-build.ts +0 -77
  321. package/src/expert-offload.ts +0 -76
  322. package/src/expert-residency.ts +0 -1134
  323. package/src/expert-usage.ts +0 -415
  324. package/src/fill/echo-index.ts +0 -188
  325. package/src/fill/fill-session.ts +0 -463
  326. package/src/fill/proposal.ts +0 -68
  327. package/src/fill/schema-rows.ts +0 -506
  328. package/src/fill/tool-boundary.ts +0 -103
  329. package/src/fit.ts +0 -249
  330. package/src/generate.ts +0 -1352
  331. package/src/grammar.ts +0 -604
  332. package/src/harness-pi.ts +0 -184
  333. package/src/hf-push.ts +0 -628
  334. package/src/hub-rest.ts +0 -242
  335. package/src/index.ts +0 -63
  336. package/src/inference/denoising.ts +0 -17
  337. package/src/inference/draft-acceptance.ts +0 -41
  338. package/src/inference/draft-output.ts +0 -49
  339. package/src/inference/graph.ts +0 -33
  340. package/src/inference/prefill.ts +0 -45
  341. package/src/inference/rollback.ts +0 -7
  342. package/src/jobs/index.ts +0 -31
  343. package/src/jobs/job-entry.ts +0 -107
  344. package/src/jobs/task-client.ts +0 -19
  345. package/src/jobs/types.ts +0 -2
  346. package/src/kv-scheme.ts +0 -279
  347. package/src/kv-store.ts +0 -1135
  348. package/src/lab/README.md +0 -22
  349. package/src/lab/curve/curve-designer.html +0 -303
  350. package/src/lab/curve/curve-sampler.ts +0 -122
  351. package/src/lab/expert-trace/expert-trace.ts +0 -69
  352. package/src/lab/paged-kv/paged-attention.ts +0 -117
  353. package/src/lab/paged-kv/paged-kv-rows.ts +0 -111
  354. package/src/lab/paged-kv/paged-kv.ts +0 -336
  355. package/src/library.ts +0 -40
  356. package/src/lora.ts +0 -408
  357. package/src/md-modules.d.ts +0 -54
  358. package/src/media-fetch.ts +0 -442
  359. package/src/memory/article.ts +0 -558
  360. package/src/memory/chunk-validation.ts +0 -150
  361. package/src/memory/chunk.ts +0 -318
  362. package/src/memory/cluster.ts +0 -351
  363. package/src/memory/coverage.ts +0 -134
  364. package/src/memory/crosslink.ts +0 -0
  365. package/src/memory/db.ts +0 -253
  366. package/src/memory/entity.ts +0 -203
  367. package/src/memory/gate.ts +0 -187
  368. package/src/memory/history.ts +0 -152
  369. package/src/memory/ingest.ts +0 -335
  370. package/src/memory/model.ts +0 -183
  371. package/src/memory/normalize.ts +0 -671
  372. package/src/memory/parse.ts +0 -22
  373. package/src/memory/pipeline.ts +0 -275
  374. package/src/memory/prompts.ts +0 -65
  375. package/src/memory/query.ts +0 -334
  376. package/src/memory/reconcile.ts +0 -734
  377. package/src/memory/redirect.ts +0 -108
  378. package/src/memory/reindex.ts +0 -325
  379. package/src/memory/resolve.ts +0 -307
  380. package/src/memory/rest.ts +0 -245
  381. package/src/memory/route.ts +0 -272
  382. package/src/memory/schedule.ts +0 -165
  383. package/src/memory/stages.ts +0 -777
  384. package/src/memory/synthesize.ts +0 -1638
  385. package/src/memory/tools.ts +0 -749
  386. package/src/memory/vault.ts +0 -1194
  387. package/src/memory/wikify.ts +0 -1064
  388. package/src/mlx/array.ts +0 -425
  389. package/src/mlx/autograd.ts +0 -304
  390. package/src/mlx/checkpoint.ts +0 -149
  391. package/src/mlx/compile.ts +0 -168
  392. package/src/mlx/custom-vjp.ts +0 -188
  393. package/src/mlx/ffi.ts +0 -531
  394. package/src/mlx/materialize.ts +0 -23
  395. package/src/mlx/metal-kernel.ts +0 -159
  396. package/src/mlx/normalized-argmax.ts +0 -116
  397. package/src/mlx/ops.ts +0 -1108
  398. package/src/mlx/quantized-rows.ts +0 -36
  399. package/src/mlx/token-bitmask.ts +0 -52
  400. package/src/mlx/turboquant-kv-decode.ts +0 -82
  401. package/src/mlx/turboquant-kv-inverse.ts +0 -122
  402. package/src/mlx/turboquant-ops.ts +0 -861
  403. package/src/mlx/turboquant-tables.ts +0 -237
  404. package/src/mmap.ts +0 -90
  405. package/src/model/artifact-identity.ts +0 -73
  406. package/src/model/batched-kv.ts +0 -235
  407. package/src/model/batched-mask.ts +0 -279
  408. package/src/model/batched-quant.ts +0 -212
  409. package/src/model/batched-quantized-kv.ts +0 -144
  410. package/src/model/batched-rotating-quant.ts +0 -353
  411. package/src/model/batched-rotating-state.ts +0 -149
  412. package/src/model/batched-rotating.ts +0 -338
  413. package/src/model/batched-row-storage.ts +0 -179
  414. package/src/model/batched-ssm.ts +0 -52
  415. package/src/model/batched-turboquant-kv.ts +0 -94
  416. package/src/model/compiled-decode.ts +0 -712
  417. package/src/model/decoded-kv-donor.ts +0 -12
  418. package/src/model/delayed-quantized-kv.ts +0 -72
  419. package/src/model/delayed-rotating-quantized-kv.ts +0 -93
  420. package/src/model/delayed-turboquant-kv.ts +0 -57
  421. package/src/model/diffusion-gemma.ts +0 -1026
  422. package/src/model/factory.ts +0 -215
  423. package/src/model/fingerprint.ts +0 -45
  424. package/src/model/flash-attention.ts +0 -694
  425. package/src/model/full-prefill-padding.ts +0 -56
  426. package/src/model/full-prefill-row.ts +0 -105
  427. package/src/model/gemma4-base.ts +0 -2370
  428. package/src/model/gemma4.ts +0 -1395
  429. package/src/model/generated/gemma4-12b.ts +0 -429
  430. package/src/model/generated/gemma4-26b.ts +0 -448
  431. package/src/model/generated/gemma4-e4b.ts +0 -717
  432. package/src/model/generated/index.ts +0 -18
  433. package/src/model/glm52-atlas.ts +0 -511
  434. package/src/model/glm52-cache.ts +0 -792
  435. package/src/model/glm52-config.ts +0 -262
  436. package/src/model/glm52-container.ts +0 -305
  437. package/src/model/glm52-coupling.ts +0 -388
  438. package/src/model/glm52-dsa.ts +0 -606
  439. package/src/model/glm52-expert-layout.ts +0 -253
  440. package/src/model/glm52-layout.ts +0 -152
  441. package/src/model/glm52-memory.ts +0 -500
  442. package/src/model/glm52-mla.ts +0 -792
  443. package/src/model/glm52-moe.ts +0 -223
  444. package/src/model/glm52-pilot.ts +0 -537
  445. package/src/model/glm52-reference.ts +0 -445
  446. package/src/model/glm52-residency.ts +0 -557
  447. package/src/model/glm52-resident-weights.ts +0 -135
  448. package/src/model/glm52-streamed-experts.ts +0 -453
  449. package/src/model/glm52-streamed-metal.ts +0 -610
  450. package/src/model/glm52-weights.ts +0 -503
  451. package/src/model/glm52.ts +0 -1132
  452. package/src/model/implementation.ts +0 -69
  453. package/src/model/kv-attention-view.ts +0 -70
  454. package/src/model/minicpm5.ts +0 -376
  455. package/src/model/profile.ts +0 -468
  456. package/src/model/quantized-append-attention.ts +0 -31
  457. package/src/model/qwen-append.ts +0 -12
  458. package/src/model/qwen-conv.ts +0 -52
  459. package/src/model/qwen3-delta-state.ts +0 -74
  460. package/src/model/qwen3-delta.ts +0 -451
  461. package/src/model/qwen3-moe.ts +0 -404
  462. package/src/model/qwen3-mrope.ts +0 -250
  463. package/src/model/qwen3.ts +0 -277
  464. package/src/model/qwen3_5-checkpoint.ts +0 -164
  465. package/src/model/qwen3_5.ts +0 -881
  466. package/src/model/rotating-kv-donor.ts +0 -35
  467. package/src/model/rotating-kv-layout.ts +0 -262
  468. package/src/model/rotating-row-transaction.ts +0 -35
  469. package/src/model/speculative-rotating-kv.ts +0 -101
  470. package/src/model/ssm-prefill-padding.ts +0 -39
  471. package/src/model/support.ts +0 -110
  472. package/src/model/token-groups.ts +0 -66
  473. package/src/model/trellis-balanced-scatter.ts +0 -51
  474. package/src/model/trellis-linear.ts +0 -672
  475. package/src/model/trellis-shared-m.ts +0 -151
  476. package/src/model/trellis-shared-scatter.ts +0 -144
  477. package/src/model/trellis-splitk-prefill.ts +0 -158
  478. package/src/model/trellis-tiled-prefill.ts +0 -136
  479. package/src/model/trellis-vector-expand.ts +0 -62
  480. package/src/model/trellis-wide-prefill.ts +0 -94
  481. package/src/model/turboquant-codec.ts +0 -89
  482. package/src/model/universal/archs.ts +0 -332
  483. package/src/model/universal/dense.ts +0 -612
  484. package/src/model/universal/modules.ts +0 -170
  485. package/src/model/universal/rope.ts +0 -381
  486. package/src/model/whisper.ts +0 -435
  487. package/src/native/expert_io.c +0 -523
  488. package/src/native/frame_extract.swift +0 -140
  489. package/src/native/mic_capture.swift +0 -92
  490. package/src/native-pack.ts +0 -163
  491. package/src/pi-provider.ts +0 -53
  492. package/src/pi-session.ts +0 -81
  493. package/src/pi-terminal.ts +0 -274
  494. package/src/pi-web.ts +0 -1940
  495. package/src/preflight.ts +0 -158
  496. package/src/prompt-cache.ts +0 -682
  497. package/src/quantize/allocator.ts +0 -627
  498. package/src/quantize/atomic-output.ts +0 -38
  499. package/src/quantize/calibration.ts +0 -181
  500. package/src/quantize/config-writer.ts +0 -145
  501. package/src/quantize/index.ts +0 -84
  502. package/src/quantize/job.ts +0 -162
  503. package/src/quantize/quantizer.ts +0 -666
  504. package/src/quantize/rotate.ts +0 -603
  505. package/src/quantize/safetensors-writer.ts +0 -249
  506. package/src/quantize/sensitivity.ts +0 -216
  507. package/src/quantize/trellis.ts +0 -586
  508. package/src/quantize/weight-transform.ts +0 -292
  509. package/src/registry.ts +0 -602
  510. package/src/runtime-config.ts +0 -77
  511. package/src/safetensors.ts +0 -165
  512. package/src/sampler.ts +0 -1012
  513. package/src/serve/admin-routes.ts +0 -184
  514. package/src/serve/assets/curve-terrain.html +0 -105
  515. package/src/serve/assets/training-inference-map.html +0 -575
  516. package/src/serve/audio-routes.ts +0 -323
  517. package/src/serve/aux-routes.ts +0 -132
  518. package/src/serve/batch-scheduler.ts +0 -8
  519. package/src/serve/checkpoint-identity.ts +0 -64
  520. package/src/serve/generation-gateway.ts +0 -537
  521. package/src/serve/isolate.ts +0 -609
  522. package/src/serve/kv-budget.ts +0 -14
  523. package/src/serve/lab-routes.ts +0 -341
  524. package/src/serve/lane-registry.ts +0 -45
  525. package/src/serve/model-admin-routes.ts +0 -155
  526. package/src/serve/model-binding.ts +0 -49
  527. package/src/serve/model-host.ts +0 -511
  528. package/src/serve/parent-application.ts +0 -56
  529. package/src/serve/prompt-response-trace.ts +0 -161
  530. package/src/serve/response-lease.ts +0 -24
  531. package/src/serve/session-search.ts +0 -326
  532. package/src/serve/static-routes.ts +0 -101
  533. package/src/serve/transcription-server.ts +0 -64
  534. package/src/serve/transcription-service.ts +0 -502
  535. package/src/server.ts +0 -1282
  536. package/src/spec/assistant-rows.ts +0 -93
  537. package/src/spec/assistant-source.ts +0 -126
  538. package/src/spec/cache-identity.ts +0 -22
  539. package/src/spec/deepspec-rows.ts +0 -29
  540. package/src/spec/deepspec-source.ts +0 -145
  541. package/src/spec/dflash-source.ts +0 -149
  542. package/src/spec/drafter.ts +0 -304
  543. package/src/spec/dspark/ab-stats.ts +0 -136
  544. package/src/spec/dspark/calibration.ts +0 -112
  545. package/src/spec/dspark/data-dflash.ts +0 -156
  546. package/src/spec/dspark/data.ts +0 -201
  547. package/src/spec/dspark/deepspec-module.ts +0 -776
  548. package/src/spec/dspark/generate-dflash.ts +0 -245
  549. package/src/spec/dspark/generate.ts +0 -290
  550. package/src/spec/dspark/loader.ts +0 -23
  551. package/src/spec/dspark/loss.ts +0 -213
  552. package/src/spec/dspark/module-dflash.ts +0 -560
  553. package/src/spec/dspark/module.ts +0 -540
  554. package/src/spec/dspark/quantize-drafter.ts +0 -91
  555. package/src/spec/dspark/sample.ts +0 -112
  556. package/src/spec/generate.ts +0 -276
  557. package/src/spec/glm52-mtp-graph.ts +0 -29
  558. package/src/spec/glm52-mtp-rows.ts +0 -132
  559. package/src/spec/glm52-mtp-source.ts +0 -318
  560. package/src/spec/glm52-mtp-state.ts +0 -33
  561. package/src/spec/ngram-source.ts +0 -226
  562. package/src/spec/projected-draft-rows.ts +0 -73
  563. package/src/spec/qwen-mtp-module.ts +0 -171
  564. package/src/spec/qwen-mtp-rows.ts +0 -230
  565. package/src/spec/qwen-mtp-source.ts +0 -386
  566. package/src/spec/qwen-mtp-state.ts +0 -30
  567. package/src/spec/serve-loop.ts +0 -587
  568. package/src/spec/source.ts +0 -228
  569. package/src/spec/two-model-rows.ts +0 -62
  570. package/src/spec/two-model.ts +0 -113
  571. package/src/ssd-cache.ts +0 -431
  572. package/src/ssd-durability.ts +0 -209
  573. package/src/storage/host-buffer.ts +0 -24
  574. package/src/storage/kv-writer.ts +0 -83
  575. package/src/storage/kv-writer.worker.js +0 -264
  576. package/src/storage/retention-policy.ts +0 -37
  577. package/src/tiered-prompt-cache.ts +0 -45
  578. package/src/tokenizer.ts +0 -335
  579. package/src/tool-approvals.ts +0 -131
  580. package/src/tool-call.ts +0 -695
  581. package/src/train/dataset.ts +0 -484
  582. package/src/train/diffusion-lora.ts +0 -216
  583. package/src/train/export.ts +0 -32
  584. package/src/train/flash-cce.ts +0 -1234
  585. package/src/train/forward.ts +0 -266
  586. package/src/train/fuse.ts +0 -243
  587. package/src/train/index.ts +0 -46
  588. package/src/train/job.ts +0 -205
  589. package/src/train/lora-params.ts +0 -254
  590. package/src/train/loss.ts +0 -1591
  591. package/src/train/merge.ts +0 -195
  592. package/src/train/optimizer.ts +0 -183
  593. package/src/train/prefix-shared.ts +0 -490
  594. package/src/train/rank.ts +0 -137
  595. package/src/train/segmented.ts +0 -1828
  596. package/src/train/steel-qmm-header.ts +0 -1461
  597. package/src/train/trainer.ts +0 -1221
  598. package/src/train/watch.ts +0 -542
  599. package/src/vision/diffusion-vision.ts +0 -620
  600. package/src/vision/embedder.ts +0 -176
  601. package/src/vision/preprocess.ts +0 -247
  602. package/src/vision/prompt.ts +0 -409
  603. package/src/vision/qwen3vl-preprocess.ts +0 -409
  604. package/src/vision/qwen3vl-prompt.ts +0 -233
  605. package/src/vision/qwen3vl-tower.ts +0 -566
  606. package/src/vision/siglip.ts +0 -607
  607. package/src/vision/video-frames.ts +0 -148
  608. package/src/web/app.html +0 -2155
  609. package/src/web/app.js +0 -5863
  610. package/src/web/skills/memory/SKILL.md +0 -120
  611. package/src/web/skills/web-research/SKILL.md +0 -46
  612. package/src/web/skills.ts +0 -97
  613. package/src/web/src/adapters-panel.ts +0 -288
  614. package/src/web/src/assistant.ts +0 -451
  615. package/src/web/src/chat.ts +0 -1060
  616. package/src/web/src/composer.ts +0 -962
  617. package/src/web/src/dataset.ts +0 -132
  618. package/src/web/src/finetune.ts +0 -267
  619. package/src/web/src/hub.ts +0 -428
  620. package/src/web/src/main.ts +0 -98
  621. package/src/web/src/markdown.ts +0 -0
  622. package/src/web/src/memory-panel.ts +0 -572
  623. package/src/web/src/model-picker.ts +0 -159
  624. package/src/web/src/palette.ts +0 -346
  625. package/src/web/src/protocol.ts +0 -26
  626. package/src/web/src/quantize.ts +0 -167
  627. package/src/web/src/rag.ts +0 -274
  628. package/src/web/src/sessions.ts +0 -469
  629. package/src/web/src/shell.ts +0 -804
  630. package/src/web/src/ui-catalog.ts +0 -78
  631. package/src/web/src/voice.ts +0 -141
  632. package/src/web/vendor/README.md +0 -61
  633. package/src/web-assets.ts +0 -47
  634. package/src/web-tools.ts +0 -478
  635. package/src/weights.ts +0 -255
  636. /package/src/{serve → server}/completion-sink.ts +0 -0
  637. /package/src/{serve → server}/generated-token-history.ts +0 -0
  638. /package/src/web/{icon.svg → public/icon.svg} +0 -0
  639. /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
  640. /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
package/src/cli.ts DELETED
@@ -1,3490 +0,0 @@
1
- #!/usr/bin/env bun
2
- // mlx-bun CLI: the appliance interface.
3
- //
4
- // mlx-bun get <org/repo> [--revision main] resumable verified download
5
- // mlx-bun scan index the HF cache
6
- // mlx-bun ls [--vision] [--max-size 10GB] [query]
7
- // mlx-bun fit <query> [--ctx 8192] [--skus]
8
- // mlx-bun serve [query] [--port 8080] [--memory-budget GB]
9
- // mlx-bun evals recent benchmark runs
10
- // mlx-bun harness pi [--base-url <url>] [--remove] connect your own pi to the local server
11
-
12
- import { existsSync } from "node:fs";
13
- import { basename, join, resolve } from "node:path";
14
- import { Registry } from "./registry";
15
- import { loadModelConfig, parseTurboQuantScheme, type TurboQuantScheme } from "./config";
16
- import { fit, skuMatrix, thisMachine, type FitKvScheme } from "./fit";
17
- import { resolveKvScheme } from "./kv-scheme";
18
- import { EvalDB } from "./evaldb";
19
- import pkg from "../package.json" with { type: "json" };
20
- import { renderHelp } from "./tui";
21
- import { isGlm52Config, isSupportedModelRecord } from "./model/support";
22
- import {
23
- GLM52_G5_ASPIRATIONAL_DECODE_TPS,
24
- GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS,
25
- GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES,
26
- GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS,
27
- GLM52_G5_MEASURED_WARM_DECODE_TPS,
28
- planGlm52MemoryForArtifact,
29
- } from "./model/glm52-memory";
30
- import type { JobEvent } from "./jobs/types";
31
- import {
32
- configureRuntime,
33
- runtimeValue,
34
- type RuntimeKey,
35
- } from "./runtime-config";
36
-
37
- const argv = process.argv.slice(2);
38
- // Private self-exec protocol for managed jobs in the single-file binary.
39
- // Literal imports in job-entry keep the numerical runners inside the bundle.
40
- if (argv[0] === "__job") {
41
- const { runJobEntry } = await import("./jobs/job-entry");
42
- await runJobEntry(argv[1]);
43
- process.exit(0);
44
- }
45
-
46
- // The appliance path: naked `mlx-bun` (or only options, e.g.
47
- // `mlx-bun --port 9000`) runs `serve` — first run downloads a model and
48
- // opens the chat UI. `--help`/`--version` and explicit subcommands still win.
49
- const HELP_VERSION_FLAGS = ["--help", "-h", "--version", "-v"];
50
- const noSubcommand =
51
- !argv[0] || (argv[0].startsWith("-") && !HELP_VERSION_FLAGS.includes(argv[0]));
52
- const cmd = noSubcommand ? "serve" : argv[0];
53
-
54
- const OVERVIEW = `mlx-bun ${pkg.version} — local AI on Apple silicon. One binary, no Python.
55
-
56
- Usage: mlx-bun [command] [options] (no command = serve)
57
-
58
- Commands:
59
- pi Launch a pi coding-agent session on a local model (the appliance path)
60
- serve Start the OpenAI/Anthropic-compatible server + web app
61
- get Download a model from Hugging Face (resumable, verified)
62
- ls List downloaded models
63
- gc Reclaim superseded model snapshots + dead blobs from the HF cache
64
- fit Will a model fit this machine? Memory + speed assessment
65
- scan Re-index the Hugging Face cache
66
- harness Connect your own pi install to the local mlx-bun server
67
- train Fine-tune a LoRA adapter (SFT / DPO / ORPO) on your data
68
- train-watch Live dashboard for a training run (tails metrics.jsonl)
69
- fuse Merge a LoRA adapter into the base weights (standalone model)
70
- convert Quantize an HF model into a local MLX snapshot
71
- upload Push a local model directory to the Hugging Face Hub
72
- perplexity Perplexity of a model over a local text/JSONL dataset
73
- memory Set up + manage your local AI's personal wiki memory
74
- generate One-shot raw text generation from a local model (no server)
75
- benchmark Measure decode/prefill speed of OUR stack on this machine
76
- embed Text embeddings from a local embedding model (Qwen3-Embedding)
77
- transcribe Speech-to-text from an audio file with a local Whisper model
78
- dictate Push-to-talk: stream your microphone into Whisper, print/copy/type the text
79
- evals Show recent benchmark runs (all stacks)
80
- help Show help for a command (also: mlx-bun <command> --help)
81
-
82
- Options:
83
- -h, --help Show help
84
- -v, --version Show version
85
-
86
- Examples:
87
- mlx-bun # download (first run) + serve + open the chat UI
88
- mlx-bun pi # start (download if needed) and chat with an agent
89
- mlx-bun serve 12B # serve the 12B; web app at http://localhost:8080/
90
- mlx-bun train e4b --data ./prefs # ORPO LoRA fine-tune on {prompt,chosen,rejected} data
91
- mlx-bun get mlx-community/gemma-4-12B-it-OptiQ-4bit`;
92
-
93
- const SERVER_FLAGS = `Server options:
94
- --model <path|query> Explicit model override (mlx_lm.server spelling):
95
- a directory containing config.json loads from that
96
- path; anything else resolves like the positional
97
- query. Wins over the positional and auto-pick.
98
- --host <addr> Interface to bind [default: 127.0.0.1, loopback
99
- only; use 0.0.0.0 to expose on your network]
100
- --port <n> Listen port [default: 8080]
101
- --memory-budget <GB> Admission-control memory budget; requests that
102
- cannot fit are rejected instead of crashing the
103
- GPU [default: machine RAM × 0.75, check-only]
104
- --kv-budget <GB> Aggregate KV budget across concurrent batch rows
105
- (decimal GB): joiners whose projected KV
106
- (prompt + max_tokens) would exceed it queue until
107
- rows finish; a request over the budget alone is
108
- rejected. Off unless set.
109
- --prompt-cache <GB> Prompt (KV) cache byte cap [default: 8 GB;
110
- --prompt-cache 0 disables the cache]
111
- --ssd-cache <dir> SSD cold tier for the prompt cache: prefix KV
112
- spills to disk on eviction and persists across
113
- restarts (long-context agent TTFT drops from a
114
- full re-prefill to a zero-copy mmap restore).
115
- Off unless set.
116
- --ssd-cache-max <GB> Optional SSD tier byte cap; 0 = unlimited
117
- [default: unlimited]
118
- --ssd-demote-idle <sec> Demote prompt-cache entries idle this long to the
119
- SSD tier, freeing their GPU memory (prefixes stay
120
- reachable via zero-copy restore). 0 disables.
121
- [default: 300 when --ssd-cache is on]
122
- --ssd-cache-verify Verify tensor hashes on every restore (reads all
123
- bytes eagerly — integrity paranoia only)
124
- --generation-checkpoint <tokens>
125
- Queue an in-flight generation snapshot to the SSD
126
- cache every N emitted tokens. Repeating the same
127
- request after restart replays and resumes it.
128
- Requires --ssd-cache; supports ordinary requests
129
- with shared cache layouts.
130
- --isolate Run the inference engine as a CHILD process and
131
- keep this process a pure UI/API proxy — the UI
132
- stays instant under GPU load and survives engine
133
- crashes (respawned). /ws/chat not proxied yet.
134
- Requests naming another model (exact /v1/models
135
- id) spawn/route to THAT model's engine child.
136
- --model-pool <n> With --isolate: max RESIDENT model engines
137
- [default: 1]. Over the cap, the least-recently-
138
- used engine drains, demotes its caches to the
139
- SSD tier, and exits — switching back respawns
140
- it with all state restored from disk.
141
- --unix <path> (internal) listen on a unix socket instead of a
142
- TCP port — the engine half of --isolate.
143
- --batch <n> Max concurrent requests decoded together
144
- [default: 8]. A lone scheduler request uses the
145
- parity- and latency-gated B=1 fast path; concurrent
146
- requests can form B=N (agent fan-out). 1 pins the
147
- preserved strict serial executor.
148
- --decode-concurrency is accepted as the
149
- mlx_lm.server alias (semantics differ; see
150
- docs/reference/server-config.md)
151
- --no-open Don't open the chat UI in your browser on start.
152
- By default an interactive terminal opens
153
- http://<host>:<port>/#/chat once the server is up.
154
- --allow-private-media Let image_url/audio_url content parts fetch from
155
- private/loopback/link-local hosts (NAS, another
156
- LAN box). Blocked by default: a request's URL is
157
- attacker-controlled input, and the default keeps
158
- the server from being steered at cloud metadata
159
- or LAN-internal services (SSRF). data: URLs are
160
- always fine; the 10 s timeout and 64 MB response
161
- cap apply either way.
162
-
163
- Model & quality:
164
- --adapter <dir> Mount a LoRA adapter at startup and use it as the
165
- default for requests that don't select one (a
166
- request's \`adapter\` field, incl. "none", wins).
167
- --adapter-path is accepted as the mlx_lm.server
168
- alias. Hot-swap via POST /v1/adapters unchanged.
169
- --whisper-model <p|query> Whisper checkpoint for /v1/audio/transcriptions
170
- [default: the first downloaded whisper model,
171
- resolved on the first request]. Serving a whisper
172
- model ALONE starts a transcription-only server.
173
- --preload Transcription-only server: load the Whisper
174
- weights before listening instead of on the first
175
- request.
176
- --whisper-idle-unload <s> Keep the Whisper weights for this many idle seconds
177
- after a take before disposing them. [default: 0 —
178
- released right after every take, so the chat model
179
- has the memory back before it prefills; a cold
180
- take costs ~150 ms more than a warm one]
181
- --whisper-resident Never dispose the Whisper weights (sotto-style).
182
- --draft-model <query> Speculative decoding: a drafter proposes tokens
183
- the main model verifies (exact results, faster
184
- decode when drafts land). The artifact's kind is
185
- auto-detected — a full same-tokenizer model
186
- (mlx_lm.server parity), a Gemma "-assistant"
187
- KV-borrowing drafter, or a DSpark checkpoint.
188
- Supported draft providers use shared batched
189
- execution; ordinary requests can batch too.
190
- --draft-kind <kind> Override draft-artifact detection:
191
- two-model | assistant | dspark | deepspec | mtp | ngram
192
- (deepspec = DeepSeek's released Gemma4DSparkModel
193
- drafters; mtp = a NATIVE multi-token-prediction
194
- head split from the target's own release, e.g.
195
- mlx-community/Qwen3.8-27B-MTP-bf16 — auto-detected
196
- from model_type *_mtp;
197
- ngram = MODEL-FREE prompt lookup — no
198
- --draft-model, drafts copied from the request's
199
- own context, lossless) [default: auto]
200
- --num-draft-tokens <n> Drafts per verify round [default: 3; ngram: 10;
201
- DSpark pins to its trained block width]
202
- --ngram-max <k> Prompt-lookup longest match window [default: 3]
203
- --ngram-min <k> Prompt-lookup shortest match window [default: 1]
204
- --mtp <on|off> GLM-5.2 checkpoint-native MTP. On by default;
205
- routes requests through the exact serial verify
206
- lane. off enables ordinary continuous batching.
207
- --context-length <n> GLM-5.2 resource-plan context reservation
208
- [default: 4096]
209
- --kv-quant <mode> KV cache quantization: config (per-layer
210
- kv_config.json when the model ships one), off
211
- (bf16), 4 / 8 (uniform bits), or turbo[:k<bits>v
212
- <bits>] (rotation-based TurboQuant — default
213
- k8v3; kBits in {2,4,5,8}, vBits in {2,3,4,5,8};
214
- docs/design/turboquant.md)
215
- [default: off — quantized KV measured 5-20% slower
216
- decode than bf16 at ≤16k (on mlx-lm too); it buys
217
- memory headroom, so opt in when context is tight.
218
- The batched engine (--batch N) is bf16-only: an
219
- explicit --kv-quant routes those requests serial.
220
- turbo is solo-only in v1 (full-attention layers
221
- only; sliding-window layers stay bf16)]
222
- --paged-kv OPTIONAL vLLM-style paged KV storage (block pool +
223
- gather before the stock SDPA; env MLX_BUN_PAGED_KV=1).
224
- Gemma4-family, bf16; supports --batch N.
225
- Refuses --kv-quant and --draft-model; bypasses the prompt
226
- cache; runs uncompiled decode. Bit-exact with the
227
- plain path; expect a small decode cost at batch=1
228
- (the gather copy). docs/design/kv-cache.md
229
- [default: off]
230
- --paged-kv-block-size <n> Tokens per KV block [default: 256 = the plain
231
- cache's growth step]
232
- --thinking <true|false> Default for the chat template's enable_thinking
233
- variable (CPM and other hybrid-reasoning models);
234
- a request's chat_template_kwargs overrides it
235
- [default: the model's own, false for CPM]
236
- --temperature <n> Server-wide sampling defaults; a per-request field
237
- --top-p <n> still overrides them, and the browser chat (which
238
- --top-k <n> sends none) inherits them. --temp is accepted as
239
- an alias for --temperature (mlx_lm.server compat;
240
- NOTE mlx_lm.server defaults temperature to 0.0 —
241
- pass --temp 0 for its unset-request behavior)
242
- [default: the model's generation_config.json]
243
- --max-tokens <n> Completion cap when a request omits max_tokens
244
- (mlx_lm.server flag; --max-tokens 512 reproduces
245
- its default) [default: 128 GLM-5.2 (memory-plan
246
- reservation); otherwise none — a defaulted
247
- request runs to EOS or the admitted context]
248
- --hlg-sampling on|off Piecewise tone-curve sampling (HLG): rolls off the
249
- top, boosts the mids, gentles the tail. Gain folds
250
- from --temperature. [default: off]
251
- See docs/archive/hlg-sampling.md
252
- --hlg-width <nats> HLG mid-region half-width [default: 4]
253
- --hlg-shoulder <nats> HLG highlight rolloff scale [default: 4]
254
- --hlg-toe <nats> HLG shadow rolloff scale [default: 6]
255
- --hlg-pivot-offset <nats> HLG pivot: nats below the top token [default: 6]
256
-
257
- Parity tier (an ALIAS for the per-fork flags below; each expands to the fastest
258
- set of forks that still holds the guarantee — a per-fork flag overrides one):
259
- --l1 Bit-for-bit IDENTICAL to mlx-lm (bf16 KV, compiled
260
- activations — the faithful @mx.compile kernels).
261
- No tier given = --l1 (decided 2026-07-05: the L1
262
- faithful kernels match mlx-lm 1.00x on every model;
263
- anything output-changing must beat that baseline in
264
- a paired A/B to earn a default).
265
- --l2 Bit-for-bit IDENTICAL to mlx-optiq (quantized KV +
266
- fused prefill SDPA, stock unfused decode — the
267
- optiq-golden composition)
268
-
269
- Kill switches (bit-exact A/B levers — each selects a slower same-parity path):
270
- --compiled-decode on|off Compiled decode graphs [default: on]
271
- --compiled-activations on|off
272
- Compiled geglu/swiglu (mlx-lm's @mx.compile kernel —
273
- bit-exact AND fewer dispatches). off = the uncompiled
274
- composition (same parity, slower) [default: on]
275
- --fused-sdpa on|off Fused SDPA path (quantized KV only) [default:
276
- follows --kv-quant — on for config (the optiq
277
- composition), off for uniform 4/8 (the mlx-lm
278
- composition) and bf16]
279
- --force-wire Wire weights into memory at load
280
- --expert-offload MoE only: serve experts from a page-aligned file
281
- mmap (built on first use) — keeps the model out of
282
- memory pressure (phys_footprint ≈ active params),
283
- bit-exact with the resident path`;
284
-
285
- const HELP: Record<string, string> = {
286
- dictate: `mlx-bun dictate — push-to-talk dictation from your microphone
287
-
288
- Usage: mlx-bun dictate [query] [options]
289
-
290
- Captures the mic through the AVAudioEngine sidecar (16 kHz mono; macOS asks
291
- for Microphone permission on first use) and streams it into a Whisper
292
- transcription session while you speak — every finished 30 s window is
293
- transcribed during capture, so the result lands about one window after you
294
- stop. Silence never runs Whisper (Silero VAD).
295
-
296
- Modes:
297
- (default) Press Enter to start a take, Enter again to stop;
298
- q + Enter or Ctrl-C quits
299
- --hotkey [keycode] Hold a key instead (default 61 = Right Option, like
300
- sotto); needs Input Monitoring for your terminal
301
- Backend:
302
- [query] / --model <p> Whisper model for in-process transcription (default:
303
- the first downloaded whisper checkpoint)
304
- --server <url> Use a running mlx-bun server's /v1/audio/sessions
305
- instead of loading the model here
306
- --idle-unload <s> Release the weights after this many idle seconds
307
- (0 = right after each take; default 30); --resident
308
- keeps them loaded
309
- Decoding:
310
- --language <code> default en; "auto" detects
311
- --beam-size <n> beam search (default greedy)
312
- --prompt "…" initial prompt / --vocabulary "a,b,c" hint terms
313
- --no-vad skip the Silero gate
314
- Output (any combination; default prints):
315
- --copy copy the transcript to the clipboard (pbcopy)
316
- --type type it into the frontmost app via System Events
317
- keystrokes (needs Accessibility for your terminal);
318
- --type-delay <s> waits first so you can Cmd-Tab
319
- (default 1 in Enter mode, 0 with --hotkey)`,
320
- transcribe: `mlx-bun transcribe — speech-to-text with a local Whisper model (no server)
321
-
322
- Usage: mlx-bun transcribe <audio-file> [query] [options]
323
-
324
- Loads a Whisper checkpoint (mlx-community/whisper-*) and prints the
325
- transcript. WAV is decoded natively; any other container CoreAudio reads
326
- (mp3/m4a/flac/ogg/aiff/…) is transcoded with afconvert first.
327
-
328
- Model:
329
- [query] / --model <p> Whisper model; omitted = the first downloaded
330
- \`whisper\` checkpoint (errors with an mlx-bun get
331
- suggestion when none)
332
- Decoding:
333
- --language <code> ISO code or name ("en", "japanese"); "auto" or omitted
334
- detects the language from the first 30 s
335
- --task translate Translate to English instead of transcribing
336
- --beam-size <n> Beam search width (default: greedy)
337
- --temperature <t> Sampling temperature; the default runs the
338
- (0, 0.2, …, 1.0) fallback ladder like mlx-whisper
339
- --no-fallback Temperature 0 only, no retry ladder
340
- --prompt "…" Initial prompt (vocabulary hints / style)
341
- --no-timestamps Decode text only (<|notimestamps|>)
342
- --no-condition Do not condition each window on the previous text
343
- --vad Silero VAD gate: skip Whisper when no speech is
344
- detected (weights: ggml-org/whisper-vad in the HF cache
345
- or --vad-model <path>); --vad-trim crops to speech
346
- --word-timestamps Word-level timestamps (verbose_json / verbose output)
347
- --faithful Run the oracle-parity graph instead of the fast path
348
- --audio-ctx <n> Lab: encode only n of the 1500 encoder positions
349
- (whisper.cpp -ac); cheaper for short clips, degrades
350
- below ~1024 — off by default
351
- Output:
352
- --format <f> text (default) · json · verbose_json · srt · vtt
353
- --verbose Print each segment with its time range as it decodes`,
354
- pi: `mlx-bun pi — drop into a coding-agent session on a local model
355
-
356
- Usage: mlx-bun pi [options] [message...]
357
-
358
- Runs the pi coding agent's own terminal UI in-process against the local
359
- model — nothing to install, pi is built in. Reuses a healthy local
360
- server when one is running; otherwise serves e4b by default (override
361
- with --query) and starts a server for the session (the server ends with
362
- the session). Fresh install: downloads a small starter model first so
363
- you're chatting in minutes, then streams e4b in the background — it
364
- becomes the default on the next run.
365
-
366
- Model selection:
367
- -q, --query <q> Model to serve when starting a server (registry query)
368
-
369
- ${SERVER_FLAGS}
370
-
371
- Run modes:
372
- message... Pre-fill the first turn (interactive)
373
- -p, --print [msg] One-shot: print the reply and exit (also reads piped
374
- stdin); add --json for the full event stream
375
- --mode json|rpc Structured print (json) or JSONL RPC over stdio
376
-
377
- Already use pi? Connect your own pi to this local model instead:
378
- mlx-bun harness pi`,
379
-
380
- serve: `mlx-bun serve — OpenAI/Anthropic-compatible server for a local model
381
-
382
- Usage: mlx-bun serve [query] [options]
383
-
384
- [query] Registry query (e.g. "12B", "e4b", a repo substring).
385
- --query <q> is accepted for the positional. Omitted:
386
- serves e4b (the default), else the largest downloaded
387
- model that fits. Fresh install: downloads a small
388
- starter model first, then e4b in the background
389
- (default on next run).
390
-
391
- ${SERVER_FLAGS}
392
-
393
- Endpoints: /v1/chat/completions, /v1/completions, /v1/messages, /v1/responses,
394
- /v1/models, /v1/adapters, /health, /stats, /fit, /library, /downloads —
395
- web app at / (lands on chat) — /chat, /status, /quantize, /finetune, /dataset deep-link its views`,
396
-
397
- get: `mlx-bun get — download a model from Hugging Face
398
-
399
- Usage: mlx-bun get <org/repo | substring> [options]
400
-
401
- <org/repo> Full repo id (e.g. mlx-community/gemma-4-12B-it-OptiQ-4bit)
402
- <substring> No "/" = a registry query over already-downloaded repos
403
- (e.g. \`mlx-bun get 12B\` re-gets/refreshes the match)
404
-
405
- Options:
406
- --revision <rev> Git revision [default: main]
407
-
408
- Resumable (Range requests against partial blobs) and verified (sha256
409
- for LFS blobs). Re-running after an interruption continues where it
410
- stopped. Before payload transfer, complete blobs and partial prefixes are
411
- credited and the target volume must fit the remainder plus a 1 GiB reserve.
412
- Uses HF_TOKEN / hf auth login credentials when present.
413
- When upstream pushed a new revision, the previous snapshot stays on
414
- disk — \`mlx-bun gc\` reclaims it.`,
415
-
416
- ls: `mlx-bun ls — list downloaded models
417
-
418
- Usage: mlx-bun ls [query] [options]
419
-
420
- One row per repo (the canonical revision: refs/main). The HF cache keeps
421
- a snapshot dir per downloaded revision — superseded ones are hidden here
422
- and reclaimed by \`mlx-bun gc\`.
423
-
424
- Options:
425
- --vision Only vision-capable models (SigLIP sidecar or the
426
- unified encoder-free tower)
427
- --max-size <size> Filter by weight size (e.g. 10GB, 800MB)
428
- --all-revisions One row per cached snapshot (canonical marked *)`,
429
-
430
- gc: `mlx-bun gc — reclaim superseded snapshots + dead blobs
431
-
432
- Usage: mlx-bun gc [options]
433
-
434
- Every \`get\` that follows an upstream push creates a NEW
435
- snapshots/<commit> dir; the old one (and any blobs only it references)
436
- stays on disk forever. gc keeps the snapshots refs/* point at, deletes
437
- the rest, then deletes blobs no surviving snapshot links to.
438
-
439
- Without --yes it only prints what would be deleted (per-repo reclaim).
440
-
441
- Options:
442
- --yes Actually delete (destructive; default prints only)
443
- --dry-run Print the plan and never delete, even with --yes
444
- --force Also prune superseded snapshots that contain files
445
- the canonical revision lacks (normally skipped with
446
- a warning — deleting them loses the only copy)`,
447
-
448
- fit: `mlx-bun fit — will this model fit on this machine?
449
-
450
- Usage: mlx-bun fit <query> [options]
451
-
452
- Options:
453
- --ctx <tokens> Context size to assess [default: 8192]
454
- --kv-quant <mode> Bill the KV cache quantized: 4 | 8 | config | off
455
- (mirrors serve --kv-quant; default bf16)
456
- --skus Also print the Apple silicon SKU matrix
457
-
458
- Solves weights + KV growth + prefill transient against wired memory and
459
- predicts decode speed from memory bandwidth.`,
460
-
461
- scan: `mlx-bun scan — re-index the Hugging Face cache
462
-
463
- Usage: mlx-bun scan
464
-
465
- Reads config.json + safetensors headers (never tensor bytes) for every
466
- snapshot in ~/.cache/huggingface/hub and refreshes the model registry.`,
467
-
468
- harness: `mlx-bun harness pi — connect your own pi to the local model
469
-
470
- Usage: mlx-bun harness pi [options]
471
-
472
- For people who already use pi: this points your existing pi install at
473
- the local mlx-bun server so you can run pi the way you already do, on a
474
- local model. (To just chat now with no setup, use \`mlx-bun pi\` — the
475
- built-in agent.)
476
-
477
- Options:
478
- --base-url <url> Server base URL [default: http://127.0.0.1:8080/v1]
479
- --remove Disconnect (delete the extension)
480
-
481
- Installs a small discovery extension into ~/.pi/agent/extensions that
482
- registers mlx-bun as a pi provider (models discovered live from
483
- /v1/models at pi startup). One file, reversible with --remove; never
484
- touches your existing pi config. Then: pi --provider mlx-bun`,
485
-
486
- benchmark: `mlx-bun benchmark — measure OUR stack on this machine
487
-
488
- Usage: mlx-bun benchmark [query] [options]
489
-
490
- Runs mlx-bun by itself (no other systems required) and records the
491
- result to the eval DB. For quotable, cross-stack numbers use
492
- bun scripts/bench-serve.ts all from the repo — it preflight-gates on an idle machine
493
- and runs the mlx-lm/optiq comparison legs.
494
-
495
- [query] Model to benchmark (--query <q> is accepted for the
496
- positional) [default: auto-pick]
497
-
498
- Options:
499
- --model <path|query> Explicit model override: a directory containing
500
- config.json loads from that path; anything else
501
- resolves like the positional query. Wins over both.
502
- --tokens <n> Tokens to decode per run [default: 256]
503
- --runs <n> Runs (median reported) [default: 3]
504
- --prompt-tokens <n> Pad the prompt to ~n tokens (long-context decode)
505
- --kv-quant <mode> config | off | 4 | 8 | turbo[:k<bits>v<bits>]
506
- [default: off, the historical baseline]
507
-
508
- Kill switches (--compiled-decode, --compiled-activations, --fused-sdpa)
509
- apply to the run — A/B by running twice.`,
510
-
511
- embed: `mlx-bun embed — text embeddings from a local embedding model
512
-
513
- Usage: mlx-bun embed [query] --text "…" [options]
514
- mlx-bun embed [query] "…" (text as a positional)
515
- echo -e "line one\\nline two" | mlx-bun embed [query]
516
-
517
- Embeds text with a local Qwen3-Embedding model (last-token pooled,
518
- L2-normalized — bit-exact vs mlx-lm). One text per --text, or one per
519
- line on stdin. Prints one JSON array (the vector) per input line.
520
-
521
- [query] Embedding model (--query <q> is accepted for the
522
- positional) [default: the first downloaded
523
- embedding model; errors with an mlx-bun get
524
- suggestion when none is downloaded]
525
-
526
- Options:
527
- --text "…" Text to embed (omit to read lines from stdin)
528
- --instruct "…" Query instruction (Qwen3-Embedding query format);
529
- steers WHICH similarity axis the geometry reflects.
530
- Omit for plain document embeddings.
531
- --json Emit one OpenAI-style {object:"list",data:[…]} object
532
- instead of bare vector arrays
533
-
534
- For a server endpoint use \`mlx-bun serve <embedding-model>\` then
535
- POST /v1/embeddings.`,
536
-
537
- generate: `mlx-bun generate — one-shot raw text generation (no server)
538
-
539
- Usage: mlx-bun generate [query] --prompt "…" [options]
540
- mlx-bun generate [query] "…" (prompt as a positional)
541
-
542
- A direct one-shot entry point: load the model, generate, print, exit —
543
- no server, no chat UI. By default the prompt is rendered through the
544
- model's chat template; --raw feeds it verbatim.
545
-
546
- [query] Model (registry query; --query is accepted for the
547
- positional) [default: auto-pick]
548
-
549
- Options:
550
- --prompt "…" The prompt (or pass it as the positional)
551
- --raw Skip the chat template (verbatim prompt)
552
- --max-tokens <n> Completion cap [default: 256]
553
- --temperature <n> Sampling temperature (--temp is accepted as an alias)
554
- --top-p <n> Nucleus sampling
555
- --top-k <n> Top-k sampling
556
- --seed <n> Sampler seed (reproducible runs)
557
-
558
- Decode-path levers mirror serve: --l1/--l2 and the per-fork flags
559
- (--fused-sdpa/--compiled-decode/--compiled-activations/--kv-quant).
560
- The tier's KV scheme applies here too (--l2 = the model's kv_config,
561
- same tokens as serve); with no tier/--kv-quant the KV cache stays bf16
562
- (the bit-exact mlx-lm greedy path). mlx-lm compat = --l1.
563
-
564
- Aliases: mlx-bun gen`,
565
-
566
- evals: `mlx-bun evals — recent benchmark runs
567
-
568
- Usage: mlx-bun evals [options]
569
-
570
- Options:
571
- --limit <n> Rows to show [default: 20]
572
- --raw Full records as JSON lines (for scripts/jq)
573
- --clear Archive the eval DB (timestamped backup) and start fresh
574
-
575
- Table view: when, model, bench kind, KV mode, decode tok/s, TTFT,
576
- peak memory, commit. Runs are written by bun scripts/bench-serve.ts all.`,
577
-
578
- train: `mlx-bun train — fine-tune a LoRA adapter on your data
579
-
580
- Usage: mlx-bun train <model> --data <dir> [options]
581
-
582
- <model> Model to fine-tune (query or snapshot path; auto-picks
583
- the default model if omitted)
584
-
585
- Data: <dir> must contain train.jsonl (+ optional valid.jsonl). Rows are
586
- {prompt, chosen, rejected} for dpo/orpo, or {messages|text} for sft.
587
-
588
- Options:
589
- --data <dir> Dataset dir with train.jsonl (required)
590
- --method <m> sft | dpo | orpo [default: orpo]
591
- --adapter <dir> Output adapter dir
592
- [default: ~/.cache/mlx-bun/mlx-bun-finetunes/<method>-<model>]
593
- --iters <n> Training iterations [default: 100]
594
- --lr <f> Learning rate [default: orpo 1e-5 · dpo 5e-5 · sft 2e-4]
595
- --rank <n> LoRA rank [default: orpo 16 · else 8]
596
- --scale <f> LoRA scale [default: orpo 2.0 · else 1.0]
597
- --seq <n> Max sequence length [default: gemma 8192 · else 4096]
598
- --batch <n> Batch size [default: 1]
599
- --grad-accum <n> Gradient accumulation steps (effective batch =
600
- batch × grad-accum at batch-size-1 memory) [default: 1]
601
- --grad-clip <f> Gradient-norm clip (0 = off) [default: 1.0]
602
- --seed <n> Data-shuffle / init seed [default: 0]
603
- --val-size <n> Max validation examples per eval [default: 256]
604
- --lambda <f> ORPO odds-ratio weight [default: 0.1]
605
- --sft-scope <s> ORPO chosen-NLL scope: full (paper/TRL-faithful,
606
- prompt+response) | response (pre-2026-07 runs,
607
- bit-exact) [default: full]
608
- --seg <n> Layers per segment (segmented backward; orpo default 2)
609
- --save-every <n> Crash-safe mountable checkpoint every n steps
610
- --resume <dir> Warm-start LoRA weights from a checkpoint/adapter dir
611
- --no-flash Disable the flash-CCE Metal head (use the MLX fused head)
612
- --no-prefix Disable prefix-sharing (two-forward branches)
613
- --no-segment Disable the segmented backward (hold all activations)
614
- --dry-run Inspect the dataset + print the resolved plan, don't train
615
-
616
- ORPO defaults run the full stack: the [M,vocab]-free flash-CCE head +
617
- prefix-sharing + segmented backward (each falls back + logs if a row's
618
- preconditions aren't met). Gemma/e4b sets its required training env flags
619
- automatically. Adapters are mountable directly (mlx-bun serve --adapter).
620
- Run long jobs detached from your own shell: nohup mlx-bun train … &
621
- Watch a run live from another tab: mlx-bun train-watch <adapter-dir>`,
622
-
623
- "train-watch": `mlx-bun train-watch — live dashboard for a training run
624
-
625
- Usage: mlx-bun train-watch [adapter-dir]
626
-
627
- Tails <adapter-dir>/metrics.jsonl (written by mlx-bun train and the web
628
- fine-tune tile) and renders a live loss/speed/memory dashboard. Point it
629
- at the same --adapter directory the training run uses.
630
-
631
- [adapter-dir] Adapter directory to watch; --adapter <dir> is
632
- accepted for the positional
633
- [default: ~/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5]`,
634
- fuse: `mlx-bun fuse — merge a LoRA adapter into the base weights
635
-
636
- Usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [options]
637
-
638
- Folds the adapter's low-rank deltas into the base weights and writes a
639
- standalone, loadable model snapshot (weights + config + tokenizer). A
640
- quantized base keeps its exact quantization layout: fused modules are
641
- dequantized, updated, and re-quantized with their own per-module spec.
642
-
643
- <model> Base model (registry query or a snapshot path);
644
- --model is accepted as the mlx_lm.fuse spelling
645
-
646
- Options:
647
- --adapter <dir> Adapter directory (adapters.safetensors +
648
- adapter_config.json). --adapter-path is accepted
649
- as the mlx_lm.fuse alias [default: adapters]
650
- --save-path <dir> Output model directory [default: fused_model]
651
-
652
- Not supported (mlx_lm.fuse flags we don't implement — the command exits
653
- with an error rather than guessing): --de-quantize, --export-gguf,
654
- --gguf-path, --upload-repo.
655
-
656
- Serve the result: mlx-bun serve <save-path>`,
657
-
658
- convert: `mlx-bun convert — quantize an HF model into a local MLX snapshot
659
-
660
- Usage: mlx-bun convert --hf-path <repo-or-path> -q [options]
661
- mlx-bun convert <repo-or-path> -q [options] (positional source)
662
-
663
- Wraps mlx-bun's native quantize pipeline (the same engine as the web
664
- /api/quantize). The source may be a local model directory, a downloaded
665
- model (registry query), or a Hugging Face repo id — an un-downloaded
666
- repo is fetched first (resumable, verified; can be many GB).
667
-
668
- Options:
669
- --hf-path <src> Source model: local path, downloaded model, or HF
670
- repo id (--model and the bare positional are
671
- accepted as aliases)
672
- --mlx-path <dir> Output directory [default: mlx_model; must not
673
- already exist, matching mlx_lm.convert]
674
- -q, --quantize Quantize the model (uniform affine)
675
- --q-bits <n> Bits per weight: 4 or 8 [default: 4]
676
- --q-group-size <n> Quantization group size: 32 or 64 [default: 64]
677
- --upload-repo <id> Push the converted model to this Hugging Face repo
678
- after converting (mlx_lm.convert parity; same engine
679
- as \`mlx-bun upload\`, needs a write token)
680
-
681
- Mixed precision (the mlx-bun differentiator — OptiQ sensitivity sweep +
682
- knapsack per-layer bit allocation; implies quantization, no -q needed):
683
- --target-bpw <f> Target bits-per-weight (e.g. 4.5). Runs
684
- calibration → per-layer KL sensitivity → greedy
685
- knapsack; writes per-module bits into config.json
686
- --candidate-bits <l> Comma list the knapsack may pick from [default: 4,8]
687
- --calibration-mix <m> "optiq" or a JSONL path [default: optiq]
688
- --n-calibration <n> Calibration samples [default: 2]
689
- --rotate-weights Fold the model's offline TurboQuant rotation before
690
- quantization (auto-detects Llama/Qwen3.5/Qwen MTP)
691
- --rotation-seed <n> Deterministic rotation seed [default: 42]
692
-
693
- Not supported (mlx_lm.convert flags we don't implement — the command
694
- exits with an error rather than guessing): --dtype, -d/--dequantize,
695
- --q-mode other than affine, and the mlx-lm --quant-predicate recipes
696
- (use --target-bpw instead). Plain non-quantizing conversion is also not
697
- supported: pass -q or --target-bpw.`,
698
-
699
- upload: `mlx-bun upload — push a local model directory to the Hugging Face Hub
700
-
701
- Usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [options]
702
-
703
- Native push-to-hub, no Python (src/hf-push.ts — the same engine as the
704
- web UI's push buttons): creates the repo if needed (idempotent), uploads
705
- weights via the git-LFS batch protocol, and commits. Flag names match
706
- mlx_lm.upload.
707
-
708
- Options:
709
- --path <dir> Local model directory to upload [default: mlx_model]
710
- --upload-repo <id> Hub repo id, org/name or bare name (required)
711
- --private Create the repo as private (mlx-bun extension)
712
-
713
- Auth: needs a Hugging Face WRITE token, resolved from ~/.mlx-bun/hf.json
714
- (web UI Settings → Hugging Face), $HF_TOKEN, or ~/.cache/huggingface/token
715
- (hf auth login). Errors before uploading anything if none is found.
716
-
717
- Convert + push in one step: mlx-bun convert … --upload-repo <org/repo>`,
718
-
719
- perplexity: `mlx-bun perplexity — perplexity over a local text/JSONL dataset
720
-
721
- Usage: mlx-bun perplexity <model-query-or-path> --data-path <file> [options]
722
-
723
- mlx_lm.perplexity methodology, exactly: samples are visited in a seeded
724
- random order, tokenized, concatenated, and cut into NON-OVERLAPPING rows
725
- of --sequence-length tokens; per batch the model scores rows[:, :-1]
726
- against rows[:, 1:] in f32 (every position counts); reported as
727
- ppl = exp(mean CE) ± the delta-method standard error. The one deliberate
728
- difference: the data source is a LOCAL file (never a Hugging Face
729
- dataset download).
730
-
731
- <model> Model (registry query or snapshot path);
732
- --model (mlx_lm spelling) and --query are accepted
733
- for the positional
734
-
735
- Options:
736
- --data-path <file> .jsonl ({"text": …} rows) or plain .txt (required)
737
- --sequence-length <n> Tokens per row [default: 512]
738
- --num-samples <n> Rows to score (-1 = all available) [default: 256]
739
- --batch-size <n> Rows per forward [default: 8]
740
- --seed <n> Sample-shuffle seed [default: 123]`,
741
-
742
- memory: `mlx-bun memory — your local AI's personal wiki
743
-
744
- Usage: mlx-bun memory <subcommand>
745
-
746
- A local, durable memory for the assistant: a wiki of Markdown articles
747
- (~/.mlx-bun/wiki) it reads to remember your projects, people, and history
748
- across sessions. It's yours — git-tracked, editable in any tool (Obsidian
749
- opens it as a vault), and it never leaves the machine. A read-only Reference/
750
- folder symlinks mlx-bun's own docs into memory on day one. Once set up, it
751
- loads automatically into every \`mlx-bun pi\` and \`mlx-bun serve\` session.
752
-
753
- Subcommands:
754
- init, setup Create the wiki + walk through setup (idempotent);
755
- offers to import an existing vault and install the
756
- nightly synthesis job
757
- status Path, article count, git + schedule state
758
- open, browse [article]
759
- Open the wiki, or a specific article, in Obsidian
760
- (falls back to Finder / the default Markdown app)
761
- list List article titles + read-only Reference docs
762
- search <query> Search articles from the terminal
763
- toc <article> Print an article's headings + anchors
764
- section <article> <anchor>
765
- Print one article section
766
- links <article> Show resolved outbound + inbound wikilinks
767
- read <article> Print an article (stem, e.g. Archie_Project)
768
- synthesize Run the FULL synthesis DAG now (--since, --model,
769
- --dry-run); also: pipeline, all
770
- segment | extract | route | synthesize-stage
771
- Run ONE decomposed stage worker (--limit N —
772
- segment/extract/synthesize-stage only; --convs a,b).
773
- Each pulls its eligible work from the DB by state, walks
774
- oldest-conversation-first, persists, and exits — resumable,
775
- and runnable as separate concurrent processes on slices.
776
- link Deterministic cross-linking stage: inline-link first
777
- mentions + rebuild ## See also (--limit N; no model)
778
- schedule Install the nightly launchd job (--at HH:MM [03:00])
779
- unschedule Remove the nightly launchd job
780
-
781
- The read path is live: the assistant reads a wiki you set up by hand or
782
- import during \`memory init\`. Ask the assistant to "open my memory" or run
783
- \`mlx-bun memory open\` to browse it in Obsidian/Finder, or
784
- \`mlx-bun memory open <article>\` to jump to a specific page. Synthesis
785
- (conversations → articles) runs the full local pipeline via
786
- \`mlx-bun memory synthesize\` (or per-stage: segment/extract/route/
787
- synthesize-stage/link); the nightly job runs it on a schedule.`,
788
- };
789
-
790
- HELP.bench = HELP.benchmark!;
791
- HELP.gen = HELP.generate!;
792
- HELP.setup = HELP.memory!;
793
-
794
- function printHelp(topic?: string): never {
795
- if (topic && HELP[topic]) console.log(renderHelp(HELP[topic]));
796
- else if (topic) {
797
- console.error(`unknown command: ${topic}\n`);
798
- console.log(renderHelp(OVERVIEW));
799
- process.exit(1);
800
- } else console.log(renderHelp(OVERVIEW));
801
- process.exit(0);
802
- }
803
-
804
- if (cmd === "--help" || cmd === "-h") printHelp();
805
- if (cmd === "--version" || cmd === "-v" || cmd === "version") {
806
- console.log(`mlx-bun ${pkg.version}`);
807
- process.exit(0);
808
- }
809
- if (cmd === "help") printHelp(argv[1]);
810
- // Per-command --help/-h — except `pi`, whose extra args pass through
811
- // (use `mlx-bun help pi` for our help, `pi --help` for pi's).
812
- if (cmd !== "pi" && (argv.includes("--help") || argv.includes("-h"))) printHelp(cmd);
813
- if (cmd === "pi" && (argv[1] === "--help" || argv[1] === "-h") && argv.length === 2) printHelp("pi");
814
-
815
- const flag = (name: string): boolean => argv.includes(`--${name}`);
816
- const opt = (name: string, dflt: string | null = null): string | null => {
817
- const i = argv.indexOf(`--${name}`);
818
- return i > -1 && argv[i + 1] ? argv[i + 1]! : dflt;
819
- };
820
- /** Single-dash value flag (e.g. -q), the short-alias counterpart to opt(). */
821
- const optShort = (short: string, dflt: string | null = null): string | null => {
822
- const i = argv.indexOf(`-${short}`);
823
- return i > -1 && argv[i + 1] ? argv[i + 1]! : dflt;
824
- };
825
- const positional = (n: number): string | undefined =>
826
- argv.filter((a, i) => !a.startsWith("--") && (i === 0 || !argv[i - 1]!.startsWith("--"))).slice(1)[n];
827
-
828
- const gb = (bytes: number) => `${(bytes / 2 ** 30).toFixed(2)} GB`;
829
-
830
- /** First-run: fetch the MLX native runtime pack if no copy is resolvable
831
- * (env / beside the binary / cache / homebrew). ~52 MB, resumable,
832
- * sha256-verified. Must run BEFORE importing anything that dlopens
833
- * (./server, ./generate — they load libmlxc at module scope). */
834
- async function ensureNative(s?: import("./tui").Step): Promise<void> {
835
- const { nativeRuntimeDir, ensureNativeRuntime, NATIVE_PACK_VERSION } = await import("./native-pack");
836
- if (nativeRuntimeDir()) return;
837
- s?.update(`downloading the MLX native runtime (v${NATIVE_PACK_VERSION}, ~52 MB)`);
838
- await ensureNativeRuntime({
839
- onProgress: (received, total) => {
840
- const pct = total ? Math.floor((received / total) * 100) : 0;
841
- const line = `MLX native runtime ${gb(received)} / ${gb(total)} (${pct}%)`;
842
- if (s) s.update(line); else process.stdout.write(`\r ${line} `);
843
- },
844
- });
845
- if (!s) process.stdout.write("\n");
846
- }
847
-
848
- /** AppleScript that focuses the first tab whose URL contains `match` in a
849
- * literal-named browser, returning "ok"/"miss". The app name MUST be a literal
850
- * (not a variable): AppleScript loads an app's scripting dictionary by literal
851
- * name at COMPILE time, so `tell application someVar` can't resolve app-specific
852
- * terms like `active tab index` (fails with -2740). Referencing an uninstalled
853
- * app by literal name is also a compile error a `try` can't catch — hence we
854
- * only ever run a browser's script when pgrep says it's running (= installed). */
855
- function focusTabScript(app: string, kind: "chromium" | "safari", match: string): string {
856
- const m = JSON.stringify(match); // safe AppleScript string literal
857
- if (kind === "safari") {
858
- return `tell application "${app}"
859
- repeat with w in windows
860
- repeat with t in tabs of w
861
- if URL of t contains ${m} then
862
- set current tab of w to t
863
- activate
864
- return "ok"
865
- end if
866
- end repeat
867
- end repeat
868
- end tell
869
- return "miss"`;
870
- }
871
- return `tell application "${app}"
872
- repeat with w in windows
873
- set k to 0
874
- repeat with t in tabs of w
875
- set k to k + 1
876
- if URL of t contains ${m} then
877
- set active tab index of w to k
878
- set index of w to 1
879
- activate
880
- return "ok"
881
- end if
882
- end repeat
883
- end repeat
884
- end tell
885
- return "miss"`;
886
- }
887
-
888
- /** Open the chat UI, reusing an already-open tab on this host:port instead of
889
- * spawning a duplicate. Each browser is scripted via its own literal-named
890
- * AppleScript, run ONLY when pgrep confirms it's running (so the literal `tell`
891
- * always compiles). Falls back to a plain `open` (new tab) when no running
892
- * browser has the tab, for browsers we don't script (e.g. Firefox), or if
893
- * AppleScript is blocked. Best-effort and fire-and-forget: any failure just
894
- * means a fresh tab, so it never regresses the plain-open path. The first focus
895
- * may trigger a one-time macOS "control your browser" permission prompt;
896
- * declining it just falls back to opening a new tab. */
897
- function openChatUi(url: string, hostPort: string): void {
898
- const browsers: Array<{ proc: string; kind: "chromium" | "safari" }> = [
899
- { proc: "Google Chrome", kind: "chromium" },
900
- { proc: "Arc", kind: "chromium" },
901
- { proc: "Brave Browser", kind: "chromium" },
902
- { proc: "Microsoft Edge", kind: "chromium" },
903
- { proc: "Safari", kind: "safari" },
904
- ];
905
- void (async () => {
906
- for (const b of browsers) {
907
- try {
908
- const running = Bun.spawn(["pgrep", "-x", b.proc], { stdout: "ignore", stderr: "ignore" });
909
- if ((await running.exited) !== 0) continue; // not running → skip (and don't compile its tell)
910
- const script = focusTabScript(b.proc, b.kind, hostPort);
911
- const p = Bun.spawn(["osascript", "-e", script], { stdout: "pipe", stderr: "ignore", timeout: 8000 });
912
- const out = await new Response(p.stdout).text();
913
- if ((await p.exited) === 0 && out.trim() === "ok") return; // focused an existing tab
914
- } catch { /* try the next browser */ }
915
- }
916
- try { Bun.spawn(["open", url], { stdout: "ignore", stderr: "ignore" }); } catch { /* best-effort */ }
917
- })();
918
- }
919
-
920
- /** Server/runtime flags shared by every mode that loads a model
921
- * (serve, pi). Runtime overrides are installed here before the
922
- * generate/compiled-decode modules read them. */
923
- /** Resolve the decode ROUTE: a tier alias (--l1/--l2) sets the whole route,
924
- * and an explicit per-fork flag (--kv-quant/--fused-sdpa/…) overrides the
925
- * alias. Installs decode runtime options and returns the kv-quant mode. See
926
- * docs/design/unified-engine-frontier-plan.md + unified-engine-frontier-plan.md.
927
- * Each tier is a GUARANTEE about which reference you reproduce bit-for-bit:
928
- * --l1 bit-for-bit IDENTICAL to mlx-lm — drop-in replacement for mlx-lm
929
- * (bf16 KV; THE naked default since 2026-07-05)
930
- * --l2 bit-for-bit IDENTICAL to mlx-optiq — drop-in replacement for
931
- * mlx-optiq: quantized KV per kv_config.json + fused N-tiled SDPA for
932
- * L>1 + STOCK unfused L=1 decode (the optiq-golden composition,
933
- * scripts/regen.ts kvq + regen-mixed-kv-goldens.ts).
934
- * There is no --l3: output-changing experiments live in the Lab (env flags
935
- * with a bench + expiry) until one beats the L1 baseline in a paired A/B —
936
- * docs/design/unified-engine-frontier-plan.md §6-7. */
937
- function applyDecodeRoute(): { kvQuant?: "off" | "config" | number; turboQuant?: TurboQuantScheme } {
938
- const onOff = (name: string): boolean | null => {
939
- const v = opt(name); if (v == null) return null;
940
- if (v === "on" || v === "1" || v === "true") return true;
941
- if (v === "off" || v === "0" || v === "false") return false;
942
- console.error(`--${name} expects on|off (got "${v}")`); process.exit(1);
943
- };
944
- if (flag("l3")) {
945
- console.error(
946
- "--l3 was removed 2026-07-05: no output-changing kernel beat the L1 baseline " +
947
- "in a paired A/B, so the tier had no content. Use --l2 for the optiq route, " +
948
- "or the Lab (docs/design/unified-engine-frontier-plan.md) for experiments.",
949
- );
950
- process.exit(1);
951
- }
952
- type Preset = { kv: "off" | "config"; fusedSdpa: boolean; compiled: boolean; compiledAct: boolean };
953
- const TIERS: Record<string, Preset> = {
954
- // A tier is the GUARANTEE; within it we default to the FAST kernel that still
955
- // holds it, and expose the slow one as an opt-in. compiled-decode is BIT-EXACT
956
- // with uncompiled (compiled-decode.test: on==off) → free speed, so it's ON in
957
- // EVERY tier (--compiled-decode off is the slow, same-tier opt-in). compiled
958
- // ACTIVATIONS (geglu/swiglu) go through the SAME libmlx as mlx-lm's `@mx.compile`
959
- // → same kernel, bit-exact AND faster → ON in every tier (--compiled-activations
960
- // off is the slower same-parity opt-in). fused-sdpa matches OPTIQ bit-for-bit
961
- // (tier-a goldens, fused-sdpa.test) and is quantized-KV only → it's the L2 bridge,
962
- // a no-op on L1's bf16.
963
- l1: { kv: "off", fusedSdpa: false, compiled: true, compiledAct: true }, // = mlx-lm bit-for-bit (bf16)
964
- l2: { kv: "config", fusedSdpa: true, compiled: true, compiledAct: true }, // = mlx-optiq bit-for-bit
965
- };
966
- // NAKED DEFAULT = L1 (decided 2026-07-05): with no tier flag you get the
967
- // mlx-lm bit-for-bit route. The 2026-07-05 benchmark pass showed the L1
968
- // faithful kernel set matches mlx-lm 1.00× on every model while every
969
- // output-changing lever failed to beat it in a paired A/B (the losers were
970
- // deleted the same day — Phase 1 of unified-engine-frontier-plan.md).
971
- // Quantized KV costs 5–20% decode at ≤16k on BOTH stacks and pays only in
972
- // memory/context headroom → explicit opt-in (--kv-quant / --l2).
973
- const tier = flag("l2") ? "l2" : "l1";
974
- const p = TIERS[tier]!;
975
- const kv = opt("kv-quant"); // explicit --kv-quant overrides the tier
976
- const pick = (name: string, base: boolean | undefined): boolean | null => {
977
- const ex = onOff(name); return ex !== null ? ex : (base ?? null); // explicit flag wins, else tier preset
978
- };
979
- const set = (name: RuntimeKey, val: boolean | null, invert = false) => {
980
- if (val !== null)
981
- configureRuntime({ [name]: (invert ? !val : val) ? "1" : "0" });
982
- };
983
- // Explicit --kv-quant picks the composition its ORACLE uses: config (the
984
- // optiq mixed scheme) runs optiq's fused SDPA (the L2 golden composition);
985
- // uniform 4/8 runs UNFUSED (mlx-lm's quantized_scaled_dot_product_attention
986
- // — the bit-exact L1-eligible scheme). --fused-sdpa still overrides either.
987
- const fusedSdpaBase = kv === "config" ? true : kv != null ? false : p.fusedSdpa;
988
- set("MLX_BUN_COMPILED_DECODE", pick("compiled-decode", p.compiled));
989
- set("MLX_BUN_NO_FUSED_SDPA", pick("fused-sdpa", fusedSdpaBase), true); // inverted env
990
- // Compiled activations (the faithful geglu/swiglu — mlx-lm's @mx.compile kernel).
991
- // One fork drives both runtime options; qwen3/qwen3.5/universal compile unconditionally,
992
- // so this toggles gemma geglu + minicpm5 swiglu (the only unfused-capable sites).
993
- set("MLX_BUN_COMPILED_GEGLU", pick("compiled-activations", p.compiledAct));
994
- set("MLX_BUN_COMPILED_SWIGLU", pick("compiled-activations", p.compiledAct));
995
- if (kv === "off") return { kvQuant: "off" };
996
- if (kv === "config") return { kvQuant: "config" };
997
- if (kv) {
998
- // TurboQuant (docs/design/turboquant.md): a distinct axis from the
999
- // affine kvQuant above — dequantize-on-fetch, stock unfused ops.sdpa,
1000
- // no kvQuant value at all (mutually exclusive with kvBits/kvConfig).
1001
- let turboQuant: TurboQuantScheme | null;
1002
- try {
1003
- turboQuant = parseTurboQuantScheme(kv);
1004
- } catch (e) {
1005
- console.error((e as Error).message);
1006
- process.exit(1);
1007
- }
1008
- if (turboQuant) return { turboQuant };
1009
- const bits = Number(kv);
1010
- if (![4, 8].includes(bits)) { console.error(`--kv-quant expects config|off|4|8|turbo (got "${kv}")`); process.exit(1); }
1011
- return { kvQuant: bits };
1012
- }
1013
- return { kvQuant: p.kv };
1014
- }
1015
-
1016
- function serverRuntimeFlags(): { port: number; serverOptions: import("./server").ServerOptions } {
1017
- const onOff = (name: string): boolean | null => {
1018
- const v = opt(name);
1019
- if (v === null) return null;
1020
- if (v === "on" || v === "1" || v === "true") return true;
1021
- if (v === "off" || v === "0" || v === "false") return false;
1022
- console.error(`--${name} expects on|off (got "${v}")`);
1023
- process.exit(1);
1024
- };
1025
- const route = applyDecodeRoute(); // --l1/--l2 tier alias, with per-fork flags overriding
1026
- if (flag("force-wire")) configureRuntime({ MLX_BUN_FORCE_WIRE: "1" });
1027
- // Remote image_url/audio_url media fetches block private/loopback/link-
1028
- // local destinations by default (SSRF — src/media-fetch.ts); this is the
1029
- // LAN-hosts escape hatch.
1030
- if (flag("allow-private-media"))
1031
- configureRuntime({ MLX_BUN_ALLOW_PRIVATE_MEDIA: "1" });
1032
-
1033
- const serverOptions: import("./server").ServerOptions = {};
1034
- const budgetGB = Number(opt("memory-budget", "0"));
1035
- if (budgetGB > 0) serverOptions.memoryBudgetBytes = budgetGB * 1e9;
1036
- // Aggregate KV budget across batch rows (batching-perf-path P3 admission).
1037
- const kvBudgetGB = Number(opt("kv-budget", "0"));
1038
- if (kvBudgetGB > 0) serverOptions.kvBudgetBytes = kvBudgetGB * 1e9;
1039
- const pcRaw = opt("prompt-cache"); // null if absent; an explicit value (incl. 0) wins → `--prompt-cache 0` DISABLES the cache
1040
- if (pcRaw !== null) serverOptions.promptCacheBytes = Math.max(0, Number(pcRaw)) * 2 ** 30;
1041
- // Engine-child mode (docs/reference/server-config.md): listen on a unix socket
1042
- // instead of TCP — the isolated parent proxies here. Internal flag,
1043
- // spawned by --isolate; usable directly for socket-level integration.
1044
- const unixRaw = opt("unix");
1045
- if (unixRaw !== null) serverOptions.unixSocket = unixRaw;
1046
- // SSD cold tier (docs/design/kv-cache.md) — off unless a dir is given.
1047
- const ssdDir = opt("ssd-cache");
1048
- if (ssdDir !== null) {
1049
- serverOptions.ssdCacheDir = ssdDir;
1050
- const maxRaw = opt("ssd-cache-max");
1051
- if (maxRaw !== null) {
1052
- const maxGb = Number(maxRaw);
1053
- if (!Number.isFinite(maxGb) || maxGb < 0) {
1054
- console.error(`--ssd-cache-max expects a number >= 0 (got "${maxRaw}")`);
1055
- process.exit(1);
1056
- }
1057
- serverOptions.ssdCacheMaxBytes = maxGb === 0
1058
- ? Number.POSITIVE_INFINITY
1059
- : maxGb * 2 ** 30;
1060
- }
1061
- if (flag("ssd-cache-verify")) serverOptions.ssdCacheVerify = true;
1062
- const demoteRaw = opt("ssd-demote-idle");
1063
- if (demoteRaw !== null) serverOptions.ssdDemoteIdleSec = Math.max(0, Number(demoteRaw));
1064
- const checkpointRaw = opt("generation-checkpoint");
1065
- if (checkpointRaw !== null) {
1066
- const n = Number(checkpointRaw);
1067
- if (!Number.isInteger(n) || n < 1) {
1068
- console.error(`--generation-checkpoint expects an integer >= 1 (got "${checkpointRaw}")`);
1069
- process.exit(1);
1070
- }
1071
- serverOptions.generationCheckpointTokens = n;
1072
- }
1073
- } else {
1074
- // Refuse-loudly house rule: sub-flags of an absent tier must not vanish.
1075
- for (const f of ["ssd-cache-max", "ssd-demote-idle", "generation-checkpoint"] as const)
1076
- if (opt(f) !== null) console.warn(`--${f} has no effect without --ssd-cache — ignored`);
1077
- if (flag("ssd-cache-verify"))
1078
- console.warn("--ssd-cache-verify has no effect without --ssd-cache — ignored");
1079
- }
1080
- // --batch N: max concurrent requests batched through the mlx-lm-parity
1081
- // engine (N=1 pins the strict serial executor). --decode-concurrency is accepted for
1082
- // drop-in compatibility with mlx_lm.server, but the semantics differ: there
1083
- // it caps per-BatchGenerator decode parallelism (default 32); in mlx-bun it
1084
- // caps continuous batching (default 8; `--batch 1` pins the serial path —
1085
- // the unified engine is bit-parity with it and costs ~nothing at B=1).
1086
- const batchRaw = opt("batch") ?? opt("decode-concurrency");
1087
- if (batchRaw !== null) {
1088
- const n = Number(batchRaw);
1089
- if (!Number.isInteger(n) || n < 1) {
1090
- console.error(`--batch expects an integer >= 1 (got "${batchRaw}")`);
1091
- process.exit(1);
1092
- }
1093
- serverOptions.batch = n;
1094
- }
1095
- if (route.kvQuant !== undefined) serverOptions.kvQuant = route.kvQuant;
1096
- if (route.turboQuant !== undefined) serverOptions.turboQuant = route.turboQuant;
1097
- // --paged-kv (env: MLX_BUN_PAGED_KV=1): OPTIONAL vLLM-style block-pool KV
1098
- // storage, default off (docs/design/kv-cache.md). The state policy owns
1099
- // block pools and cache reuse; paging preserves the selected batch size.
1100
- if (flag("paged-kv") || runtimeValue("MLX_BUN_PAGED_KV") === "1") {
1101
- const bsRaw = opt("paged-kv-block-size");
1102
- serverOptions.pagedKv = bsRaw !== null ? { blockSize: Number(bsRaw) } : {};
1103
- }
1104
- // Bind loopback unless asked otherwise (mlx_lm.server parity); --host
1105
- // 0.0.0.0 is the explicit opt-in for LAN exposure. The chat-UI open and
1106
- // `mlx-bun pi` attach both go through localhost, so loopback-only is
1107
- // transparent to them.
1108
- serverOptions.hostname = opt("host", "127.0.0.1")!;
1109
- // Speech-to-text companion: explicit checkpoint + residency policy.
1110
- const whisperIdleRaw = opt("whisper-idle-unload");
1111
- if (whisperIdleRaw !== null) {
1112
- const n = Number(whisperIdleRaw);
1113
- if (!Number.isFinite(n) || n < 0) {
1114
- console.error(`--whisper-idle-unload expects seconds >= 0 (got "${whisperIdleRaw}")`);
1115
- process.exit(1);
1116
- }
1117
- serverOptions.whisperIdleUnloadSec = n;
1118
- }
1119
- if (flag("whisper-resident")) serverOptions.whisperResident = true;
1120
- // Server-wide default for the chat template's enable_thinking variable
1121
- // (CPM/MiniCPM5 and other hybrid-reasoning models). Unset ⇒ the model's
1122
- // own default (false for MiniCPM5). onOff accepts true|false|on|off|1|0.
1123
- const thinking = onOff("thinking");
1124
- if (thinking !== null) serverOptions.defaultThinking = thinking;
1125
- // Server-wide sampling defaults. Unset ⇒ the model's generation_config.json
1126
- // (MiniCPM5 ships 0.9 / 0.95 — its think recipe). A per-request field still
1127
- // overrides these; the browser chat sends none, so this is its only lever.
1128
- const numFlag = (name: string, lo: number, hi: number): number | null => {
1129
- const v = opt(name);
1130
- if (v === null) return null;
1131
- const n = Number(v);
1132
- if (!Number.isFinite(n) || n < lo || n > hi) {
1133
- console.error(`--${name} expects a number in [${lo}, ${hi}] (got "${v}")`);
1134
- process.exit(1);
1135
- }
1136
- return n;
1137
- };
1138
- // --temp is the mlx_lm.server spelling; explicit --temperature wins.
1139
- const temp = numFlag("temperature", 0, 5) ?? numFlag("temp", 0, 5);
1140
- if (temp !== null) serverOptions.defaultTemperature = temp;
1141
- const topP = numFlag("top-p", 0, 1);
1142
- if (topP !== null) serverOptions.defaultTopP = topP;
1143
- const topK = numFlag("top-k", 0, 1_000_000);
1144
- if (topK !== null) serverOptions.defaultTopK = topK;
1145
- // mlx_lm.server's --max-tokens: the completion cap when a request omits
1146
- // max_tokens. Our built-in default stays 65,536 (chat) / 512 (raw text
1147
- // completion); `--max-tokens 512` reproduces mlx-lm's behavior exactly.
1148
- const maxTok = numFlag("max-tokens", 1, 10_000_000);
1149
- if (maxTok !== null) serverOptions.defaultMaxTokens = Math.floor(maxTok);
1150
- // HLG sampling — piecewise tone curve on the logprobs (default off). Knobs in
1151
- // nats; the mid gain folds from --temperature. docs/archive/hlg-sampling.md.
1152
- if (onOff("hlg-sampling") === true) {
1153
- serverOptions.hlg = {
1154
- enabled: true,
1155
- width: numFlag("hlg-width", 0, 100) ?? 4,
1156
- shoulder: numFlag("hlg-shoulder", 0, 100) ?? 4,
1157
- toe: numFlag("hlg-toe", 0, 100) ?? 6,
1158
- pivotOffset: numFlag("hlg-pivot-offset", 0, 100) ?? 6,
1159
- pivot: "top",
1160
- };
1161
- }
1162
- return { port: Number(opt("port", "8080")), serverOptions };
1163
- }
1164
-
1165
- /** Direct Colibri runtime defaults shared by `serve` and the embedded `pi`
1166
- * server. Other model families ignore this load-only option block. */
1167
- function glm52RuntimeFlags(
1168
- serverOptions: import("./server").ServerOptions,
1169
- ): import("./model/factory").Glm52RuntimeOpenOptions {
1170
- const mtpRaw = opt("mtp");
1171
- let enableMtp: boolean | undefined;
1172
- if (mtpRaw !== null) {
1173
- if (["on", "1", "true"].includes(mtpRaw)) enableMtp = true;
1174
- else if (["off", "0", "false"].includes(mtpRaw)) enableMtp = false;
1175
- else {
1176
- console.error(`--mtp expects on|off (got "${mtpRaw}")`);
1177
- process.exit(1);
1178
- }
1179
- }
1180
- const contextRaw = opt("context-length");
1181
- let contextTokens: number | undefined;
1182
- if (contextRaw !== null) {
1183
- contextTokens = Number(contextRaw);
1184
- if (!Number.isSafeInteger(contextTokens) || contextTokens < 1) {
1185
- console.error(`--context-length expects an integer >= 1 (got "${contextRaw}")`);
1186
- process.exit(1);
1187
- }
1188
- }
1189
- return {
1190
- memoryBudgetBytes: serverOptions.memoryBudgetBytes,
1191
- batchSize: serverOptions.batch ?? 8,
1192
- maxGenerationTokens: serverOptions.defaultMaxTokens ?? 128,
1193
- ...(contextTokens !== undefined ? { contextTokens } : {}),
1194
- ...(enableMtp !== undefined ? { enableMtp } : {}),
1195
- };
1196
- }
1197
-
1198
- /** `--adapter <dir>` (alias `--adapter-path`, mlx_lm.server's spelling):
1199
- * mount a LoRA adapter at startup — the same machinery as POST /v1/adapters —
1200
- * and make it the default for requests that don't select one (a request's
1201
- * explicit `adapter`, including "none", still wins; hot-swap unchanged).
1202
- * This is the flag `mlx-bun train`'s completion message points at. Shared by
1203
- * `serve` and the `pi` session server. Exits on a bad adapter — better a
1204
- * loud startup failure than silently serving the base model. */
1205
- async function mountStartupAdapter(
1206
- ctx: import("./server").ServerContext,
1207
- serverOptions: import("./server").ServerOptions,
1208
- ): Promise<void> {
1209
- const adapterDir = opt("adapter") ?? opt("adapter-path");
1210
- if (!adapterDir) return;
1211
- const { step, style } = await import("./tui");
1212
- const sAdp = step("mounting adapter");
1213
- try {
1214
- const adapterId = adapterDir.replace(/\/+$/, "").split("/").pop()!;
1215
- const info = await ctx.adapters.mount(adapterId, adapterDir);
1216
- serverOptions.defaultAdapter = info.id;
1217
- sAdp.done(`adapter ${style.bold(info.id)} ${style.dim(`· ${info.mountedLayers} layers · default for requests (select others via \`adapter\`)`)}`);
1218
- } catch (e) {
1219
- sAdp.fail(`adapter mount failed: ${(e as Error).message}`);
1220
- process.exit(1);
1221
- }
1222
- }
1223
-
1224
- /** One-line summary of the active runtime levers for the ready card. */
1225
- function runtimeSummary(o: import("./server").ServerOptions): string {
1226
- const kv = o.turboQuant ? `turbo-k${o.turboQuant.kBits}v${o.turboQuant.vBits}`
1227
- : o.kvQuant === "off" ? "off" : typeof o.kvQuant === "number" ? `kv${o.kvQuant}` : "config";
1228
- const lever = (name: RuntimeKey, dflt: string) => runtimeValue(name) ?? dflt;
1229
- return `kv-quant ${kv} · compiled-decode ${lever("MLX_BUN_COMPILED_DECODE", "1") === "1" ? "on" : "off"}` +
1230
- (lever("MLX_BUN_COMPILED_GEGLU", "1") === "0" ? " · compiled-activations off" : "") +
1231
- (o.batch && o.batch > 1 ? ` · batch ${o.batch}` : "") +
1232
- (o.defaultThinking !== undefined ? ` · thinking ${o.defaultThinking ? "on" : "off"}` : "") +
1233
- (o.defaultTemperature !== undefined ? ` · temp ${o.defaultTemperature}` : "") +
1234
- (o.defaultTopP !== undefined ? ` · top-p ${o.defaultTopP}` : "") +
1235
- (o.defaultTopK !== undefined ? ` · top-k ${o.defaultTopK}` : "");
1236
- }
1237
-
1238
- /** Shared model resolution: explicit query wins; otherwise e4b (the default
1239
- * model everywhere) if it's downloaded, else the largest supported model that
1240
- * fits this machine. Downloads the recommended model first on a fresh install.
1241
- * A query that is a DIRECTORY containing config.json loads directly via
1242
- * scanSnapshot (no registry dependency) — the `--model <path>` bench/CI shape. */
1243
- async function resolveModelAuto(query: string | null): Promise<{ m: import("./registry").ModelRecord; picked: boolean }> {
1244
- if (query && existsSync(join(query, "config.json"))) {
1245
- const { scanSnapshot } = await import("./registry");
1246
- const dir = resolve(query);
1247
- // Recover the repo id from an HF-cache path (…/models--org--name/snapshots/<rev>);
1248
- // otherwise the directory basename is the display id.
1249
- const hf = /models--([^/]+)--([^/]+)\/snapshots\//.exec(dir);
1250
- const repoId = hf ? `${hf[1]}/${hf[2]}` : basename(dir);
1251
- const m = await scanSnapshot(dir, repoId);
1252
- if (!m) throw new Error(`${dir} has config.json but no weights (*.safetensors)`);
1253
- return { m, picked: false };
1254
- }
1255
- const reg = new Registry();
1256
- if (reg.list().length === 0) await reg.scan();
1257
- if (query) return { m: reg.resolve(query), picked: false };
1258
- const { recommendedRepoId } = await import("./fit");
1259
- let candidates = reg.list().filter((r) => isSupportedModelRecord(r.modelType, r.repoId));
1260
- if (candidates.length === 0) {
1261
- // First run: starter-model flow (PRODUCT_ROADMAP / PLAN Phase 16).
1262
- // Download the sub-GB starter first so the user is chatting quickly,
1263
- // then stream the recommended Gemma for this Mac in the background —
1264
- // it becomes the auto-pick next run.
1265
- const starterRepo = "mlx-community/MiniCPM5-1B-OptiQ-4bit";
1266
- const recRepo = recommendedRepoId();
1267
- const { step, style } = await import("./tui");
1268
- const { downloadModel } = await import("./download");
1269
- const s = step(`downloading starter model ${starterRepo}`);
1270
- await downloadModel(starterRepo, {
1271
- onProgress: (file, received, total) => {
1272
- const pct = total ? Math.floor((received / total) * 100) : 0;
1273
- s.update(`starter model ${starterRepo} — ${file} ${gb(received)} / ${gb(total)} (${pct}%)`);
1274
- },
1275
- });
1276
- s.done(`starter model ready ${style.dim(`· ${starterRepo}`)}`);
1277
- await reg.scan();
1278
- if (recRepo !== starterRepo) {
1279
- console.log(` ${style.dim(`downloading ${recRepo} (recommended for this Mac) in the background —`)}`);
1280
- console.log(` ${style.dim("it becomes the default next run · progress in the web app's Status view · resumable")}`);
1281
- downloadModel(recRepo)
1282
- .then(async () => {
1283
- await new Registry().scan();
1284
- console.log(` ${style.dim(`background download complete: ${recRepo} (used on next run)`)}`);
1285
- })
1286
- .catch(() => { /* resumable — the next run continues it */ });
1287
- }
1288
- candidates = reg.list().filter((r) => isSupportedModelRecord(r.modelType, r.repoId));
1289
- }
1290
- // No query: e4b by default; otherwise the largest model that still leaves
1291
- // the machine usable for other apps. Never auto-grab the 26B "dedicate the
1292
- // machine" model on a small-RAM Mac — that's an explicit --query choice —
1293
- // but allow it as a last resort if it's all the user has downloaded.
1294
- const { DEFAULT_REPO_ID, COEXIST_FRACTION, chooseAutoModel, thisMachine } = await import("./fit");
1295
- const machine = thisMachine();
1296
- const fitsFull = new Map<string, boolean>();
1297
- const fitsCoexist = new Map<string, boolean>();
1298
- for (const r of candidates) {
1299
- const config = await loadModelConfig(r.path);
1300
- fitsFull.set(r.repoId, fit(config, r.sizeBytes, 8192, machine, undefined, r.expertsBytes).fits);
1301
- fitsCoexist.set(
1302
- r.repoId,
1303
- fit(config, r.sizeBytes, 8192, machine, undefined, r.expertsBytes, machine.ramBytes * COEXIST_FRACTION).fits,
1304
- );
1305
- }
1306
- const chosen = chooseAutoModel(
1307
- candidates,
1308
- DEFAULT_REPO_ID,
1309
- (c) => fitsFull.get(c.repoId) ?? false,
1310
- (c) => fitsCoexist.get(c.repoId) ?? false,
1311
- );
1312
- if (chosen) return { m: chosen, picked: true };
1313
- console.error("no downloaded supported model fits this machine — pick one explicitly (mlx-bun ls)");
1314
- process.exit(1);
1315
- }
1316
-
1317
- function parseSize(s: string): number {
1318
- const m = /^([\d.]+)\s*(GB|MB|G|M)?$/i.exec(s.trim());
1319
- if (!m) throw new Error(`bad size: ${s}`);
1320
- const mult = /^m/i.test(m[2] ?? "G") ? 2 ** 20 : 2 ** 30;
1321
- return Number(m[1]) * mult;
1322
- }
1323
-
1324
- switch (cmd) {
1325
- case "get": {
1326
- let repoId = positional(0);
1327
- if (!repoId) {
1328
- console.error("usage: mlx-bun get <org/repo | substring> [--revision main]");
1329
- process.exit(1);
1330
- }
1331
- if (!repoId.includes("/")) {
1332
- // No org/name — treat it as a registry query over already-downloaded
1333
- // repos (re-get/refresh, e.g. `mlx-bun get 12B` after an upstream push).
1334
- try {
1335
- const regQ = new Registry();
1336
- if (regQ.list().length === 0) await regQ.scan();
1337
- repoId = regQ.resolve(repoId).repoId;
1338
- } catch (e) {
1339
- console.error((e as Error).message);
1340
- console.error(`usage: mlx-bun get <org/repo> [--revision main] — or a substring of a downloaded repo (try \`mlx-bun ls ${repoId}\`)`);
1341
- process.exit(1);
1342
- }
1343
- }
1344
- const { downloadModel } = await import("./download");
1345
- const { step, style } = await import("./tui");
1346
- const { join: joinPath, basename } = await import("node:path");
1347
- const { readdirSync, existsSync } = await import("node:fs");
1348
- const { DEFAULT_HUB, planRepoGc } = await import("./registry");
1349
- const repoDir = joinPath(DEFAULT_HUB, `models--${repoId.replaceAll("/", "--")}`);
1350
- const priorSnapshots = existsSync(joinPath(repoDir, "snapshots"))
1351
- ? readdirSync(joinPath(repoDir, "snapshots"))
1352
- : [];
1353
- const s = step(`downloading ${repoId}`);
1354
- const snap = await downloadModel(repoId, {
1355
- revision: opt("revision", "main")!,
1356
- onProgress: (file, received, total) => {
1357
- const pct = total ? Math.floor((received / total) * 100) : 0;
1358
- s.update(`${style.bold(repoId)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
1359
- },
1360
- });
1361
- s.done(`${style.bold(repoId)} ${style.dim("downloaded · verified")}`);
1362
- const sScan = step("updating registry");
1363
- const reg = new Registry();
1364
- await reg.scan();
1365
- sScan.done(`registry updated ${style.dim(`· ${snap}`)}`);
1366
- // Upstream pushed a new revision → this get created a NEW snapshots/<sha>
1367
- // dir; the old one stays on disk (that's how ~25 GB of dead blobs pile
1368
- // up). Say so, with what `gc` would get back.
1369
- if (priorSnapshots.length > 0 && !priorSnapshots.includes(basename(snap))) {
1370
- const plan = planRepoGc(repoDir);
1371
- const note = plan.skippedSnapshots.length > 0
1372
- ? " (a previous snapshot has files this revision lacks — gc will warn; --force to prune it)"
1373
- : "";
1374
- console.log(style.dim(
1375
- ` previous revision kept on disk — \`mlx-bun gc\` reclaims ~${gb(plan.reclaimBytes)}${note}`,
1376
- ));
1377
- }
1378
- break;
1379
- }
1380
-
1381
- case "scan": {
1382
- const { step } = await import("./tui");
1383
- const s = step("scanning the Hugging Face cache");
1384
- const reg = new Registry();
1385
- const n = await reg.scan();
1386
- s.done(`indexed ${n} model snapshot(s)`);
1387
- break;
1388
- }
1389
-
1390
- case "ls": {
1391
- const reg = new Registry();
1392
- if ((reg.list().length ?? 0) === 0) await reg.scan();
1393
- const maxSize = opt("max-size");
1394
- const allRevisions = flag("all-revisions");
1395
- const lsFilter = {
1396
- vision: flag("vision") ? true : undefined,
1397
- maxBytes: maxSize ? parseSize(maxSize) : undefined,
1398
- query: positional(0),
1399
- };
1400
- // One row per repo by default: the HF cache keeps a snapshots/<commit>
1401
- // dir per downloaded revision, and re-getting after an upstream push
1402
- // strands the old one — those are duplicates, not separate models.
1403
- // --all-revisions shows the per-snapshot truth (canonical marked *).
1404
- const models = allRevisions ? reg.list(lsFilter) : reg.listCanonical(lsFilter);
1405
- if (models.length === 0) {
1406
- console.log("no models match (try `mlx-bun scan`)");
1407
- break;
1408
- }
1409
- const { visionCapable } = await import("./registry");
1410
- const { supportTier } = await import("./model/support");
1411
- const canonicalPaths = new Set(reg.listCanonical(lsFilter).map((m) => m.path));
1412
- const capabilities = (m: (typeof models)[number]) => {
1413
- const tier = supportTier(m.modelType, m.repoId);
1414
- return [
1415
- tier ? `supported (${tier})` : `unsupported (${m.modelType})`,
1416
- visionCapable(m) ? "vision" : null,
1417
- m.hasToolTemplate ? "tools" : null,
1418
- m.hasKvConfig ? "kv-quant" : null,
1419
- ].filter(Boolean).join(" · ");
1420
- };
1421
- const { table, style, h1 } = await import("./tui");
1422
- h1("library");
1423
- console.log();
1424
- table(
1425
- [
1426
- { header: "model", paint: (c) => style.bold(c) },
1427
- ...(allRevisions ? [{ header: "revision" }] : []),
1428
- { header: "size", align: "right" as const },
1429
- { header: "params", align: "right" as const },
1430
- { header: "quant" },
1431
- { header: "license", paint: (c: string) => style.dim(c) },
1432
- { header: "capabilities", paint: (c: string) => style.dim(c) },
1433
- ],
1434
- models.map((m) => [
1435
- m.repoId,
1436
- ...(allRevisions
1437
- ? [`${(m.path.split("/snapshots/")[1] ?? "").slice(0, 12)}${canonicalPaths.has(m.path) ? " *" : ""}`]
1438
- : []),
1439
- gb(m.sizeBytes),
1440
- m.paramCount ? `${(m.paramCount / 1e9).toFixed(1)}B` : "?",
1441
- m.quantBits ? `${m.quantBits}-bit g${m.quantGroupSize}` : "full",
1442
- m.license ?? "?",
1443
- capabilities(m),
1444
- ]),
1445
- );
1446
- console.log();
1447
- if (allRevisions) {
1448
- console.log(style.dim(
1449
- ` ${models.length} snapshot(s) · * = canonical (refs/main) · superseded snapshots: \`mlx-bun gc\``,
1450
- ));
1451
- } else {
1452
- console.log(style.dim(` ${models.length} model(s) · mlx-bun fit <query> for a memory assessment`));
1453
- }
1454
- break;
1455
- }
1456
-
1457
- case "gc": {
1458
- const { planGc, executeGc, DEFAULT_HUB } = await import("./registry");
1459
- const { table, style, h1 } = await import("./tui");
1460
- const force = flag("force");
1461
- const plans = planGc(DEFAULT_HUB, { force }).filter(
1462
- (p) => p.pruneSnapshots.length || p.skippedSnapshots.length || p.deadBlobs.length,
1463
- );
1464
- h1("gc — superseded snapshots + dead blobs");
1465
- console.log();
1466
- if (plans.length === 0) {
1467
- console.log(" nothing to reclaim — every snapshot is referenced by refs/*");
1468
- break;
1469
- }
1470
- table(
1471
- [
1472
- { header: "repo", paint: (c) => style.bold(c) },
1473
- { header: "keep", align: "right" },
1474
- { header: "prune", align: "right" },
1475
- { header: "skip", align: "right" },
1476
- { header: "reclaims", align: "right" },
1477
- ],
1478
- plans.map((p) => [
1479
- p.repoId,
1480
- String(p.keepSnapshots.length),
1481
- String(p.pruneSnapshots.length),
1482
- String(p.skippedSnapshots.length),
1483
- gb(p.reclaimBytes),
1484
- ]),
1485
- );
1486
- console.log();
1487
- for (const p of plans) {
1488
- for (const s of p.skippedSnapshots) {
1489
- const rev = (s.path.split("/snapshots/")[1] ?? s.path).slice(0, 12);
1490
- console.log(style.accent(
1491
- ` WARNING ${p.repoId}@${rev}: superseded snapshot has files the canonical revision lacks — skipped.`,
1492
- ));
1493
- console.log(style.dim(` only copy of: ${s.extraFiles.join(", ")}`));
1494
- console.log(style.dim(" deleting it loses those files — rerun with --force if that's intended."));
1495
- }
1496
- }
1497
- const totalReclaim = plans.reduce((a, p) => a + p.reclaimBytes, 0);
1498
- const totalSnaps = plans.reduce((a, p) => a + p.pruneSnapshots.length, 0);
1499
- const totalBlobs = plans.reduce((a, p) => a + p.deadBlobs.length, 0);
1500
- console.log(
1501
- ` total: ${totalSnaps} snapshot(s), ${totalBlobs} blob(s), ${style.bold(gb(totalReclaim))}`,
1502
- );
1503
- if (flag("dry-run") || !flag("yes")) {
1504
- console.log(style.dim(" nothing deleted — rerun with --yes to delete (destructive)."));
1505
- break;
1506
- }
1507
- const res = executeGc(plans);
1508
- console.log(
1509
- ` deleted ${res.snapshots} snapshot(s) + ${res.blobs} blob(s) — reclaimed ${style.bold(gb(res.reclaimedBytes))}`,
1510
- );
1511
- const reg = new Registry();
1512
- await reg.scan(); // reap deleted snapshots from the registry
1513
- break;
1514
- }
1515
-
1516
- case "fit": {
1517
- const query = positional(0);
1518
- if (!query) { console.error("usage: mlx-bun fit <query> [--ctx N] [--kv-quant 4|8|config] [--skus]"); process.exit(1); }
1519
- const reg = new Registry();
1520
- if (reg.list().length === 0) await reg.scan();
1521
- const m = reg.resolve(query);
1522
- const config = await loadModelConfig(m.path);
1523
- const { box, table, style, h1, gradient } = await import("./tui");
1524
- const glm = isGlm52Config(config);
1525
- const ctx = Number(opt("ctx", glm ? "4096" : "8192"));
1526
- if (glm) {
1527
- const machine = thisMachine();
1528
- let plan;
1529
- try {
1530
- plan = await planGlm52MemoryForArtifact(m.path, {
1531
- machineBytes: machine.ramBytes,
1532
- processLimitBytes: Math.min(
1533
- GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES,
1534
- machine.ramBytes,
1535
- ),
1536
- contextTokens: ctx,
1537
- maxGenerationTokens: Math.min(
1538
- ctx,
1539
- GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS,
1540
- ),
1541
- batchSize: 1,
1542
- enableMtp: true,
1543
- });
1544
- } catch (error) {
1545
- console.error(
1546
- `GLM-5.2 resource plan refused: ` +
1547
- `${error instanceof Error ? error.message : String(error)}`,
1548
- );
1549
- process.exitCode = 1;
1550
- break;
1551
- }
1552
- const gib = (bytes: number): string => `${(bytes / 2 ** 30).toFixed(2)} GiB`;
1553
- const li = plan.lineItems;
1554
- const kvBytes = li.targetKvBytes + li.mtpKvBytes;
1555
- const otherBytes = plan.plannedProcessBytes - li.residentWeightsBytes -
1556
- li.mainExpertSlabBytes - li.mtpExpertSlabBytes - kvBytes;
1557
- h1("will it fit?");
1558
- console.log(` ${style.bold(m.repoId)} ${style.dim(`@ ${ctx.toLocaleString()} context · streamed GLM plan`)}`);
1559
- console.log();
1560
- box([
1561
- `artifact on disk ${gib(m.sizeBytes).padStart(10)} ${style.dim("(streamed, not resident)")}`,
1562
- `resident weights ${gib(li.residentWeightsBytes).padStart(10)}`,
1563
- `main expert slab ${gib(li.mainExpertSlabBytes).padStart(10)}`,
1564
- `MTP expert slab ${gib(li.mtpExpertSlabBytes).padStart(10)}`,
1565
- `target + MTP KV ${gib(kvBytes).padStart(10)}`,
1566
- `runtime reserves ${gib(otherBytes).padStart(10)}`,
1567
- `process plan ${gib(plan.plannedProcessBytes).padStart(10)} ${style.dim(`of ${gib(plan.processLimitBytes)}`)} ${style.green(style.bold("FITS"))}`,
1568
- `macOS headroom ${gib(plan.machineHeadroomBytes).padStart(10)}`,
1569
- "",
1570
- `max safe context ${style.bold(plan.contextTokens.toLocaleString())} tokens`,
1571
- `max generation ${style.bold(plan.maxGenerationTokens.toLocaleString())} tokens`,
1572
- `measured warm ${gradient(`${GLM52_G5_MEASURED_WARM_DECODE_TPS.toFixed(3)} tok/s`)} ${style.dim("(quality-preserving, MTP on)")}`,
1573
- `direct oracle ${GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS.toFixed(2)} tok/s`,
1574
- `aspirational ${GLM52_G5_ASPIRATIONAL_DECODE_TPS.toFixed(1)} tok/s ${style.dim("(not a fit prediction or release gate)")}`,
1575
- ]);
1576
- if (flag("skus")) {
1577
- console.log(style.dim(
1578
- " GLM-5.2's direct-container preset is validated on the M1 Max 32 GB; " +
1579
- "the generic resident-weight SKU projection does not apply.",
1580
- ));
1581
- }
1582
- console.log();
1583
- break;
1584
- }
1585
- // --kv-quant mirrors serve: bill the quantized cache's true bytes so
1586
- // the reported window matches what serving with the same flag admits.
1587
- const kvQuantOpt = opt("kv-quant");
1588
- let fitKvScheme: FitKvScheme | undefined;
1589
- if (kvQuantOpt === "4" || kvQuantOpt === "8") {
1590
- fitKvScheme = resolveKvScheme({ override: Number(kvQuantOpt) }).fitOptions;
1591
- } else if (kvQuantOpt === "config") {
1592
- if (!config.kvQuant?.length) {
1593
- console.error(`${m.repoId} ships no per-layer kv-quant config — use --kv-quant 4|8`);
1594
- process.exit(1);
1595
- }
1596
- fitKvScheme = resolveKvScheme({
1597
- override: "config",
1598
- config: config.kvQuant,
1599
- missingConfig: "error",
1600
- }).fitOptions;
1601
- } else if (kvQuantOpt != null && kvQuantOpt !== "off") {
1602
- console.error(`--kv-quant ${kvQuantOpt}: expected 4, 8, config, or off`);
1603
- process.exit(1);
1604
- }
1605
- const r = fit(config, m.sizeBytes, ctx, thisMachine(), undefined, m.expertsBytes, undefined, fitKvScheme);
1606
- h1("will it fit?");
1607
- const kvNote = fitKvScheme
1608
- ? style.dim(` · kv-quant ${fitKvScheme.kvBits ? `${fitKvScheme.kvBits}-bit` : "config"}`)
1609
- : "";
1610
- console.log(` ${style.bold(m.repoId)} ${style.dim(`@ ${ctx.toLocaleString()} context · this machine`)}${kvNote}`);
1611
- console.log();
1612
- const expertsNote = config.text.enableMoeBlock && m.expertsBytes > 0
1613
- ? style.dim(` (experts ${gb(m.expertsBytes)}; top ${config.text.topKExperts}/${config.text.numExperts} read per token)`)
1614
- : "";
1615
- box([
1616
- `weights ${gb(r.weightsBytes).padStart(9)}${expertsNote}`,
1617
- ...(m.sidecarBytes > 0
1618
- ? [style.dim(` + vision sidecar ${gb(m.sidecarBytes)} (bf16, loads only for vision)`)] : []),
1619
- `kv cache ${gb(r.kvBytes).padStart(9)}`,
1620
- `transient ${gb(r.transientBytes).padStart(9)}`,
1621
- `total ${gb(r.totalBytes).padStart(9)} ${style.dim(`of ${gb(r.usableBytes)} usable`)} ${r.fits ? style.green(style.bold("FITS")) : style.bold("DOES NOT FIT")}`,
1622
- "",
1623
- `max safe context ${style.bold(r.maxSafeContext.toLocaleString())} tokens`,
1624
- `predicted decode ${gradient(`${r.predictedDecodeTps.toFixed(1)} tok/s`)}`,
1625
- ]);
1626
- if (flag("skus")) {
1627
- h1("apple silicon matrix");
1628
- console.log();
1629
- table(
1630
- [
1631
- { header: "chip" }, { header: "ram", align: "right" },
1632
- { header: "fits", paint: (c) => (c.includes("fits") ? style.green(c) : style.dim(c)) },
1633
- { header: "max context", align: "right" }, { header: "decode", align: "right" },
1634
- ],
1635
- skuMatrix(config, m.sizeBytes, ctx, m.expertsBytes, fitKvScheme).map((row) => [
1636
- row.sku, `${row.ramGB} GB`, row.fits ? "fits" : "—",
1637
- row.fits ? row.maxContext.toLocaleString() : "—",
1638
- row.fits ? `~${row.decodeTps.toFixed(0)} tok/s` : "—",
1639
- ]),
1640
- );
1641
- }
1642
- console.log();
1643
- break;
1644
- }
1645
-
1646
- case "serve": {
1647
- const { banner, step, box, style } = await import("./tui");
1648
- banner(pkg.version);
1649
- if (noSubcommand)
1650
- console.log(style.dim(" no command given — starting the server · mlx-bun --help for all commands"));
1651
- const rt = serverRuntimeFlags();
1652
- // Friendly collision check before loading gigabytes of weights.
1653
- if (!rt.serverOptions.unixSocket) {
1654
- const { probeServer } = await import("./harness-pi");
1655
- const running = await probeServer(`http://localhost:${rt.port}/v1`);
1656
- if (running) {
1657
- console.error(`port ${rt.port} is already serving ${running.map((m) => m.id).join(", ")}.`);
1658
- console.error("reuse it (mlx-bun pi attaches automatically), stop it, or pick --port <other>.");
1659
- console.error("NOTE: a second server is a second model in memory — check `mlx-bun fit` first.");
1660
- process.exit(1);
1661
- }
1662
- }
1663
- // Runtime isolation (--isolate, docs/reference/server-config.md): THIS process
1664
- // becomes a pure reactor — it binds the TCP port and reverse-proxies to
1665
- // an engine CHILD (the full server on a unix socket, spawned from the
1666
- // same argv minus parent-only flags). Zero MLX calls in the parent: the
1667
- // UI stays instant under any GPU load, and an engine crash respawns
1668
- // without taking the session down.
1669
- if (!flag("isolate") && opt("model-pool") !== null)
1670
- console.warn("--model-pool has no effect without --isolate (child-per-model pool) — ignored");
1671
- if (flag("isolate")) {
1672
- const { startProxyServer, engineArgvForModel, defaultSocketPath } = await import("./serve/isolate");
1673
- const sock = defaultSocketPath();
1674
- // Re-spawn ourselves: compiled binary = execPath alone (argv[1] is a
1675
- // $bunfs virtual path); dev = bun + the script path.
1676
- const selfArgv = process.argv[1] && !process.argv[1].includes("$bunfs")
1677
- ? [process.execPath, process.argv[1]]
1678
- : [process.execPath];
1679
- const rawArgs = process.argv.slice(2);
1680
- // Resolve the DEFAULT model in the parent (registry lookup — no
1681
- // weights load) so the pool has its key and the child gets a pinned
1682
- // --model path instead of re-running fuzzy resolution.
1683
- const { m: m0 } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
1684
- // STRICT resolver for the pool (P2): only EXACT /v1/models ids
1685
- // switch models. Fuzzy matching here would hijack drop-in clients
1686
- // that send model:"gpt-4"-style strings (which must keep today's
1687
- // ignored-field semantics and ride the default engine).
1688
- const { Registry } = await import("./registry");
1689
- const poolResolve = (query: string): { repoId: string; path: string } | null => {
1690
- if (query === m0.repoId) return { repoId: m0.repoId, path: m0.path };
1691
- try {
1692
- const reg = new Registry();
1693
- try {
1694
- const hit = reg.listCanonical().find((r) => r.repoId === query);
1695
- return hit ? { repoId: hit.repoId, path: hit.path } : null;
1696
- } finally {
1697
- reg.close();
1698
- }
1699
- } catch {
1700
- return null;
1701
- }
1702
- };
1703
- const poolMaxRaw = opt("model-pool");
1704
- const poolMax = Math.max(1, Number(poolMaxRaw ?? 1) || 1);
1705
- const { engine, close } = startProxyServer({
1706
- port: rt.port,
1707
- ...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
1708
- engine: {
1709
- argv: [...selfArgv, ...engineArgvForModel(rawArgs, sock, m0.path)],
1710
- socketPath: sock,
1711
- },
1712
- pool: {
1713
- rawArgs, selfArgv, poolMax,
1714
- resolve: poolResolve,
1715
- defaultKey: m0.repoId,
1716
- },
1717
- });
1718
- const shownHost0 = rt.serverOptions.hostname ?? "localhost";
1719
- console.log(style.dim(` isolated mode: engine child starting (socket ${sock})`));
1720
- console.log(` ${style.green("●")} ${style.bold("proxy up")} http://${shownHost0}:${rt.port} ${style.dim(`· ${m0.repoId} loading behind it · model pool ${poolMax}`)}`);
1721
- const onExit = () => { void close().finally(() => process.exit(0)); };
1722
- process.on("SIGINT", onExit);
1723
- process.on("SIGTERM", onExit);
1724
- await engine.ready;
1725
- console.log(` ${style.green("●")} ${style.bold("engine ready")} ${style.dim(`pid ${engine.pid}`)}`);
1726
- await new Promise(() => {}); // serve until killed
1727
- }
1728
- // --model <path-or-query> is the explicit override (mlx_lm.server's
1729
- // spelling — previously accepted-but-IGNORED here, which silently
1730
- // auto-picked the wrong model). A directory with config.json loads
1731
- // straight from that path; anything else resolves like the positional
1732
- // fuzzy query. Precedence: --model > positional > --query > auto-pick.
1733
- const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
1734
- if (m.modelType === "whisper") {
1735
- // Transcription-only server: no chat model, weights paged in on demand.
1736
- const sNative0 = step("native runtime");
1737
- await ensureNative(sNative0);
1738
- sNative0.done("native runtime ready");
1739
- const { createTranscriptionServer } = await import("./serve/transcription-server");
1740
- const { server, service } = await createTranscriptionServer({
1741
- modelDir: m.path, modelId: m.repoId, port: rt.port,
1742
- ...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
1743
- ...(rt.serverOptions.unixSocket ? { unixSocket: rt.serverOptions.unixSocket } : {}),
1744
- idleUnloadSec: rt.serverOptions.whisperIdleUnloadSec,
1745
- resident: rt.serverOptions.whisperResident,
1746
- preload: flag("preload"),
1747
- });
1748
- const where = rt.serverOptions.unixSocket ?? `http://${rt.serverOptions.hostname === "0.0.0.0" ? "localhost" : rt.serverOptions.hostname}:${server.port}`;
1749
- box([
1750
- `${style.bold(m.repoId)} ${style.dim("· transcription-only server")}`,
1751
- `POST ${where}/v1/audio/transcriptions ${style.dim(`· ${service.stats.idle_unload_sec === null ? "always resident" : service.idleUnloadSec === 0 ? "released after every take" : `idle unload ${service.idleUnloadSec}s`}${service.resident ? " · loaded" : " · loads on first request"}`)}`,
1752
- ]);
1753
- const stop = () => { service.dispose(); server.stop(true); process.exit(0); };
1754
- process.once("SIGINT", stop);
1755
- process.once("SIGTERM", stop);
1756
- break;
1757
- }
1758
- const whisperQuery = opt("whisper-model");
1759
- if (whisperQuery !== null) {
1760
- const { m: wm } = await resolveModelAuto(whisperQuery);
1761
- if (wm.modelType !== "whisper") {
1762
- console.error(`--whisper-model ${whisperQuery} resolved to ${wm.repoId} (model_type ${wm.modelType}), not a Whisper checkpoint`);
1763
- process.exit(1);
1764
- }
1765
- rt.serverOptions.whisperModelDir = wm.path;
1766
- rt.serverOptions.whisperModelId = wm.repoId;
1767
- }
1768
- const sFit = step("assessing fit");
1769
- const config = await loadModelConfig(m.path);
1770
- const report = fit(config, m.sizeBytes, 8192, undefined, undefined, m.expertsBytes);
1771
- sFit.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)} · ~${report.predictedDecodeTps.toFixed(0)} tok/s predicted`)}${picked ? style.dim(" · auto-picked (override: mlx-bun serve <query>)") : ""}`);
1772
- const sNative = step("native runtime");
1773
- await ensureNative(sNative);
1774
- sNative.done("native runtime ready");
1775
- const { createServer, loadContext, shutdownServer } = await import("./server");
1776
- if (flag("expert-offload")) {
1777
- if (m.expertsBytes === 0) {
1778
- console.error(style.dim("--expert-offload ignored: this model has no experts (dense)"));
1779
- } else {
1780
- const sOff = step("expert offload");
1781
- const { ensureOffloadFile } = await import("./expert-offload-build");
1782
- const { activateExpertOffload } = await import("./expert-offload");
1783
- // builds <model>/.mlx-bun-offload on first use (~one-time), then reuses
1784
- const dir = await ensureOffloadFile(m.path, (msg) => sOff.update(msg));
1785
- activateExpertOffload(dir);
1786
- sOff.done(`experts mmap'd ${style.dim(dir)} ${style.dim("· phys_footprint ≈ core")}`);
1787
- }
1788
- }
1789
- // Speculative decoding (mlx_lm.server parity): --draft-model <path|query>
1790
- // resolves like the main model; --num-draft-tokens defaults to 3 (the
1791
- // server's default upstream — mlx_lm.generate's is 2). Qualified providers
1792
- // use shared execution; ordinary requests retain their own placement.
1793
- const draftQuery = opt("draft-model");
1794
- let draftModelDir: string | undefined;
1795
- if (draftQuery !== null) {
1796
- const { m: dm } = await resolveModelAuto(draftQuery);
1797
- draftModelDir = dm.path;
1798
- }
1799
- const numDraftRaw = opt("num-draft-tokens");
1800
- const numDraftTokens = numDraftRaw !== null ? Number(numDraftRaw) : undefined;
1801
- if (numDraftTokens !== undefined && (!Number.isInteger(numDraftTokens) || numDraftTokens < 1)) {
1802
- console.error(`--num-draft-tokens expects an integer >= 1 (got "${numDraftRaw}")`);
1803
- process.exit(1);
1804
- }
1805
- const draftKindRaw = opt("draft-kind");
1806
- const draftKind = (draftKindRaw ?? undefined) as
1807
- | "dspark" | "deepspec" | "assistant" | "two-model" | "ngram" | "mtp" | undefined;
1808
- if (draftKind !== undefined && !["dspark", "deepspec", "assistant", "two-model", "ngram", "mtp"].includes(draftKind)) {
1809
- console.error(`--draft-kind expects two-model|assistant|dspark|deepspec|mtp|ngram (got "${draftKindRaw}")`);
1810
- process.exit(1);
1811
- }
1812
- // Prompt-lookup window bounds (ngram kind only; validated here so a typo
1813
- // fails the launch, not the first request).
1814
- const ngramInt = (name: string): number | undefined => {
1815
- const raw = opt(name);
1816
- if (raw === null) return undefined;
1817
- const v = Number(raw);
1818
- if (!Number.isInteger(v) || v < 1) {
1819
- console.error(`--${name} expects an integer >= 1 (got "${raw}")`);
1820
- process.exit(1);
1821
- }
1822
- return v;
1823
- };
1824
- const ngramMax = ngramInt("ngram-max");
1825
- const ngramMin = ngramInt("ngram-min");
1826
- if (ngramMax !== undefined && ngramMin !== undefined && ngramMin > ngramMax) {
1827
- console.error(`--ngram-min (${ngramMin}) must be <= --ngram-max (${ngramMax})`);
1828
- process.exit(1);
1829
- }
1830
- if ((ngramMax !== undefined || ngramMin !== undefined) && draftKind !== "ngram")
1831
- console.warn("--ngram-max/--ngram-min only apply with --draft-kind ngram — ignored");
1832
- const sLoad = step("loading weights");
1833
- const t0 = performance.now();
1834
- const ctx = await loadContext(m.path, m.repoId, {
1835
- memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
1836
- glm: glm52RuntimeFlags(rt.serverOptions),
1837
- // ngram is model-free; mtp resolves the bundled <model>/mtp/ companion
1838
- // in loadContext when no --draft-model is given (cli.md contract —
1839
- // this gate silently dropped that case until 2026-08-20).
1840
- ...(draftModelDir || draftKind === "ngram" || draftKind === "mtp"
1841
- ? {
1842
- ...(draftModelDir ? { draftModelDir } : {}),
1843
- numDraftTokens,
1844
- ...(draftKind ? { draftKind } : {}),
1845
- ...(ngramMax !== undefined ? { ngramMax } : {}),
1846
- ...(ngramMin !== undefined ? { ngramMin } : {}),
1847
- }
1848
- : {}),
1849
- });
1850
- sLoad.done(`weights loaded ${style.dim(`in ${(performance.now() - t0).toFixed(0)} ms`)}${draftModelDir ? style.dim(` · draft: ${draftModelDir.split("/").filter(Boolean).at(-1)}`) : draftKind === "ngram" ? style.dim(" · draft: ngram (prompt lookup)") : ""}`);
1851
- await mountStartupAdapter(ctx, rt.serverOptions);
1852
- const server = createServer(ctx, rt.port, { ...rt.serverOptions, owner: "serve" });
1853
- let shutdownStarted = false;
1854
- const onServeSignal = (signal: "SIGINT" | "SIGTERM"): void => {
1855
- if (shutdownStarted) return;
1856
- shutdownStarted = true;
1857
- const rawTimeout = Number(runtimeValue("MLX_BUN_SHUTDOWN_TIMEOUT_MS"));
1858
- const timeoutMs = Number.isFinite(rawTimeout) && rawTimeout > 0 ? rawTimeout : 120_000;
1859
- console.log(`\n[server] ${signal}: flushing prompt-cache durability`);
1860
- void shutdownServer(server, timeoutMs).then((result) => {
1861
- if (!result.durability.durable) {
1862
- const d = result.durability;
1863
- console.warn(
1864
- `[server] cache flush incomplete after ${Math.round(d.elapsedMs)} ms: ` +
1865
- `${d.pendingSnapshots} snapshots, ${d.pendingSpills} spills, ` +
1866
- `${d.droppedSpills} dropped, ${d.failedSpills} failed`,
1867
- );
1868
- }
1869
- process.exit(0);
1870
- }).catch((error) => {
1871
- console.warn(`[server] shutdown flush failed: ${(error as Error).message}`);
1872
- process.exit(0);
1873
- });
1874
- };
1875
- process.once("SIGINT", () => onServeSignal("SIGINT"));
1876
- process.once("SIGTERM", () => onServeSignal("SIGTERM"));
1877
- const shownHost = rt.serverOptions.hostname ?? "localhost";
1878
- console.log();
1879
- box([
1880
- `${style.green("●")} ${style.bold("serving")} ${m.repoId}`,
1881
- "",
1882
- `API ${style.url(`http://${shownHost}:${server.port}/v1`)} ${style.dim("(OpenAI · Anthropic · Responses)")}`,
1883
- `App ${style.url(`http://${shownHost}:${server.port}/`)} ${style.dim("(chat · quantize · finetune · dataset · status)")}`,
1884
- ...(rt.serverOptions.memoryBudgetBytes
1885
- ? [`Mem ${gb(rt.serverOptions.memoryBudgetBytes)} budget ${style.dim("(admission control on)")}`] : []),
1886
- `Perf ${style.dim(runtimeSummary(rt.serverOptions))}`,
1887
- "",
1888
- style.dim("agent session: mlx-bun pi stop: Ctrl+C"),
1889
- ]);
1890
- // Open the chat UI for interactive runs (--no-open, or a non-TTY such as
1891
- // a piped/headless/benchmark run, skips it). 0.0.0.0/:: bind → localhost
1892
- // for the browser. Reuses an existing tab on this host:port when possible
1893
- // (see openChatUi); the URL is printed above either way.
1894
- if (!flag("no-open") && process.stdout.isTTY) {
1895
- const browserHost = shownHost === "0.0.0.0" || shownHost === "::" ? "localhost" : shownHost;
1896
- openChatUi(`http://${browserHost}:${server.port}/#/chat`, `${browserHost}:${server.port}`);
1897
- }
1898
- break;
1899
- }
1900
-
1901
- case "gen":
1902
- case "generate": {
1903
- // RAW generate — a direct one-shot entry point (no server, no chat UI). The
1904
- // three entry points (raw / OpenAI API / chat UI) differ only in how params
1905
- // are POPULATED; here every param is explicit. Decode-path levers mirror
1906
- // serve so you can pin the route (mlx-lm compat = --l1).
1907
- // --l1/--l2 + per-fork overrides set the decode runtime options; the
1908
- // returned KV scheme is applied below so `generate --l2` runs the same
1909
- // quantized-KV route (and produces the same tokens) as `serve --l2`.
1910
- const route = applyDecodeRoute();
1911
-
1912
- const prompt = opt("prompt") ?? positional(1);
1913
- if (!prompt) {
1914
- console.error('usage: mlx-bun generate [query] --prompt "…" [--raw] [--max-tokens N]');
1915
- console.error(' sampling: --temperature N (alias --temp) --top-p N --top-k N --seed N');
1916
- console.error(' decode path: --l1 (= mlx-lm compat, the default) / --l2, or per-fork flags');
1917
- process.exit(1);
1918
- }
1919
- const { loadTaskModel, generateText } = await import("./eval/runner");
1920
- const maxTokens = Number(opt("max-tokens", "256"));
1921
- const tm = await loadTaskModel(
1922
- positional(0) ?? opt("query") ?? "",
1923
- undefined,
1924
- { maxGenerationTokens: maxTokens, enableMtp: false },
1925
- );
1926
- // Resolve the tier/--kv-quant KV scheme exactly the way serve does
1927
- // (server.ts kvScheme): "config" = the model's kv_config.json (absent →
1928
- // bf16, same silent fallback as serve), N = uniform bits from decode
1929
- // start. Unset (no tier, no --kv-quant) keeps generate's historical
1930
- // default: bf16 bit-exact greedy (generateText's parity path).
1931
- const kvScheme = route.turboQuant || route.kvQuant !== undefined
1932
- ? resolveKvScheme({
1933
- override: route.kvQuant,
1934
- turboQuant: route.turboQuant,
1935
- config: tm.config.kvQuant,
1936
- }).generationOptions
1937
- : undefined;
1938
- const num = (n: string): number | undefined => { const v = opt(n); return v == null ? undefined : Number(v); };
1939
- const text = await generateText(tm, prompt, {
1940
- maxTokens,
1941
- useChat: !flag("raw"),
1942
- ...(kvScheme !== undefined ? { kvScheme } : {}),
1943
- sampler: { temperature: num("temperature") ?? num("temp"), topP: num("top-p"), topK: num("top-k"), seed: num("seed") },
1944
- });
1945
- process.stdout.write(text.endsWith("\n") ? text : text + "\n");
1946
- break;
1947
- }
1948
-
1949
- case "embed": {
1950
- // Raw text embeddings — one-shot, no server. Text from --text, or one per
1951
- // line on stdin. Uses the same embedPooled path the parity test verifies.
1952
- const { Weights } = await import("./weights");
1953
- const { createModel } = await import("./model/factory");
1954
- const { loadTokenizer } = await import("./tokenizer");
1955
- const { embedMany, isEmbeddingModel } = await import("./embed");
1956
-
1957
- let texts: string[];
1958
- const oneText = opt("text") ?? positional(1);
1959
- if (oneText != null) {
1960
- texts = [oneText];
1961
- } else if (!process.stdin.isTTY) {
1962
- texts = (await Bun.stdin.text()).split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
1963
- } else {
1964
- texts = [];
1965
- }
1966
- if (texts.length === 0) {
1967
- console.error('usage: mlx-bun embed [query] --text "…" (or pipe text, one per line)');
1968
- console.error(' --instruct "…" query instruction · --json OpenAI-style output');
1969
- process.exit(1);
1970
- }
1971
-
1972
- // Model resolution: an explicit query resolves normally; the no-query
1973
- // path auto-picks the first DOWNLOADED embedding model (plain Qwen3 —
1974
- // the record-level equivalent of isEmbeddingModel, which the factory
1975
- // maps to Qwen3Model). Never the chat default, and never the fresh-
1976
- // install starter download — a chat model can't embed, so downloading
1977
- // one here only to fail the isEmbeddingModel check would be a trap.
1978
- const embedQuery = positional(0) ?? opt("query");
1979
- let m: import("./registry").ModelRecord;
1980
- if (embedQuery) {
1981
- ({ m } = await resolveModelAuto(embedQuery));
1982
- } else {
1983
- const reg = new Registry();
1984
- if (reg.list().length === 0) await reg.scan();
1985
- const embedder = reg.list().find((r) => r.modelType === "qwen3");
1986
- if (!embedder) {
1987
- console.error("no embedding model downloaded — try: mlx-bun get mlx-community/Qwen3-Embedding-4B-4bit-DWQ");
1988
- process.exit(1);
1989
- }
1990
- m = embedder;
1991
- }
1992
- const config = await loadModelConfig(m.path);
1993
- const model = createModel(await Weights.open(m.path), config);
1994
- if (!isEmbeddingModel(model)) {
1995
- console.error(`"${m.repoId}" is not an embedding model (need plain Qwen3, e.g. Qwen3-Embedding).`);
1996
- process.exit(1);
1997
- }
1998
- const tok = await loadTokenizer(m.path);
1999
- const results = embedMany(model, tok, texts, opt("instruct") ?? undefined);
2000
-
2001
- if (flag("json")) {
2002
- let total = 0;
2003
- const data = results.map((r, index) => { total += r.tokens; return { object: "embedding", index, embedding: Array.from(r.vector) }; });
2004
- process.stdout.write(JSON.stringify({
2005
- object: "list", data, model: m.repoId,
2006
- usage: { prompt_tokens: total, total_tokens: total },
2007
- }) + "\n");
2008
- } else {
2009
- for (const r of results) process.stdout.write(JSON.stringify(Array.from(r.vector)) + "\n");
2010
- }
2011
- break;
2012
- }
2013
-
2014
- case "transcribe": {
2015
- // Speech-to-text, one-shot, no server: audio file → Whisper → text.
2016
- const { openWhisperModel } = await import("./model/factory");
2017
- const { loadWhisperTokenizer } = await import("./audio/whisper-tokenizer");
2018
- const { WhisperTranscriber } = await import("./audio/whisper-transcribe");
2019
- const { formatTranscription, TRANSCRIPTION_FORMATS } = await import("./audio/whisper-format");
2020
- const { TranscriptionService } = await import("./serve/transcription-service");
2021
- const { formatTimestamp } = await import("./audio/whisper-format");
2022
-
2023
- const audioPath = positional(0);
2024
- if (!audioPath) {
2025
- console.error("usage: mlx-bun transcribe <audio-file> [query] [--language en] [--format text|json|verbose_json|srt|vtt]");
2026
- process.exit(1);
2027
- }
2028
- const format = (opt("format") ?? "text") as import("./audio/whisper-format").TranscriptionFormat;
2029
- if (!TRANSCRIPTION_FORMATS.includes(format)) {
2030
- console.error(`--format must be one of ${TRANSCRIPTION_FORMATS.join(", ")}`);
2031
- process.exit(1);
2032
- }
2033
- const bytes = new Uint8Array(await Bun.file(audioPath).arrayBuffer());
2034
- const samples = await TranscriptionService.decodeAudio(bytes);
2035
- const durationSeconds = samples.length / 16_000;
2036
-
2037
- const modelQuery = opt("model") ?? positional(1) ?? opt("query");
2038
- let modelDir: string;
2039
- if (modelQuery && existsSync(join(modelQuery, "config.json"))) modelDir = modelQuery;
2040
- else if (modelQuery) modelDir = (await resolveModelAuto(modelQuery)).m.path;
2041
- else {
2042
- const reg = new Registry();
2043
- if (reg.list().length === 0) await reg.scan();
2044
- const w = reg.list().find((r) => r.modelType === "whisper");
2045
- if (!w) {
2046
- console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo");
2047
- process.exit(1);
2048
- }
2049
- modelDir = w.path;
2050
- }
2051
- let vadGate: { segments: { start: number; end: number }[] } | null = null;
2052
- if (flag("vad")) {
2053
- const { SileroVad } = await import("./audio/silero-vad");
2054
- const tv = performance.now();
2055
- const vad = SileroVad.load(opt("vad-model"));
2056
- const { segments } = vad.detect(samples, { threshold: Number(opt("vad-threshold") ?? 0.5) });
2057
- vadGate = { segments };
2058
- if (flag("verbose")) console.error(`vad: ${segments.length} speech segment(s) in ${(performance.now() - tv).toFixed(0)} ms`);
2059
- if (segments.length === 0) { process.stdout.write(format === "text" ? "\n" : JSON.stringify({ text: "", vad: { speech: false, segments: [] } }) + "\n"); break; }
2060
- }
2061
- const { model } = await openWhisperModel(modelDir);
2062
- const tok = await loadWhisperTokenizer(modelDir, model.dims.nVocab);
2063
- const transcriber = new WhisperTranscriber(model, tok);
2064
- void vadGate;
2065
- const langOpt = opt("language");
2066
- const task = opt("task") === "translate" ? "translate" as const : "transcribe" as const;
2067
- const beam = opt("beam-size");
2068
- const temperature = opt("temperature");
2069
- const verbose = flag("verbose");
2070
- const t0 = performance.now();
2071
- const result = await transcriber.transcribe(samples, {
2072
- language: !langOpt || langOpt === "auto" ? null : langOpt,
2073
- task,
2074
- beamSize: beam ? Number(beam) : null,
2075
- ...(temperature ? { temperature: Number(temperature) } : flag("no-fallback") ? { temperature: 0 } : {}),
2076
- initialPrompt: opt("prompt"),
2077
- withoutTimestamps: flag("no-timestamps"),
2078
- conditionOnPreviousText: !flag("no-condition"),
2079
- fast: !flag("faithful"),
2080
- wordTimestamps: flag("word-timestamps"),
2081
- audioCtx: opt("audio-ctx") ? Number(opt("audio-ctx")) : null,
2082
- onSegment: verbose
2083
- ? (s) => console.error(`[${formatTimestamp(s.start, ".")} --> ${formatTimestamp(s.end, ".")}] ${s.text.trim()}`)
2084
- : undefined,
2085
- });
2086
- const elapsed = (performance.now() - t0) / 1000;
2087
- if (verbose)
2088
- console.error(`${durationSeconds.toFixed(2)} s audio in ${elapsed.toFixed(3)} s (${(durationSeconds / elapsed).toFixed(1)}× realtime), language ${result.language}`);
2089
- process.stdout.write(formatTranscription(result, format, durationSeconds, task));
2090
- transcriber.dispose();
2091
- model.dispose();
2092
- break;
2093
- }
2094
-
2095
- case "dictate": {
2096
- const { startMicCapture } = await import("./audio/mic-capture");
2097
- const { TranscriptionService, defaultWhisperRecord } = await import("./serve/transcription-service");
2098
- const { style } = await import("./tui");
2099
- const serverUrl = opt("server");
2100
- const hotkeyFlag = argv.includes("--hotkey");
2101
- const hotkey = hotkeyFlag ? Number(opt("hotkey") ?? 61) || 61 : null;
2102
- const language = opt("language") ?? "en";
2103
- const beam = opt("beam-size");
2104
- const vocabulary = opt("vocabulary")?.split(",").map((s) => s.trim()).filter(Boolean);
2105
- const params = {
2106
- language: language === "auto" ? null : language,
2107
- beamSize: beam ? Number(beam) : null,
2108
- temperature: 0,
2109
- prompt: opt("prompt"),
2110
- vocabulary,
2111
- vad: flag("no-vad") ? null : { threshold: 0.5, minSpeechMs: 120 },
2112
- };
2113
- const typeDelay = Number(opt("type-delay") ?? (hotkey ? 0 : 1));
2114
-
2115
- // backend: in-process service or a running server's sessions
2116
- let service: InstanceType<typeof TranscriptionService> | null = null;
2117
- if (!serverUrl) {
2118
- const q = opt("model") ?? positional(0) ?? opt("query");
2119
- let modelDir: string; let modelId: string;
2120
- if (q && existsSync(join(q, "config.json"))) { modelDir = q; modelId = q; }
2121
- else if (q) { const { m } = await resolveModelAuto(q); modelDir = m.path; modelId = m.repoId; }
2122
- else {
2123
- const rec = await defaultWhisperRecord();
2124
- if (!rec) { console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo"); process.exit(1); }
2125
- modelDir = rec.path; modelId = rec.repoId;
2126
- }
2127
- const idle = opt("idle-unload");
2128
- service = new TranscriptionService({ modelDir, modelId, idleUnloadSec: idle !== null ? Number(idle) : 30, resident: flag("resident"), log: () => {} });
2129
- process.stderr.write(style.dim("loading whisper… "));
2130
- await service.ensureLoaded();
2131
- service.vad();
2132
- process.stderr.write(style.dim("ready\n"));
2133
- }
2134
- const base = serverUrl?.replace(/\/$/, "");
2135
- interface Take { feed(pcm: Float32Array): Promise<void>; finish(): Promise<{ text: string; ms: number }>; }
2136
- const startTake = async (): Promise<Take> => {
2137
- if (service) {
2138
- const session = await service.createSession(params);
2139
- return {
2140
- feed: async (pcm) => { await session.append(pcm); },
2141
- finish: async () => { const t0 = performance.now(); const o = await session.finish(); return { text: o.result.text.trim(), ms: performance.now() - t0 }; },
2142
- };
2143
- }
2144
- const r = await fetch(`${base}/v1/audio/sessions`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ language: params.language, beam_size: params.beamSize, temperature: 0, prompt: params.prompt, vocabulary, vad: !!params.vad, vad_min_speech_ms: 120 }) });
2145
- if (!r.ok) throw new Error(`session create failed: ${r.status} ${await r.text()}`);
2146
- const { id } = await r.json() as { id: string };
2147
- return {
2148
- feed: async (pcm) => { await fetch(`${base}/v1/audio/sessions/${id}/audio`, { method: "POST", headers: { "content-type": "audio/pcm;rate=16000" }, body: new Uint8Array(pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength) as ArrayBuffer) }); },
2149
- finish: async () => { const t0 = performance.now(); const f = await fetch(`${base}/v1/audio/sessions/${id}/finish`, { method: "POST" }); const b = await f.json() as { text: string }; return { text: (b.text ?? "").trim(), ms: performance.now() - t0 }; },
2150
- };
2151
- };
2152
-
2153
- const deliver = async (text: string) => {
2154
- if (!text) { console.log(style.dim("(no speech)")); return; }
2155
- console.log(text);
2156
- if (flag("copy")) { const p = Bun.spawn(["pbcopy"], { stdin: "pipe" }); p.stdin.write(text); p.stdin.end(); await p.exited; }
2157
- if (flag("type")) {
2158
- if (typeDelay > 0) await Bun.sleep(typeDelay * 1000);
2159
- const escaped = text.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
2160
- const r = Bun.spawnSync(["osascript", "-e", `tell application "System Events" to keystroke "${escaped}"`], { stderr: "pipe" });
2161
- if (r.exitCode !== 0) console.error(style.dim(`typing failed (grant Accessibility to your terminal): ${r.stderr.toString().trim()}`));
2162
- }
2163
- };
2164
-
2165
- const mic = await startMicCapture({ hotkey });
2166
- let take: Take | null = null;
2167
- let pending: Float32Array[] = []; let pendingLen = 0;
2168
- let feeding: Promise<void> = Promise.resolve();
2169
- const flush = () => {
2170
- if (!take || pendingLen === 0) return;
2171
- const buf = new Float32Array(pendingLen); let o = 0; for (const p of pending) { buf.set(p, o); o += p.length; }
2172
- pending = []; pendingLen = 0;
2173
- const t = take;
2174
- feeding = feeding.then(() => t.feed(buf)).catch((e) => console.error(style.dim(`feed failed: ${(e as Error).message}`)));
2175
- };
2176
- const begin = async () => { if (take) return; take = await startTake(); pending = []; pendingLen = 0; process.stderr.write(style.bold("● recording") + style.dim(hotkey ? " (release to stop)\n" : " (Enter to stop)\n")); };
2177
- const end = async () => {
2178
- if (!take) return;
2179
- flush();
2180
- await feeding;
2181
- const t = take; take = null;
2182
- process.stderr.write(style.dim("transcribing… "));
2183
- try { const { text, ms } = await t.finish(); process.stderr.write(style.dim(`${ms.toFixed(0)} ms\n`)); await deliver(text); }
2184
- catch (e) { console.error(`transcription failed: ${(e as Error).message}`); }
2185
- };
2186
- if (!hotkey) {
2187
- console.error(style.dim("Enter: start/stop a take · q: quit"));
2188
- (async () => {
2189
- for await (const line of console) {
2190
- if (line.trim() === "q") { mic.stop(); process.exit(0); }
2191
- if (take) await end(); else await begin();
2192
- }
2193
- })();
2194
- }
2195
- process.once("SIGINT", () => { mic.stop(); process.exit(0); });
2196
- for await (const ev of mic.events) {
2197
- if (ev.kind === "ready") { if (hotkey) console.error(style.dim(`hold key ${hotkey} to talk · Ctrl-C quits · mic ${ev.info}`)); }
2198
- else if (ev.kind === "error") { console.error(ev.message); mic.stop(); process.exit(1); }
2199
- else if (ev.kind === "hotkey") { if (ev.down) await begin(); else await end(); }
2200
- else if (ev.kind === "pcm" && take) {
2201
- pending.push(ev.samples); pendingLen += ev.samples.length;
2202
- if (pendingLen >= 16_000 / 4) flush(); // every 250 ms of audio
2203
- }
2204
- }
2205
- break;
2206
- }
2207
-
2208
- case "bench":
2209
- case "benchmark": {
2210
- const { banner, step, box, style } = await import("./tui");
2211
- banner(pkg.version);
2212
- serverRuntimeFlags(); // decode options (--compiled-decode, --fused-sdpa, ...) apply to the run
2213
- const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
2214
- const tokens = Number(opt("tokens", "256"));
2215
- const runs = Number(opt("runs", "3"));
2216
- const promptTokens = Number(opt("prompt-tokens", "0"));
2217
- const kvMode = opt("kv-quant", "off")!;
2218
- console.log(` ${style.dim("measures THIS machine, OUR stack only — no other systems needed.")}`);
2219
- console.log(` ${style.dim("numbers on a loaded machine are not quotable; bun scripts/bench-serve.ts all is the preflight-gated harness.")}`);
2220
- const sNative = step("native runtime");
2221
- await ensureNative(sNative);
2222
- sNative.done("native runtime ready");
2223
- const sLoad = step(`loading ${m.repoId}${picked ? " (auto-picked)" : ""}`);
2224
- const { Weights } = await import("./weights");
2225
- const { createModel } = await import("./model/factory");
2226
- const { generate } = await import("./generate");
2227
- const { ChatTemplate } = await import("./chat-template");
2228
- const { loadTokenizer } = await import("./tokenizer");
2229
- const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
2230
- const config = await loadModelConfig(m.path);
2231
- const weights = await Weights.open(m.path);
2232
- const model = createModel(weights, config);
2233
- const tok = await loadTokenizer(m.path);
2234
- const template = await ChatTemplate.load(m.path);
2235
- sLoad.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)}`)}`);
2236
-
2237
- let userMsg = "Write a detailed essay about the history of computing, starting with mechanical calculators.";
2238
- if (promptTokens > 0) {
2239
- const filler = "Background context: the history of computation spans mechanical " +
2240
- "calculators, electromechanical relays, vacuum tubes, transistors, " +
2241
- "integrated circuits, and modern accelerators. ";
2242
- while (tok.encode(userMsg).length < promptTokens - 24) userMsg = filler + userMsg;
2243
- }
2244
- const rendered = template.render([{ role: "user", content: userMsg }]);
2245
- const ids = tok.encode(rendered);
2246
- const promptIds = ids[0] === ids[1] && ids[0] === tok.bosTokenId ? ids.slice(1) : ids;
2247
- let turboScheme: TurboQuantScheme | null;
2248
- try {
2249
- turboScheme = parseTurboQuantScheme(kvMode);
2250
- } catch (e) {
2251
- console.error((e as Error).message);
2252
- process.exit(1);
2253
- }
2254
- const kvOptions = turboScheme
2255
- ? { turboQuant: turboScheme, quantizedKvStart: 0 }
2256
- : kvMode === "config"
2257
- ? (() => {
2258
- if (!config.kvQuant?.length) { console.error("model has no kv_config.json (--kv-quant config)"); process.exit(1); }
2259
- return { kvConfig: config.kvQuant };
2260
- })()
2261
- : kvMode !== "off" ? { kvBits: Number(kvMode), quantizedKvStart: 0 } : {};
2262
-
2263
- // Warmup materializes weights so prefill timing measures prefill,
2264
- // not lazy page-in (same protocol as bench.ts (deleted 2026-08-23; git history)).
2265
- const sWarm = step("warmup");
2266
- {
2267
- const wCache = model.makeCache();
2268
- const wGen = generate(model, promptIds.slice(0, Math.min(8, promptIds.length - 1)), {
2269
- maxTokens: 1, temperature: 0, cache: wCache,
2270
- });
2271
- for await (const _ of wGen) { /* discard */ }
2272
- for (const c of wCache) c.dispose();
2273
- }
2274
- resetPeakMemory();
2275
- sWarm.done("warmup complete");
2276
-
2277
- const decodes: number[] = [];
2278
- const prefills: number[] = [];
2279
- let lastStats: { promptTokens: number; generatedTokens: number } = { promptTokens: 0, generatedTokens: 0 };
2280
- for (let i = 0; i < runs; i++) {
2281
- const sRun = step(`run ${i + 1}/${runs} · ${tokens} tokens`);
2282
- const gen = generate(model, promptIds, { maxTokens: tokens, temperature: 0, ...kvOptions });
2283
- for await (const _ of gen) { /* timing only */ }
2284
- const s = gen.stats!;
2285
- decodes.push(s.decodeTps);
2286
- prefills.push(s.prefillTps);
2287
- lastStats = s;
2288
- sRun.done(`run ${i + 1}/${runs} ${style.dim("·")} ${style.green(style.bold(`${s.decodeTps.toFixed(1)} tok/s`))} ${style.dim(`decode · prefill ${s.prefillTps.toFixed(0)} tok/s`)}`);
2289
- }
2290
- const median = (a: number[]) => [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)]!;
2291
- const peak = peakMemory();
2292
-
2293
- const { EvalDB: DB, gitCommit } = await import("./evaldb");
2294
- new DB().record({
2295
- modelPath: m.path,
2296
- commitSha: gitCommit() ?? undefined,
2297
- promptTokens: lastStats.promptTokens, cachedTokens: 0,
2298
- generatedTokens: lastStats.generatedTokens,
2299
- prefillTps: median(prefills), decodeTps: median(decodes),
2300
- peakBytes: peak,
2301
- notes: `cli-bench median-of-${runs} ${tokens}tok kv=${kvMode}${promptTokens ? ` ctx=${lastStats.promptTokens}` : ""} decode[${decodes.map((d) => d.toFixed(1)).join(",")}]`,
2302
- });
2303
- console.log();
2304
- box([
2305
- `${style.green("●")} ${style.bold("benchmark complete")} ${style.dim(`· ${m.repoId}`)}`,
2306
- "",
2307
- `decode ${style.green(style.bold(`${median(decodes).toFixed(1)} tok/s`))} ${style.dim(`median of ${runs} · [${decodes.map((d) => d.toFixed(1)).join(", ")}]`)}`,
2308
- `prefill ${style.bold(`${median(prefills).toFixed(0)} tok/s`)} ${style.dim(`· ${lastStats.promptTokens} prompt tokens`)}`,
2309
- `peak mem ${style.bold(gb(peak))} ${style.dim("(generation only)")}`,
2310
- `levers ${style.dim(runtimeSummary(turboScheme ? { turboQuant: turboScheme } : { kvQuant: kvMode === "off" ? "off" : kvMode === "config" ? undefined : Number(kvMode) }))}`,
2311
- "",
2312
- style.dim("recorded to the eval DB — see mlx-bun evals"),
2313
- ]);
2314
- break;
2315
- }
2316
-
2317
- case "evals": {
2318
- if (flag("clear")) {
2319
- // Eval rows are real data — archive, never destroy.
2320
- const { DEFAULT_EVAL_DB } = await import("./evaldb");
2321
- const { existsSync, renameSync } = await import("node:fs");
2322
- if (!existsSync(DEFAULT_EVAL_DB)) { console.log("no eval DB to clear"); break; }
2323
- const stamp = new Date().toISOString().replace(/[:.]/g, "-");
2324
- const backup = DEFAULT_EVAL_DB.replace(/\.sqlite$/, `-${stamp}.sqlite`);
2325
- renameSync(DEFAULT_EVAL_DB, backup);
2326
- console.log(`eval DB archived to ${backup}`);
2327
- console.log("fresh DB starts with the next mlx-bun benchmark run");
2328
- break;
2329
- }
2330
- const db = new EvalDB();
2331
- const limit = Number(opt("limit", "20"));
2332
- const rows = db.recent(limit);
2333
- if (flag("raw")) {
2334
- for (const r of rows) console.log(JSON.stringify(r));
2335
- break;
2336
- }
2337
- const { style } = await import("./tui");
2338
- if (rows.length === 0) {
2339
- console.log("no eval runs recorded yet — bun scripts/bench-serve.ts all writes them");
2340
- break;
2341
- }
2342
- const ago = (ts: number) => {
2343
- const s = (Date.now() - ts) / 1000;
2344
- if (s < 90) return `${Math.round(s)}s ago`;
2345
- if (s < 5400) return `${Math.round(s / 60)}m ago`;
2346
- if (s < 129600) return `${Math.round(s / 3600)}h ago`;
2347
- return `${Math.round(s / 86400)}d ago`;
2348
- };
2349
- const modelName = (p: string) =>
2350
- /models--[^/]+--([^/]+)/.exec(p)?.[1] ?? p.split("/").at(-1) ?? p;
2351
- const note = (r: Record<string, unknown>, key: string) =>
2352
- new RegExp(`${key}=([\\w.]+)`).exec((r.notes as string) ?? "")?.[1] ?? "";
2353
- const BENCH_LABELS: Record<string, string> = {
2354
- "h2h-server": "server e2e",
2355
- "h2h-direct": "direct decode",
2356
- "bench.ts": "decode script",
2357
- "cli-bench": "cli benchmark",
2358
- };
2359
- const table = rows.map((r) => ({
2360
- when: ago(r.ts as number),
2361
- model: modelName(r.model_path as string).replace(/-OptiQ-4bit$/, "").replace(/^gemma-4-/, "g4-"),
2362
- bench: BENCH_LABELS[((r.notes as string) ?? "").split(" ")[0] ?? ""] ?? ((r.notes as string) ?? "").split(" ")[0] ?? "",
2363
- kv: note(r, "kv"),
2364
- stack: (r.stack as string) ?? "mlx-bun",
2365
- prefill: (r.prefill_tps as number) > 0 ? `${(r.prefill_tps as number).toFixed(0)}` : "",
2366
- decode: (r.decode_tps as number).toFixed(1),
2367
- ttft: note(r, "ttft_ms") ? `${note(r, "ttft_ms")}ms` : "",
2368
- peak: gb(r.peak_bytes as number),
2369
- commit: (r.commit_sha as string) ?? "",
2370
- }));
2371
- type Row = (typeof table)[number];
2372
- const cols: Array<[string, keyof Row, "left" | "right"]> = [
2373
- ["WHEN", "when", "right"], ["STACK", "stack", "left"], ["MODEL", "model", "left"],
2374
- ["BENCH", "bench", "left"], ["KV", "kv", "left"],
2375
- ["PREFILL", "prefill", "right"], ["DECODE", "decode", "right"], ["TTFT", "ttft", "right"],
2376
- ["PEAK", "peak", "right"], ["COMMIT", "commit", "left"],
2377
- ];
2378
- const widths = cols.map(([h, k]) => Math.max(h.length, ...table.map((t) => t[k].length)));
2379
- console.log();
2380
- console.log(" " + cols.map(([h], i) => style.dim(h.padEnd(widths[i]!))).join(" "));
2381
- for (const t of table) {
2382
- console.log(" " + cols.map(([, k, align], i) => {
2383
- const cell = align === "right" ? t[k].padStart(widths[i]!) : t[k].padEnd(widths[i]!);
2384
- if (k === "decode") return style.green(style.bold(cell));
2385
- if (k === "model") return style.bold(cell);
2386
- if (k === "stack") return cell.trimEnd() === "mlx-bun" ? style.accent(cell) : style.dim(cell);
2387
- if (k === "commit" || k === "when") return style.dim(cell);
2388
- return cell;
2389
- }).join(" "));
2390
- }
2391
- console.log();
2392
- console.log(style.dim(" STACK engine that produced the number (mlx-bun = us; mlx-lm/optiq = Python references)"));
2393
- console.log(style.dim(" BENCH server e2e = full HTTP round-trip · direct decode = engine only, no server"));
2394
- console.log(style.dim(" decode script / cli benchmark = single-stack decode measurement"));
2395
- console.log(style.dim(" KV KV-cache quantization (off = bf16, config = per-layer kv_config.json)"));
2396
- console.log(style.dim(" PREFILL prompt ingestion tok/s (in-process benches) · DECODE generation tok/s"));
2397
- console.log(style.dim(" TTFT request → first token, incl. server overhead (server benches)"));
2398
- console.log(style.dim(" PEAK max memory during generation"));
2399
- console.log();
2400
- console.log(style.dim(` ${rows.length} run(s) · --limit <n> for more · --raw for full records · mlx-bun benchmark to add a run`));
2401
- break;
2402
- }
2403
-
2404
- case "pi": {
2405
- // mlx-bun consumes its own flags; the rest describe how the built-in
2406
- // agent runs. `mlx-bun pi` always drives pi's own TUI in-process
2407
- // (src/pi-terminal.ts) — pi is bundled, nothing to install. Users who
2408
- // already run their own pi connect it with `mlx-bun harness pi`.
2409
- const OURS_VAL = new Set([
2410
- "--query", "-q", "--model", "--port", "--host", "--memory-budget", "--kv-budget", "--prompt-cache",
2411
- "--ssd-cache", "--ssd-cache-max", "--ssd-demote-idle", "--generation-checkpoint", "--kv-quant", "--unix",
2412
- "--batch", "--decode-concurrency", "--adapter", "--adapter-path",
2413
- "--draft-model", "--num-draft-tokens", "--draft-kind", "--ngram-max", "--ngram-min",
2414
- "--compiled-decode", "--compiled-activations", "--fused-sdpa", "--thinking",
2415
- "--temperature", "--temp", "--top-p", "--top-k", "--max-tokens",
2416
- "--mtp", "--context-length",
2417
- "--hlg-sampling", "--hlg-width", "--hlg-shoulder", "--hlg-toe", "--hlg-pivot-offset",
2418
- "--paged-kv-block-size",
2419
- ]);
2420
- const OURS_BOOL = new Set(["--force-wire", "--expert-offload", "--no-open", "--ssd-cache-verify", "--l1", "--l2", "--l3", "--isolate", "--paged-kv"]);
2421
- const passthrough: string[] = [];
2422
- for (let i = 1; i < argv.length; i++) {
2423
- const a = argv[i]!;
2424
- if (OURS_VAL.has(a)) { i++; continue; }
2425
- if (OURS_BOOL.has(a)) continue;
2426
- passthrough.push(a);
2427
- }
2428
-
2429
- const { parsePiArgs } = await import("./pi-terminal");
2430
- const stdinPiped = !process.stdin.isTTY;
2431
- const parsed = parsePiArgs(passthrough, stdinPiped);
2432
-
2433
- // Headless machine-readable runs (`-p`, `--mode json`, `--mode rpc`) must
2434
- // keep stdout pristine for the response/JSONL stream — route our bring-up
2435
- // logging to stderr. Interactive keeps the pretty banner on stdout.
2436
- const quiet = parsed.mode !== "interactive";
2437
- const origStdoutWrite = process.stdout.write.bind(process.stdout);
2438
- if (quiet) {
2439
- (process.stdout as unknown as { write: typeof process.stderr.write }).write =
2440
- process.stderr.write.bind(process.stderr);
2441
- }
2442
- const restoreStdout = () => {
2443
- if (quiet) (process.stdout as unknown as { write: typeof origStdoutWrite }).write = origStdoutWrite;
2444
- };
2445
-
2446
- const rt = serverRuntimeFlags();
2447
- const port = rt.port;
2448
- const baseUrl = `http://127.0.0.1:${port}/v1`;
2449
- const { probeServer } = await import("./harness-pi");
2450
- let models = await probeServer(baseUrl);
2451
- let startedServer = false;
2452
- if (models) {
2453
- console.log(`reusing running server at ${baseUrl} (${models.map((m) => m.id).join(", ")})`);
2454
- // Server-shaping flags can't apply to a server we didn't start —
2455
- // drop them loudly rather than silently or fatally.
2456
- const ignored = [...OURS_VAL, ...OURS_BOOL].filter((f) => f !== "--port" && argv.includes(f));
2457
- if (ignored.length > 0) {
2458
- const { style } = await import("./tui");
2459
- console.log(style.dim(` ignoring ${ignored.join(", ")} — the running server keeps its own configuration`));
2460
- console.log(style.dim(" (stop it or use --port <other> to start a fresh server with these flags)"));
2461
- }
2462
- try {
2463
- const stats = await (await fetch(`http://127.0.0.1:${port}/stats`)).json() as
2464
- { server?: { owner?: string } };
2465
- if (stats.server?.owner === "pi-session") {
2466
- console.log("note: that server belongs to another `mlx-bun pi` session and ends when it does —");
2467
- console.log(" run `mlx-bun serve` for a persistent server.");
2468
- }
2469
- } catch {}
2470
- } else {
2471
- const { banner, step, style } = await import("./tui");
2472
- banner(pkg.version);
2473
- const { m, picked } = await resolveModelAuto(opt("query") ?? optShort("q"));
2474
- if (picked)
2475
- console.log(` ${style.dim(`auto-picked ${m.repoId} (largest supported model that fits; override with --query)`)}`);
2476
- const sNative = step("native runtime");
2477
- await ensureNative(sNative);
2478
- sNative.done("native runtime ready");
2479
- const { createServer, loadContext } = await import("./server");
2480
- const sLoad = step(`loading ${m.repoId}`);
2481
- const t0 = performance.now();
2482
- const ctx = await loadContext(m.path, m.repoId, {
2483
- memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
2484
- glm: glm52RuntimeFlags(rt.serverOptions),
2485
- });
2486
- await mountStartupAdapter(ctx, rt.serverOptions);
2487
- createServer(ctx, port, { ...rt.serverOptions, owner: "pi-session" });
2488
- startedServer = true;
2489
- sLoad.done(`serving ${style.bold(m.repoId)} ${style.dim(`at ${baseUrl} · ready in ${(performance.now() - t0).toFixed(0)} ms`)}`);
2490
- models = await probeServer(baseUrl);
2491
- if (!models) { restoreStdout(); console.error("server started but /v1/models probe failed"); process.exit(1); }
2492
- }
2493
- const exitNote = () => {
2494
- // The server we started lives exactly as long as the pi session — any
2495
- // exit tears it down. A standalone server belongs to `mlx-bun serve`
2496
- // (which we reuse and never stop).
2497
- if (startedServer && parsed.mode === "interactive")
2498
- console.log("pi exited — shutting down (use `mlx-bun serve` for a persistent server)");
2499
- };
2500
-
2501
- // pi's own TUI embedded in-process — nothing to install.
2502
- const { runEmbeddedPi } = await import("./pi-terminal");
2503
- let message = parsed.message;
2504
- if (parsed.mode === "print" && stdinPiped) {
2505
- const piped = (await Bun.stdin.text()).trim();
2506
- if (piped) message = message ? `${message}\n\n${piped}` : piped;
2507
- }
2508
- if (parsed.mode === "interactive")
2509
- console.log(`launching mlx-bun pi (model ${models[0]!.id}) — /help for commands, double-Ctrl+C to exit`);
2510
- restoreStdout();
2511
- if (parsed.ignored.length) {
2512
- const { style } = await import("./tui");
2513
- console.log(style.dim(` ignoring pi flags: ${parsed.ignored.join(", ")}`));
2514
- }
2515
- const code = await runEmbeddedPi({
2516
- baseUrl,
2517
- modelLabel: models[0]!.id,
2518
- contextWindow: models[0]!.contextWindow,
2519
- reasoning: models[0]!.reasoning,
2520
- vision: models[0]!.vision,
2521
- verbose: parsed.verbose,
2522
- mode: parsed.mode,
2523
- printFormat: parsed.printFormat,
2524
- initialMessage: message,
2525
- });
2526
- exitNote();
2527
- process.exit(code);
2528
- }
2529
-
2530
- case "harness": {
2531
- const target = positional(0);
2532
- if (target !== "pi") {
2533
- console.error("usage: mlx-bun harness pi [--base-url <url>] [--remove]");
2534
- process.exit(1);
2535
- }
2536
- const { detectPi, installPiExtension, removePiExtension, DEFAULT_BASE_URL } =
2537
- await import("./harness-pi");
2538
- if (flag("remove")) {
2539
- console.log(removePiExtension() ? "mlx-bun provider extension removed" : "nothing to remove");
2540
- break;
2541
- }
2542
- const pi = detectPi();
2543
- if (!pi.found) {
2544
- console.error("pi not found on PATH. Install it first:");
2545
- console.error(" curl -fsSL https://pi.dev/install.sh | sh");
2546
- console.error("then re-run: mlx-bun harness pi");
2547
- process.exit(1);
2548
- }
2549
- const { step, box, style } = await import("./tui");
2550
- const sPi = step("detecting pi");
2551
- sPi.done(`pi ${style.dim(`${pi.binPath}${pi.version ? ` · v${pi.version}` : ""}`)}`);
2552
- const baseUrl = opt("base-url", DEFAULT_BASE_URL)!;
2553
- const sExt = step("installing provider extension");
2554
- const result = await installPiExtension(baseUrl);
2555
- sExt.done(`provider extension installed ${style.dim(`· ${result.path}`)}`);
2556
- const sProbe = step("probing server");
2557
- if (result.serverReachable) sProbe.done(`server live ${style.dim(`· ${result.bakedModels.join(", ")}`)}`);
2558
- else sProbe.done(`server not running ${style.dim(`· models discovered when pi starts against ${baseUrl}`)}`);
2559
- console.log();
2560
- box([
2561
- `${style.green("●")} ${style.bold("your pi is connected to mlx-bun")}`,
2562
- "",
2563
- `launch ${style.accent("pi --provider mlx-bun")}`,
2564
- `select ${style.dim("/model inside pi · or scope cycling:")} ${style.accent('pi --models "mlx-bun/*"')}`,
2565
- `undo ${style.dim("mlx-bun harness pi --remove")}`,
2566
- ]);
2567
- break;
2568
- }
2569
-
2570
- case "train": {
2571
- const { banner, step, box, style } = await import("./tui");
2572
- const query = positional(0) ?? opt("query");
2573
- const dataDir = opt("data");
2574
- if (!dataDir) {
2575
- console.error("usage: mlx-bun train <model> --data <dir> (see: mlx-bun help train)");
2576
- process.exit(1);
2577
- }
2578
- if (!(await Bun.file(`${dataDir}/train.jsonl`).exists())) {
2579
- console.error(`no train.jsonl in ${dataDir}`);
2580
- process.exit(1);
2581
- }
2582
- const method = opt("method", "orpo")!;
2583
- if (method !== "sft" && method !== "dpo" && method !== "orpo") {
2584
- console.error(`--method must be sft | dpo | orpo (got "${method}")`);
2585
- process.exit(1);
2586
- }
2587
- const isOrpo = method === "orpo";
2588
- // ORPO chosen-NLL scope (trainer sftScope; job.ts sft_scope). "full" =
2589
- // paper/TRL-faithful prompt+response CE (the default); "response" =
2590
- // response-only, bit-exact to pre-2026-07 runs. Validated here so junk
2591
- // fails fast, before any model resolution or weight loading.
2592
- const sftScope = opt("sft-scope");
2593
- if (sftScope !== null && sftScope !== "full" && sftScope !== "response") {
2594
- console.error(`--sft-scope must be full | response (got "${sftScope}")`);
2595
- process.exit(1);
2596
- }
2597
- // Validated numeric flag: default if absent, hard-exit on a non-number.
2598
- const numFlag = (name: string, dflt: number): number => {
2599
- const v = opt(name);
2600
- if (v == null) return dflt;
2601
- const n = Number(v);
2602
- if (!Number.isFinite(n)) { console.error(`--${name} expects a number (got "${v}")`); process.exit(1); }
2603
- return n;
2604
- };
2605
-
2606
- banner(pkg.version);
2607
- const sNative = step("native runtime");
2608
- await ensureNative(sNative);
2609
- sNative.done("native runtime ready");
2610
-
2611
- const { m, picked } = await resolveModelAuto(query);
2612
-
2613
- const isGemma = (await Bun.file(`${m.path}/config.json`).text()).toLowerCase().includes("gemma");
2614
-
2615
- const modelTag = isGemma ? "e4b" : "cpm5";
2616
- const adapter = opt("adapter") ?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/${method}-${modelTag}`;
2617
- const iters = numFlag("iters", 100);
2618
- const seq = numFlag("seq", isGemma ? 8192 : 4096);
2619
- const seg = flag("no-segment") ? 0 : numFlag("seg", isOrpo ? 2 : 0);
2620
- const saveEvery = numFlag("save-every", 0);
2621
- const flashOn = !flag("no-flash");
2622
- const prefixOn = !flag("no-prefix");
2623
- const resume = opt("resume") ?? "";
2624
-
2625
- // Build the snake_case submit record — the exact shape the server hands to
2626
- // the finetune job runner (src/train/job.ts parseConfig).
2627
- const cfg: Record<string, unknown> = {
2628
- model_dir: m.path,
2629
- data_dir: dataDir,
2630
- adapter_path: adapter,
2631
- method,
2632
- rank: numFlag("rank", isOrpo ? 16 : 8),
2633
- scale: numFlag("scale", isOrpo ? 2.0 : 1.0),
2634
- rank_scaling: "by_bits",
2635
- num_layers: -1,
2636
- iters,
2637
- learning_rate: numFlag("lr", isOrpo ? 1e-5 : method === "dpo" ? 5e-5 : 2e-4),
2638
- max_seq_length: seq,
2639
- batch_size: numFlag("batch", 1),
2640
- grad_accumulation_steps: numFlag("grad-accum", 1),
2641
- seed: numFlag("seed", 0),
2642
- steps_per_report: 1,
2643
- steps_per_eval: saveEvery > 0 ? saveEvery : 1_000_000,
2644
- save_checkpoints: saveEvery > 0,
2645
- segment_size: seg,
2646
- grad_clip_norm: numFlag("grad-clip", 1.0),
2647
- val_max_examples: numFlag("val-size", 256),
2648
- warm_start_adapter: resume,
2649
- ...(sftScope ? { sft_scope: sftScope } : {}),
2650
- ...(isOrpo ? {
2651
- orpo_lambda: numFlag("lambda", 0.1),
2652
- orpo_lr_schedule: "cosine",
2653
- orpo_warmup_iters: Math.min(10, Math.floor(iters / 10)),
2654
- orpo_chunk_size: 512,
2655
- orpo_flash_ce: flashOn,
2656
- orpo_fused_ce: !flashOn,
2657
- orpo_prefix_shared: prefixOn,
2658
- } : {}),
2659
- };
2660
-
2661
- // Pre-flight: dataset counts + detected format (bail before loading the model).
2662
- const { inspectDataset } = await import("./train/job");
2663
- const ds = await inspectDataset(dataDir);
2664
- if (!ds.ok) { console.error(`dataset: ${ds.error}`); process.exit(1); }
2665
-
2666
- const planLines = [
2667
- `${style.green("●")} ${style.bold(`train ${method}`)} ${style.dim(`· ${m.repoId}${picked ? " (auto-picked)" : ""}${isGemma ? " · e4b defaults" : ""}`)}`,
2668
- "",
2669
- `data ${style.bold(`${ds.n_train} train`)}${ds.n_valid ? ` · ${ds.n_valid} valid` : ""} ${style.dim(`· format ${ds.format}`)}`,
2670
- `loop ${style.dim(`iters ${iters} · lr ${cfg.learning_rate} · rank ${cfg.rank} · scale ${cfg.scale} · seq ${seq} · batch ${cfg.batch_size}`)}`,
2671
- ];
2672
- if (isOrpo)
2673
- planLines.push(
2674
- `head ${style.dim(flashOn ? "flash-CCE Metal ([M,vocab]-free)" : "MLX fused linear-CE")}`,
2675
- `stack ${style.dim(`prefix-share ${prefixOn ? "on" : "off"} · segmented ${seg > 0 ? `${seg}/seg` : "off"} · λ ${cfg.orpo_lambda}`)}`,
2676
- `stability ${style.dim(`grad-clip ${cfg.grad_clip_norm || "off"} · val-size ${cfg.val_max_examples}${(cfg.grad_accumulation_steps as number) > 1 ? ` · grad-accum ${cfg.grad_accumulation_steps} (eff batch ${(cfg.batch_size as number) * (cfg.grad_accumulation_steps as number)})` : ""}`)}`,
2677
- );
2678
- else planLines.push(`stack ${style.dim(`segmented ${seg > 0 ? `${seg}/seg` : "off"}`)}`);
2679
- if (resume) planLines.push(`warm-start ${style.dim(`from ${resume} (weights only)`)}`);
2680
- if (saveEvery > 0) planLines.push(`checkpoint ${style.dim(`every ${saveEvery} steps`)}`);
2681
- planLines.push("", `adapter ${style.dim(adapter)}`);
2682
- console.log();
2683
- box(planLines);
2684
- console.log();
2685
- console.log(` ${style.dim("watch live (other tab):")} ${style.accent(`mlx-bun train-watch ${adapter}`)}`);
2686
- console.log();
2687
-
2688
- if (flag("dry-run")) { console.log(` ${style.dim("dry run — not training.")}`); break; }
2689
-
2690
- // Run the finetune job runner IN-PROCESS (foreground), streaming metrics to
2691
- // the terminal — the same runner the server drives via submitSubprocess.
2692
- // Run long jobs detached from your own shell: nohup mlx-bun train … &
2693
- const { finetuneRunner } = await import("./train/job");
2694
- const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
2695
- const losses: number[] = [];
2696
- const stepMs: number[] = [];
2697
- let lastStepT = Date.now();
2698
- resetPeakMemory();
2699
- const emit = (e: JobEvent) => {
2700
- if (e.type === "stage" && e.message) console.log(` ${style.dim("·")} ${e.message}`);
2701
- else if (e.type === "metric" && e.kind === "train") {
2702
- const now = Date.now(); stepMs.push(now - lastStepT); lastStepT = now;
2703
- losses.push(e.loss);
2704
- const n = losses.length;
2705
- if (n <= 3 || n % 10 === 0)
2706
- console.log(` step ${n}/${iters}: loss ${style.bold(e.loss.toFixed(4))} ${style.dim(`(${(stepMs[stepMs.length - 1]! / 1000).toFixed(1)}s/step · peak ${gb(peakMemory())})`)}`);
2707
- }
2708
- };
2709
- try {
2710
- await finetuneRunner(emit, cfg);
2711
- } catch (err) {
2712
- console.error(`\n training failed: ${err instanceof Error ? err.message : String(err)}`);
2713
- process.exit(1);
2714
- }
2715
- const finite = losses.every((l) => Number.isFinite(l));
2716
- const sorted = stepMs.slice(1).sort((a, b) => a - b);
2717
- const med = sorted[Math.floor(sorted.length / 2)] ?? stepMs[0] ?? 0;
2718
- console.log();
2719
- box([
2720
- `${style.green("●")} ${style.bold("training complete")} ${style.dim(`· ${losses.length} steps`)}`,
2721
- "",
2722
- `loss ${style.bold(`${losses[0]?.toFixed(4) ?? "—"} → ${losses[losses.length - 1]?.toFixed(4) ?? "—"}`)}${finite ? "" : " (NON-FINITE!)"}`,
2723
- `speed ${style.dim(`${(med / 1000).toFixed(1)}s/step median · peak ${gb(peakMemory())}`)}`,
2724
- "",
2725
- `adapter ${style.bold(adapter)}`,
2726
- `serve it ${style.accent(`mlx-bun serve ${m.repoId} --adapter ${adapter}`)}`,
2727
- ]);
2728
- break;
2729
- }
2730
-
2731
- case "train-watch": {
2732
- // Live mactop-style dashboard for a training run: tails <adapter>/metrics.jsonl.
2733
- const dir = positional(0) ?? opt("adapter")
2734
- ?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5`;
2735
- const { runWatch } = await import("./train/watch");
2736
- await runWatch(dir);
2737
- break;
2738
- }
2739
-
2740
- case "fuse": {
2741
- // mlx_lm.fuse counterpart over our native machinery (src/train/fuse.ts):
2742
- // fold a LoRA adapter into the base weights → standalone snapshot.
2743
- const unsupported = ["--de-quantize", "--dequantize", "--export-gguf", "--gguf-path", "--upload-repo"]
2744
- .filter((f) => argv.includes(f));
2745
- if (unsupported.length > 0) {
2746
- console.error(`${unsupported.join(", ")}: not supported (see: mlx-bun help fuse)`);
2747
- process.exit(1);
2748
- }
2749
- const modelArg = positional(0) ?? opt("model");
2750
- if (!modelArg) {
2751
- console.error("usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [--save-path <dir>]");
2752
- process.exit(1);
2753
- }
2754
- const adapterDir = opt("adapter") ?? opt("adapter-path") ?? "adapters";
2755
- const savePath = opt("save-path") ?? "fused_model";
2756
- const { existsSync } = await import("node:fs");
2757
- if (!existsSync(adapterDir)) {
2758
- console.error(`adapter dir not found: ${adapterDir}`);
2759
- process.exit(1);
2760
- }
2761
- let modelDir = modelArg;
2762
- if (!existsSync(`${modelArg}/config.json`)) {
2763
- const reg = new Registry();
2764
- if (reg.list().length === 0) await reg.scan();
2765
- modelDir = reg.resolve(modelArg).path;
2766
- }
2767
- const { banner, step, box, style } = await import("./tui");
2768
- banner(pkg.version);
2769
- const sNative = step("native runtime");
2770
- await ensureNative(sNative);
2771
- sNative.done("native runtime ready");
2772
- const s = step(`fusing ${adapterDir} into ${modelDir}`);
2773
- const { fuseAdapter } = await import("./train");
2774
- try {
2775
- const stats = await fuseAdapter(modelDir, adapterDir, savePath, (e) => s.update(e.message));
2776
- s.done(`fused ${stats.fusedModules} module(s) ${style.dim(`· ${stats.totalTensors} tensors written`)}`);
2777
- console.log();
2778
- box([
2779
- `${style.green("●")} ${style.bold("fuse complete")}`,
2780
- "",
2781
- `base ${style.dim(modelDir)}`,
2782
- `adapter ${style.dim(adapterDir)}`,
2783
- `model ${style.bold(stats.outDir)}`,
2784
- ...(stats.skippedAdapterTensors > 0
2785
- ? [`skipped ${style.dim(`${stats.skippedAdapterTensors} adapter tensor(s) with no matching base weight`)}`] : []),
2786
- "",
2787
- `serve it ${style.accent(`mlx-bun serve ${stats.outDir}`)}`,
2788
- ]);
2789
- } catch (e) {
2790
- s.fail(`fuse failed: ${e instanceof Error ? e.message : String(e)}`);
2791
- process.exit(1);
2792
- }
2793
- break;
2794
- }
2795
-
2796
- case "convert": {
2797
- // mlx_lm.convert counterpart over our native quantize pipeline (the same
2798
- // engine as the web /api/quantize): uniform affine 4/8-bit, or the OptiQ
2799
- // mixed-precision path via --target-bpw.
2800
- // --upload-repo (mlx_lm.convert parity): push the converted model to the
2801
- // Hub afterwards, via the same engine as `mlx-bun upload`. The token is
2802
- // resolved BEFORE any conversion work — failing after minutes of
2803
- // quantizing because auth is missing would be cruel.
2804
- const uploadRepo = ((): string | null => {
2805
- if (!argv.includes("--upload-repo")) return null;
2806
- const v = opt("upload-repo");
2807
- if (!v || v.startsWith("-")) {
2808
- console.error("--upload-repo expects a repo id (org/name)");
2809
- process.exit(1);
2810
- }
2811
- return v;
2812
- })();
2813
- let uploadToken: string | null = null;
2814
- if (uploadRepo) {
2815
- const { getHfToken } = await import("./hf-push");
2816
- uploadToken = getHfToken();
2817
- if (!uploadToken) {
2818
- console.error("--upload-repo needs a Hugging Face WRITE token and none was found —");
2819
- console.error("run `hf auth login`, export HF_TOKEN, or save one in the web UI (Settings → Hugging Face).");
2820
- process.exit(1);
2821
- }
2822
- }
2823
- const unsupported = ["--dtype", "-d", "--dequantize", "--quant-predicate"]
2824
- .filter((f) => argv.includes(f));
2825
- if (unsupported.length > 0) {
2826
- console.error(`${unsupported.join(", ")}: not supported (mixed precision: --target-bpw; see: mlx-bun help convert)`);
2827
- process.exit(1);
2828
- }
2829
- const qMode = opt("q-mode", "affine")!;
2830
- if (qMode !== "affine") {
2831
- console.error(`--q-mode ${qMode}: only "affine" is supported`);
2832
- process.exit(1);
2833
- }
2834
- const hfPath = opt("hf-path") ?? opt("model") ?? positional(0);
2835
- if (!hfPath) {
2836
- console.error("usage: mlx-bun convert --hf-path <repo-or-path> -q [--q-bits N] [--q-group-size N] [--mlx-path <dir>] [--target-bpw F]");
2837
- process.exit(1);
2838
- }
2839
- const targetBpwRaw = opt("target-bpw");
2840
- if (!argv.includes("-q") && !argv.includes("--quantize") && targetBpwRaw === null) {
2841
- console.error("plain (non-quantizing) conversion is not supported yet — pass -q or --target-bpw");
2842
- process.exit(1);
2843
- }
2844
- const targetBpw = targetBpwRaw !== null ? Number(targetBpwRaw) : undefined;
2845
- if (targetBpwRaw !== null && (!Number.isFinite(targetBpw!) || targetBpw! <= 0)) {
2846
- console.error(`--target-bpw expects a positive number (got "${targetBpwRaw}")`);
2847
- process.exit(1);
2848
- }
2849
- const qBits = Number(opt("q-bits", "4"));
2850
- if (qBits !== 4 && qBits !== 8) {
2851
- console.error(`--q-bits must be 4 or 8 (got "${opt("q-bits")}")`);
2852
- process.exit(1);
2853
- }
2854
- const qGroup = Number(opt("q-group-size", "64"));
2855
- if (qGroup !== 32 && qGroup !== 64) {
2856
- console.error(`--q-group-size must be 32 or 64 (got "${opt("q-group-size")}")`);
2857
- process.exit(1);
2858
- }
2859
- const candidateBits = opt("candidate-bits")?.split(",").map((s) => Number(s.trim()));
2860
- if (candidateBits && candidateBits.some((b) => !Number.isInteger(b) || b < 2 || b > 8)) {
2861
- console.error(`--candidate-bits expects a comma list of integers in [2, 8] (got "${opt("candidate-bits")}")`);
2862
- process.exit(1);
2863
- }
2864
- const rotateWeights = flag("rotate-weights");
2865
- const rotationSeed = Number(opt("rotation-seed", "42"));
2866
- if (!Number.isInteger(rotationSeed)) {
2867
- console.error(`--rotation-seed expects an integer (got "${opt("rotation-seed")}")`);
2868
- process.exit(1);
2869
- }
2870
- const mlxPath = opt("mlx-path", "mlx_model")!;
2871
- const { existsSync } = await import("node:fs");
2872
- if (existsSync(mlxPath)) {
2873
- console.error(`Cannot save to the path ${mlxPath} as it already exists — delete it or pass a fresh --mlx-path.`);
2874
- process.exit(1);
2875
- }
2876
- const { banner, step, box, style } = await import("./tui");
2877
- banner(pkg.version);
2878
- const sNative = step("native runtime");
2879
- await ensureNative(sNative);
2880
- sNative.done("native runtime ready");
2881
- // Resolve the source: local dir → itself; downloaded model → registry;
2882
- // otherwise an HF repo id, downloaded first (the point of the verb).
2883
- let srcDir = hfPath;
2884
- if (!existsSync(`${hfPath}/config.json`)) {
2885
- const reg = new Registry();
2886
- if (reg.list().length === 0) await reg.scan();
2887
- try {
2888
- srcDir = reg.resolve(hfPath).path;
2889
- } catch (e) {
2890
- if (!/^[\w.-]+\/[\w.-]+$/.test(hfPath)) {
2891
- console.error(e instanceof Error ? e.message : String(e));
2892
- process.exit(1);
2893
- }
2894
- const { downloadModel } = await import("./download");
2895
- const sDl = step(`downloading ${hfPath}`);
2896
- srcDir = await downloadModel(hfPath, {
2897
- onProgress: (file, received, total) => {
2898
- const pct = total ? Math.floor((received / total) * 100) : 0;
2899
- sDl.update(`${style.bold(hfPath)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
2900
- },
2901
- });
2902
- sDl.done(`${style.bold(hfPath)} ${style.dim("downloaded · verified")}`);
2903
- await reg.scan();
2904
- }
2905
- }
2906
- const sQ = step(
2907
- targetBpw !== undefined
2908
- ? `quantizing (mixed, target ${targetBpw} bpw — sensitivity sweep, ~minutes)`
2909
- : `quantizing (${qBits}-bit, group ${qGroup})`,
2910
- );
2911
- const { quantizeModelDir, automaticRotationWeightTransform } = await import("./quantize");
2912
- try {
2913
- const r = await quantizeModelDir(
2914
- srcDir,
2915
- mlxPath,
2916
- {
2917
- bits: qBits as 4 | 8,
2918
- groupSize: qGroup as 32 | 64,
2919
- mode: "affine",
2920
- ...(targetBpw !== undefined ? { targetBpw } : {}),
2921
- ...(candidateBits ? { candidateBits } : {}),
2922
- ...(opt("calibration-mix") ? { calibrationMix: opt("calibration-mix")! } : {}),
2923
- ...(opt("n-calibration") ? { nCalibration: Number(opt("n-calibration")) } : {}),
2924
- ...(rotateWeights
2925
- ? { weightTransform: automaticRotationWeightTransform({ seed: rotationSeed }) }
2926
- : {}),
2927
- },
2928
- (e) => sQ.update(e.message),
2929
- );
2930
- sQ.done(`quantized ${r.nQuantized} module(s) ${style.dim(`· ${r.achievedBpw.toFixed(2)} bpw achieved`)}`);
2931
- console.log();
2932
- box([
2933
- `${style.green("●")} ${style.bold("convert complete")}`,
2934
- "",
2935
- `source ${style.dim(srcDir)}`,
2936
- `model ${style.bold(r.outDir)} ${style.dim(`· ${gb(r.write.totalSize)}`)}`,
2937
- `quant ${style.dim(targetBpw !== undefined ? `mixed ${r.achievedBpw.toFixed(2)} bpw (target ${targetBpw})` : `${qBits}-bit g${qGroup} affine`)}`,
2938
- ...(rotateWeights ? [`transform ${style.dim(`TurboQuant rotation seed ${rotationSeed}`)}`] : []),
2939
- "",
2940
- `serve it ${style.accent(`mlx-bun serve ${r.outDir}`)}`,
2941
- ]);
2942
- if (uploadRepo) {
2943
- const sUp = step(`uploading ${r.outDir} → ${uploadRepo}`);
2944
- try {
2945
- const { uploadFolder } = await import("./hf-push");
2946
- const up = await uploadFolder(r.outDir, uploadRepo, {
2947
- repoType: "model",
2948
- token: uploadToken,
2949
- commitMessage: "Upload with mlx-bun convert",
2950
- onProgress: (file, sent, total) =>
2951
- sUp.update(`${style.bold(uploadRepo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
2952
- });
2953
- sUp.done(`uploaded ${style.bold(up.url)}`);
2954
- } catch (e) {
2955
- sUp.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
2956
- console.error(`the converted model is intact at ${r.outDir} — retry with: mlx-bun upload --path ${r.outDir} --upload-repo ${uploadRepo}`);
2957
- process.exit(1);
2958
- }
2959
- }
2960
- } catch (e) {
2961
- sQ.fail(`convert failed: ${e instanceof Error ? e.message : String(e)}`);
2962
- process.exit(1);
2963
- }
2964
- break;
2965
- }
2966
-
2967
- case "upload": {
2968
- // mlx_lm.upload counterpart (same flag names: --path, --upload-repo) over
2969
- // our native push-to-hub (src/hf-push.ts — the exact engine the web
2970
- // /api/{quantize,finetune,dataset}/push routes call). Model-free until
2971
- // the network: everything before uploadFolder is arg + token validation.
2972
- const dir = opt("path", "mlx_model")!; // mlx_lm.upload's --path default
2973
- const repo = ((): string | null => {
2974
- const v = opt("upload-repo");
2975
- return v && !v.startsWith("-") ? v : null;
2976
- })();
2977
- if (!repo) {
2978
- console.error("usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [--private]");
2979
- process.exit(1);
2980
- }
2981
- const { existsSync, statSync } = await import("node:fs");
2982
- if (!existsSync(dir) || !statSync(dir).isDirectory()) {
2983
- console.error(`not a directory: ${dir} (pass the model directory via --path)`);
2984
- process.exit(1);
2985
- }
2986
- const { getHfToken, uploadFolder } = await import("./hf-push");
2987
- const token = getHfToken();
2988
- if (!token) {
2989
- console.error("no Hugging Face token found — uploading needs a WRITE token from one of:");
2990
- console.error(" hf auth login (~/.cache/huggingface/token)");
2991
- console.error(" export HF_TOKEN=hf_…");
2992
- console.error(" web UI Settings → Hugging Face (~/.mlx-bun/hf.json)");
2993
- process.exit(1);
2994
- }
2995
- const { banner, step, box, style } = await import("./tui");
2996
- banner(pkg.version);
2997
- const s = step(`uploading ${dir} → ${repo}`);
2998
- try {
2999
- const r = await uploadFolder(dir, repo, {
3000
- repoType: "model",
3001
- private: flag("private"),
3002
- token,
3003
- commitMessage: "Upload with mlx-bun",
3004
- onProgress: (file, sent, total) =>
3005
- s.update(`${style.bold(repo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
3006
- });
3007
- s.done(`uploaded ${style.bold(repo)}`);
3008
- console.log();
3009
- box([
3010
- `${style.green("●")} ${style.bold("upload complete")}`,
3011
- "",
3012
- `source ${style.dim(dir)}`,
3013
- `repo ${style.url(r.url)}${flag("private") ? style.dim(" · private") : ""}`,
3014
- "",
3015
- `get it back anywhere: ${style.accent(`mlx-bun get ${repo}`)}`,
3016
- ]);
3017
- } catch (e) {
3018
- s.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
3019
- process.exit(1);
3020
- }
3021
- break;
3022
- }
3023
-
3024
- case "perplexity": {
3025
- // mlx_lm.perplexity counterpart (methodology reproduced exactly — see
3026
- // src/eval/perplexity.ts) over a LOCAL dataset file; never downloads data.
3027
- const modelArg = positional(0) ?? opt("model") ?? opt("query");
3028
- const dataPath = opt("data-path");
3029
- if (!modelArg || !dataPath) {
3030
- console.error("usage: mlx-bun perplexity <model-query-or-path> --data-path <file.txt|file.jsonl> [--sequence-length 512] [--num-samples 256] [--batch-size 8] [--seed 123]");
3031
- process.exit(1);
3032
- }
3033
- const { existsSync } = await import("node:fs");
3034
- if (!existsSync(dataPath)) {
3035
- console.error(`data file not found: ${dataPath} (a local .txt or .jsonl — HF datasets are not downloaded)`);
3036
- process.exit(1);
3037
- }
3038
- const intFlag = (name: string, dflt: number, lo: number): number => {
3039
- const v = Number(opt(name, String(dflt)));
3040
- if (!Number.isInteger(v) || (v < lo && v !== -1)) {
3041
- console.error(`--${name} expects an integer >= ${lo} (got "${opt(name)}")`);
3042
- process.exit(1);
3043
- }
3044
- return v;
3045
- };
3046
- const seqLen = intFlag("sequence-length", 512, 2);
3047
- const numSamples = intFlag("num-samples", 256, 1); // -1 = all
3048
- const batchSize = intFlag("batch-size", 8, 1);
3049
- const seed = intFlag("seed", 123, 0);
3050
- const { banner, step, box, style } = await import("./tui");
3051
- banner(pkg.version);
3052
- const sNative = step("native runtime");
3053
- await ensureNative(sNative);
3054
- sNative.done("native runtime ready");
3055
- let modelDir = modelArg;
3056
- if (!existsSync(`${modelArg}/config.json`)) {
3057
- const reg = new Registry();
3058
- if (reg.list().length === 0) await reg.scan();
3059
- modelDir = reg.resolve(modelArg).path;
3060
- }
3061
- const sLoad = step(`loading ${modelDir}`);
3062
- const { Weights } = await import("./weights");
3063
- const { createModel } = await import("./model/factory");
3064
- const { loadTokenizer } = await import("./tokenizer");
3065
- const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
3066
- const config = await loadModelConfig(modelDir);
3067
- const model = createModel(await Weights.open(modelDir), config);
3068
- const tok = await loadTokenizer(modelDir);
3069
- sLoad.done(`model loaded ${style.dim(`· ${modelDir}`)}`);
3070
-
3071
- const { parseSamples, packRows, evalPpl } = await import("./eval/perplexity");
3072
- const sData = step(`tokenizing ${dataPath}`);
3073
- const samples = parseSamples(await Bun.file(dataPath).text(), dataPath);
3074
- const rows = packRows(
3075
- samples.map((t) => tok.encode(t)),
3076
- { sequenceLength: seqLen, numSamples, seed },
3077
- );
3078
- if (rows.length === 0) {
3079
- sData.fail(`dataset too small: fewer than ${seqLen} tokens (need at least one full row)`);
3080
- process.exit(1);
3081
- }
3082
- sData.done(`${rows.length} row(s) × ${seqLen} tokens ${style.dim(`· ${samples.length} sample(s), seed ${seed}`)}`);
3083
-
3084
- const sEval = step(`evaluating (batch ${batchSize})`);
3085
- resetPeakMemory();
3086
- const t0 = performance.now();
3087
- const r = evalPpl(model, rows, batchSize, (done, total) =>
3088
- sEval.update(`batch ${done}/${total}`));
3089
- const seconds = (performance.now() - t0) / 1000;
3090
- sEval.done(`evaluated ${r.tokens.toLocaleString()} tokens ${style.dim(`in ${seconds.toFixed(1)} s`)}`);
3091
- console.log();
3092
- box([
3093
- `${style.green("●")} ${style.bold("perplexity")} ${style.dim(`· ${modelDir.split("/snapshots/")[0]?.split("/").at(-1) ?? modelDir}`)}`,
3094
- "",
3095
- `ppl ${style.green(style.bold(r.ppl.toFixed(3)))} ${style.dim(`± ${r.standardError.toFixed(3)}`)}`,
3096
- `mean CE ${style.bold(r.meanLoss.toFixed(4))} ${style.dim("nats/token")}`,
3097
- `tokens ${style.bold(r.tokens.toLocaleString())} ${style.dim(`· ${r.rows} row(s) × ${seqLen}`)}`,
3098
- `speed ${style.dim(`${(r.tokens / seconds).toFixed(0)} tok/s · peak ${gb(peakMemory())}`)}`,
3099
- ]);
3100
- break;
3101
- }
3102
-
3103
- // `setup` is a true alias for `memory` — same handler, same subcommands
3104
- // (`mlx-bun setup init` == `mlx-bun memory init`; the handler reads the
3105
- // subcommand from argv[1] onward, identical for both spellings).
3106
- case "setup":
3107
- case "memory": {
3108
- const { style, box, step, banner } = await import("./tui");
3109
- const {
3110
- vaultRoot, vaultStatus, setupVault, importArticlesFrom, commitVault,
3111
- listMemoryDocuments, readArticle, resolveArticlePath, searchArticles,
3112
- parseToc, extractSection, getArticleLinks,
3113
- } = await import("./memory/vault");
3114
- const { installSchedule, removeSchedule, scheduleStatus, parseAt } = await import("./memory/schedule");
3115
- const root = vaultRoot();
3116
- const fmtAt = (a: { hour: number; minute: number }) => `${String(a.hour).padStart(2, "0")}:${String(a.minute).padStart(2, "0")}`;
3117
- const openMemoryTarget = async (targetPath = root): Promise<"obsidian" | "fallback"> => {
3118
- // For articles, Obsidian's URL handler opens the exact file inside the
3119
- // vault. For the vault root, `open -a Obsidian <folder>` opens the folder
3120
- // as a vault. Both fall back to macOS `open` if Obsidian is unavailable.
3121
- if (targetPath !== root) {
3122
- const uri = `obsidian://open?path=${encodeURIComponent(targetPath)}`;
3123
- const byUri = Bun.spawn(["open", uri], { stdout: "ignore", stderr: "ignore" });
3124
- if ((await byUri.exited) === 0) return "obsidian";
3125
- } else {
3126
- const byApp = Bun.spawn(["open", "-a", "Obsidian", root], { stdout: "ignore", stderr: "ignore" });
3127
- if ((await byApp.exited) === 0) return "obsidian";
3128
- }
3129
- const fallback = Bun.spawn(["open", targetPath], { stdout: "ignore", stderr: "ignore" });
3130
- await fallback.exited;
3131
- return "fallback";
3132
- };
3133
-
3134
- // Words after `memory`, minus flags: [subcommand, ...query].
3135
- const words = argv.slice(1).filter((a) => !a.startsWith("-"));
3136
- const sub = (words[0] ?? "status").toLowerCase();
3137
- const rest = words.slice(1).join(" ").trim();
3138
-
3139
- // TTY-only line prompt; in a non-interactive shell returns the default so
3140
- // the wizard degrades to a non-destructive no-op rather than hanging.
3141
- const ask = async (q: string, dflt = ""): Promise<string> => {
3142
- if (!process.stdin.isTTY) return dflt;
3143
- const { createInterface } = await import("node:readline/promises");
3144
- const rl = createInterface({ input: process.stdin, output: process.stdout });
3145
- try {
3146
- return ((await rl.question(q)).trim() || dflt);
3147
- } finally {
3148
- rl.close();
3149
- }
3150
- };
3151
- const confirmYN = async (q: string, defaultYes: boolean): Promise<boolean> => {
3152
- const a = (await ask(`${q} ${defaultYes ? "[Y/n]" : "[y/N]"} `)).toLowerCase();
3153
- if (!a) return defaultYes;
3154
- return a === "y" || a === "yes";
3155
- };
3156
-
3157
- if (sub === "init" || sub === "setup") {
3158
- banner(pkg.version);
3159
- box([
3160
- `${style.accent("●")} ${style.bold("Personal memory for your local AI")}`,
3161
- "",
3162
- "A wiki of Markdown articles your local assistant reads to remember",
3163
- "your projects, people, and history across sessions. It's yours —",
3164
- `git-tracked at ${style.dim(root)}, editable in any tool, and it`,
3165
- "never leaves this machine.",
3166
- ]);
3167
- const sSetup = step("creating your wiki");
3168
- const res = await setupVault(root);
3169
- sSetup.done(res.alreadySetUp ? `already set up ${style.dim(`· ${root}`)}` : `wiki ready ${style.dim(`· ${root}`)}`);
3170
-
3171
- // Offer to seed from an existing vault (e.g. a lucien ~/Dreaming) so the
3172
- // assistant has real content to read on day one, before synthesis exists.
3173
- const { homedir } = await import("node:os");
3174
- const { join } = await import("node:path");
3175
- const { readdir } = await import("node:fs/promises");
3176
- const seedRoot = await ask(
3177
- `\n Seed from an existing wiki? Enter its path, or blank to skip ${style.dim(`[${join(homedir(), "Dreaming")}]`)}: `,
3178
- "",
3179
- );
3180
- if (seedRoot) {
3181
- const srcArticles = join(seedRoot.replace(/^~(?=$|[/\\])/, homedir()), "articles");
3182
- let count = 0;
3183
- try { count = (await readdir(srcArticles)).filter((n) => n.endsWith(".md")).length; } catch { count = 0; }
3184
- if (count === 0) {
3185
- console.log(style.dim(` no articles found under ${srcArticles} — skipping import`));
3186
- } else if (await confirmYN(` Import ${count} article(s) from ${srcArticles}?`, true)) {
3187
- const sImp = step(`importing ${count} article(s)`);
3188
- const imported = await importArticlesFrom(srcArticles, root);
3189
- await commitVault(root, `Import ${imported.length} article(s) from ${seedRoot}`);
3190
- sImp.done(`imported ${imported.length} article(s)`);
3191
- }
3192
- }
3193
-
3194
- // Offer to install the nightly synthesis job (a launchd agent). The job
3195
- // runs the real synthesis DAG (`mlx-bun memory synthesize`) — create new
3196
- // entity articles, then the editorial wikify sweep — so installing the
3197
- // schedule keeps your wiki current automatically. Persistent action →
3198
- // TTY-only, defaults to no.
3199
- let scheduled: string | null = null;
3200
- console.log(style.dim(
3201
- "\n Synthesis (turning your conversations into articles) runs as a nightly job;\n" +
3202
- " install it now so your wiki stays current automatically.",
3203
- ));
3204
- if (await confirmYN(" Install the nightly synthesis job (runs in the background)?", false)) {
3205
- const atRaw = await ask(` At what time? 24h HH:MM ${style.dim("[03:00]")}: `, "03:00");
3206
- const sSched = step(`installing nightly job at ${fmtAt(parseAt(atRaw))}`);
3207
- const r = await installSchedule({ at: atRaw });
3208
- scheduled = fmtAt(r.at);
3209
- sSched.done(
3210
- r.loaded
3211
- ? `nightly job installed ${style.dim(`· ${scheduled} · ${r.plistPath}`)}`
3212
- : `plist written ${style.dim(`· ${r.plistPath} · launchctl load failed (load it manually)`)}`,
3213
- );
3214
- }
3215
-
3216
- const st = await vaultStatus(root);
3217
- console.log();
3218
- box([
3219
- `${style.green("●")} ${style.bold("memory is set up")} ${style.dim(`· ${st.articleCount} article(s) · ${st.referenceCount} reference doc(s)`)}`,
3220
- "",
3221
- `chat now ${style.accent("mlx-bun pi")} ${style.dim("— the assistant now reads your memory automatically")}`,
3222
- `inspect ${style.accent("mlx-bun memory status")} ${style.dim("·")} ${style.accent("mlx-bun memory search <q>")}`,
3223
- `browse ${style.dim(`open ${root} in Obsidian, or any editor`)}`,
3224
- scheduled
3225
- ? `nightly ${style.green("scheduled")} ${style.dim(`· ${scheduled} daily · mlx-bun memory unschedule to undo`)}`
3226
- : `nightly ${style.dim("not scheduled · mlx-bun memory schedule to set it up")}`,
3227
- "",
3228
- style.dim("Synthesis (conversations → articles) runs the full DAG: create new entity"),
3229
- style.dim("articles, then an editorial wikify sweep. Run it now: mlx-bun memory synthesize."),
3230
- ]);
3231
- break;
3232
- }
3233
-
3234
- if (sub === "status") {
3235
- const st = await vaultStatus(root);
3236
- if (!st.exists) {
3237
- console.log(`no memory wiki yet at ${style.dim(root)}`);
3238
- console.log(`set one up: ${style.accent("mlx-bun memory init")}`);
3239
- break;
3240
- }
3241
- const sched = await scheduleStatus();
3242
- console.log();
3243
- box([
3244
- `${style.bold("memory")} ${style.dim(`· ${root}`)}`,
3245
- "",
3246
- `articles ${style.bold(String(st.articleCount))}`,
3247
- `reference ${style.bold(String(st.referenceCount))} ${style.dim("read-only docs")}`,
3248
- `git ${st.isGitRepo ? style.green("tracked") : style.dim("not a git repo")}`,
3249
- `synthesis ${style.green("available")} ${style.dim("· mlx-bun memory synthesize")}`,
3250
- `nightly ${
3251
- sched.installed
3252
- ? `${sched.loaded ? style.green("scheduled") : style.accent("installed (not loaded)")} ${style.dim(`· ${sched.plistPath}`)}`
3253
- : style.dim("not scheduled · mlx-bun memory schedule")
3254
- }`,
3255
- st.recentArticles.length
3256
- ? `recent ${st.recentArticles.slice(0, 5).map((r) => r.article).join(", ")}`
3257
- : `recent ${style.dim("none")}`,
3258
- ]);
3259
- break;
3260
- }
3261
-
3262
- if (sub === "open" || sub === "browse") {
3263
- const st = await vaultStatus(root);
3264
- if (!st.exists) {
3265
- console.log(`no memory wiki yet at ${style.dim(root)}`);
3266
- console.log(`set one up: ${style.accent("mlx-bun memory init")}`);
3267
- break;
3268
- }
3269
- let targetPath = root;
3270
- let label = "memory";
3271
- if (rest) {
3272
- try {
3273
- targetPath = await resolveArticlePath(root, rest);
3274
- label = rest.endsWith(".md") ? rest.slice(0, -3) : rest;
3275
- } catch (err) {
3276
- console.error(err instanceof Error ? err.message : String(err));
3277
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3278
- process.exit(1);
3279
- }
3280
- }
3281
- const where = await openMemoryTarget(targetPath);
3282
- console.log(
3283
- where === "obsidian"
3284
- ? `opened ${label} in Obsidian: ${targetPath}`
3285
- : `opened ${label}: ${targetPath}`,
3286
- );
3287
- break;
3288
- }
3289
-
3290
- // Independent, chronological, resumable STAGE WORKERS. Each pulls its own
3291
- // eligible work from the DB by state, processes a bounded batch, persists,
3292
- // and exits — so a user can run the four as four separate processes on
3293
- // different slices concurrently (GPU/memory allowing). `synthesize` (below)
3294
- // remains the FULL DAG; these are its decomposed pieces.
3295
- if (sub === "segment" || sub === "extract" || sub === "route" || sub === "synthesize-stage" || sub === "stage-synthesize") {
3296
- const stages = await import("./memory/stages");
3297
- const { MemoryStore } = await import("./memory/db");
3298
- const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
3299
- const convsRaw = opt("convs");
3300
- const convIds = convsRaw ? convsRaw.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
3301
- const onEvent = (e: import("./memory/pipeline").SynthesisEvent) => {
3302
- if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
3303
- else console.log(` ${e.message}`);
3304
- };
3305
- const store = new MemoryStore();
3306
- console.log();
3307
- try {
3308
- if (sub === "segment") {
3309
- const r = await stages.runSegmentStage(store, { convIds, limit, onEvent });
3310
- console.log(style.dim(`\n segment: ${r.valid} segmented, ${r.chunks} chunks, ${r.skipped} skipped, ${r.errored} errored`));
3311
- } else if (sub === "extract") {
3312
- const r = await stages.runExtractStage(store, { convIds, limit, onEvent });
3313
- console.log(style.dim(`\n extract: ${r.extracted} chunk(s) extracted, ${r.remaining} still pending`));
3314
- } else if (sub === "route") {
3315
- const r = await stages.runRouteStage(store, { convIds, onEvent });
3316
- console.log(style.dim(`\n route: ${r.decisions.length} entities — ${r.createEligible} create-eligible, ${r.captured.length} captured`));
3317
- } else {
3318
- const r = await stages.runSynthesizeStage(store, { root, convIds, limit, onEvent });
3319
- console.log(style.dim(`\n synthesize: ${r.created.length} created, ${r.patched.length} patched, ${r.skippedByGate.length} gated`));
3320
- }
3321
- } finally {
3322
- store.close();
3323
- }
3324
- break;
3325
- }
3326
-
3327
- // CROSS-LINK — the dedicated edge-building stage. Deterministic + idempotent:
3328
- // inline-link first mentions of other articles + rebuild each ## See also from
3329
- // mentions + co-occurrence. No model required.
3330
- if (sub === "link") {
3331
- const { runLinkStage } = await import("./memory/crosslink");
3332
- const { MemoryStore } = await import("./memory/db");
3333
- const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
3334
- const store = new MemoryStore();
3335
- console.log();
3336
- try {
3337
- const r = await runLinkStage(store, {
3338
- root,
3339
- limit,
3340
- onEvent: (e) => console.log(e.type === "stage" ? ` ${style.dim("·")} ${e.message}` : ` ${e.message}`),
3341
- });
3342
- console.log(
3343
- style.dim(
3344
- `\n link: ${r.linked.length} article(s) linked · ${r.mentionEdges} mention edge(s) · ${r.skippedByGate.length} gated`,
3345
- ),
3346
- );
3347
- } finally {
3348
- store.close();
3349
- }
3350
- break;
3351
- }
3352
-
3353
- if (sub === "synthesize" || sub === "pipeline" || sub === "all") {
3354
- const { runSynthesis } = await import("./memory/pipeline");
3355
- const dryRun = flag("dry-run");
3356
- console.log();
3357
- const summary = await runSynthesis(
3358
- { since: opt("since") ?? undefined, model: opt("model") ?? undefined, dryRun },
3359
- (e) => {
3360
- if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
3361
- else console.log(` ${e.message}`);
3362
- },
3363
- );
3364
- console.log(style.dim(`\n ${summary.note}`));
3365
- break;
3366
- }
3367
-
3368
- if (sub === "schedule") {
3369
- const atRaw = opt("at") ?? "03:00";
3370
- const r = await installSchedule({ at: atRaw });
3371
- console.log();
3372
- box([
3373
- r.loaded
3374
- ? `${style.green("●")} ${style.bold("nightly synthesis scheduled")} ${style.dim(`· ${fmtAt(r.at)} daily`)}`
3375
- : `${style.accent("●")} ${style.bold("plist written, but launchctl load failed")}`,
3376
- "",
3377
- `plist ${style.dim(r.plistPath)}`,
3378
- `runs ${style.dim("mlx-bun memory synthesize (full DAG: create + wikify sweep)")}`,
3379
- `undo ${style.accent("mlx-bun memory unschedule")}`,
3380
- ]);
3381
- break;
3382
- }
3383
-
3384
- if (sub === "unschedule") {
3385
- const removed = await removeSchedule();
3386
- console.log(removed ? "nightly synthesis job removed" : "no nightly job was installed");
3387
- break;
3388
- }
3389
-
3390
- if (sub === "list") {
3391
- const docs = await listMemoryDocuments(root);
3392
- if (docs.length === 0) { console.log("no articles or reference docs yet"); break; }
3393
- for (const s of docs) console.log(` ${s}`);
3394
- const refs = docs.filter((s) => s.startsWith("Reference/")).length;
3395
- console.log(style.dim(`\n ${docs.length - refs} article(s), ${refs} read-only reference doc(s)`));
3396
- break;
3397
- }
3398
-
3399
- if (sub === "search") {
3400
- if (!rest) { console.error("usage: mlx-bun memory search <query>"); process.exit(1); }
3401
- const { summaries, hits } = await searchArticles(root, rest, { limit: 12 });
3402
- if (summaries.length === 0) { console.log(`no articles match "${rest}"`); break; }
3403
- console.log();
3404
- for (const s of summaries.slice(0, 15)) {
3405
- const terms = s.matched_terms?.length ? style.dim(` [${s.matched_terms.join(", ")}]`) : "";
3406
- console.log(` ${style.bold(s.article)} ${style.dim(`· ${s.occurrences} hit(s)`)}${terms}`);
3407
- }
3408
- if (hits.length) {
3409
- console.log(style.dim("\n sample lines:"));
3410
- for (const h of hits.slice(0, 6)) {
3411
- const where = h.anchor ? `${h.article}#${h.anchor}` : h.article;
3412
- console.log(` ${style.dim(`${where}:${h.line}`)} ${h.excerpt}`);
3413
- }
3414
- }
3415
- console.log(style.dim(`\n read one: mlx-bun memory read ${summaries[0]!.article}`));
3416
- break;
3417
- }
3418
-
3419
- if (sub === "toc") {
3420
- if (!rest) { console.error("usage: mlx-bun memory toc <article>"); process.exit(1); }
3421
- try {
3422
- const { content } = await readArticle(root, rest);
3423
- const toc = parseToc(content);
3424
- if (toc.length === 0) { console.log("no headings"); break; }
3425
- for (const h of toc) console.log(`${" ".repeat(Math.max(0, h.depth - 1))}- ${h.title} ${style.dim(`#${h.anchor}`)}`);
3426
- } catch (err) {
3427
- console.error(err instanceof Error ? err.message : String(err));
3428
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3429
- process.exit(1);
3430
- }
3431
- break;
3432
- }
3433
-
3434
- if (sub === "section") {
3435
- const [article, anchorRaw] = words.slice(1);
3436
- const anchor = anchorRaw?.replace(/^#/, "");
3437
- if (!article || !anchor) { console.error("usage: mlx-bun memory section <article> <anchor>"); process.exit(1); }
3438
- try {
3439
- const { content } = await readArticle(root, article);
3440
- const section = extractSection(content, anchor);
3441
- if (!section) { console.error(`no section #${anchor} in ${article}`); console.error(`try: ${style.accent(`mlx-bun memory toc ${article}`)}`); process.exit(1); }
3442
- console.log(section);
3443
- } catch (err) {
3444
- console.error(err instanceof Error ? err.message : String(err));
3445
- console.error(`try: ${style.accent(`mlx-bun memory search ${article}`)}`);
3446
- process.exit(1);
3447
- }
3448
- break;
3449
- }
3450
-
3451
- if (sub === "links") {
3452
- if (!rest) { console.error("usage: mlx-bun memory links <article>"); process.exit(1); }
3453
- try {
3454
- const { outbound, inbound } = await getArticleLinks(root, rest);
3455
- console.log(`${style.bold("outbound")}:`);
3456
- console.log(outbound.length ? outbound.map((s) => ` ${s}`).join("\n") : " (none)");
3457
- console.log(`\n${style.bold("inbound")}:`);
3458
- console.log(inbound.length ? inbound.map((s) => ` ${s}`).join("\n") : " (none)");
3459
- } catch (err) {
3460
- console.error(err instanceof Error ? err.message : String(err));
3461
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3462
- process.exit(1);
3463
- }
3464
- break;
3465
- }
3466
-
3467
- if (sub === "read") {
3468
- if (!rest) { console.error("usage: mlx-bun memory read <article>"); process.exit(1); }
3469
- try {
3470
- const { content } = await readArticle(root, rest);
3471
- console.log(content);
3472
- } catch (err) {
3473
- console.error(err instanceof Error ? err.message : String(err));
3474
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3475
- process.exit(1);
3476
- }
3477
- break;
3478
- }
3479
-
3480
-
3481
- // Unknown subcommand: help on stdout, but a FAILURE exit — scripts and
3482
- // launchd jobs must be able to detect typos (printHelp would exit 0).
3483
- console.error(`unknown: mlx-bun memory ${sub}`);
3484
- console.log(renderHelp(HELP.memory!));
3485
- process.exit(1);
3486
- }
3487
-
3488
- default:
3489
- printHelp(cmd);
3490
- }