mlx-bun 0.4.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (601) hide show
  1. package/README.md +1453 -88
  2. package/THIRD_PARTY_NOTICES.md +882 -0
  3. package/bin/mlx-bun.convert.mjs +4 -0
  4. package/bin/mlx-bun.fuse.mjs +4 -0
  5. package/bin/mlx-bun.generate.mjs +4 -0
  6. package/bin/mlx-bun.lora.mjs +4 -0
  7. package/bin/mlx-bun.mjs +13 -21
  8. package/bin/mlx-bun.server.mjs +4 -0
  9. package/bin/mlx-bun.upload.mjs +4 -0
  10. package/dist/web/app.js +9584 -0
  11. package/package.json +50 -53
  12. package/src/cli/ane-install.ts +76 -0
  13. package/src/cli/args.ts +179 -0
  14. package/src/cli/engine-entry.ts +68 -0
  15. package/src/cli/inference.ts +214 -0
  16. package/src/cli/job-entry.ts +97 -0
  17. package/src/cli/library-host.ts +117 -0
  18. package/src/cli/main.ts +57 -0
  19. package/src/cli/memory-engine.ts +163 -0
  20. package/src/cli/memory.ts +36 -0
  21. package/src/cli/mlx-lm-aliases.ts +309 -0
  22. package/src/cli/model-choice.ts +33 -0
  23. package/src/cli/model-selection.ts +84 -0
  24. package/src/cli/module-host.ts +11 -0
  25. package/src/cli/module-verbs.ts +50 -0
  26. package/src/cli/numerical-policy.ts +12 -0
  27. package/src/cli/perplexity.ts +89 -0
  28. package/src/cli/serve-host.ts +423 -0
  29. package/src/cli/serve-isolated.ts +217 -0
  30. package/src/cli/serve-options.ts +106 -0
  31. package/src/cli/serve-state.ts +205 -0
  32. package/src/cli/serve.ts +500 -0
  33. package/src/cli/served-model-host.ts +102 -0
  34. package/src/cli/server-entry.ts +148 -0
  35. package/src/cli/serving-unit.ts +177 -0
  36. package/src/{tui.ts → cli/terminal.ts} +13 -8
  37. package/src/cli/worker-adapters.ts +34 -0
  38. package/src/cli/worker-entry.ts +272 -0
  39. package/src/cli/worker-unit.ts +179 -0
  40. package/src/engine/adapter-operation.ts +26 -0
  41. package/src/engine/cache-services.ts +114 -0
  42. package/src/engine/completion.ts +39 -0
  43. package/src/engine/generation-gateway.ts +358 -0
  44. package/src/engine/index.ts +59 -0
  45. package/src/engine/kv-budget.ts +120 -0
  46. package/src/engine/media-preparation.ts +232 -0
  47. package/src/engine/model-binding.ts +24 -0
  48. package/src/engine/model-host.ts +513 -0
  49. package/src/engine/model-serving.ts +89 -0
  50. package/src/{serve → engine}/preparation.ts +10 -8
  51. package/src/{serve → engine}/session-completion-engine.ts +6 -6
  52. package/src/engine/telemetry.ts +109 -0
  53. package/src/hub/downloads.ts +116 -0
  54. package/src/jobs/db.ts +16 -7
  55. package/src/jobs/events.ts +29 -0
  56. package/src/jobs/executable.ts +5 -0
  57. package/src/jobs/host.ts +51 -0
  58. package/src/jobs/in-process.ts +55 -0
  59. package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
  60. package/src/jobs/runner.ts +150 -231
  61. package/src/jobs/service.ts +75 -0
  62. package/src/jobs/sse.ts +59 -23
  63. package/src/jobs/worker-memory.ts +46 -0
  64. package/src/jobs/worker-process.ts +163 -0
  65. package/src/jobs/worker-supervisor.ts +258 -0
  66. package/src/modules.ts +58 -0
  67. package/src/publishing/catalog-hub.ts +18 -0
  68. package/src/publishing/credentials.ts +31 -0
  69. package/src/publishing/upload.ts +37 -0
  70. package/src/residency/leased-adapters.ts +21 -0
  71. package/src/residency/local-records.ts +16 -0
  72. package/src/residency/model-residency.ts +405 -0
  73. package/src/residency/record-index.ts +28 -0
  74. package/src/residency/resident-estimate.ts +50 -0
  75. package/src/{anthropic.ts → server/anthropic.ts} +3 -81
  76. package/src/server/cache-routes.ts +36 -0
  77. package/src/{serve → server}/chat-request.ts +12 -10
  78. package/src/{serve → server}/chat-stage.ts +27 -22
  79. package/src/{client.ts → server/client.ts} +28 -2
  80. package/src/{serve → server}/completion-executor.ts +42 -42
  81. package/src/server/decision-body.ts +55 -0
  82. package/src/server/decision-routes.ts +134 -0
  83. package/src/server/decision-wire.ts +228 -0
  84. package/src/{serve → server}/discovery-routes.ts +42 -83
  85. package/src/server/generated-token-history.ts +57 -0
  86. package/src/{serve → server}/http.ts +25 -13
  87. package/src/{serve → server}/inference-request.ts +5 -5
  88. package/src/server/job-routes.ts +20 -0
  89. package/src/server/media-prompt.ts +63 -0
  90. package/src/server/memory-completion-client.ts +46 -0
  91. package/src/server/model-routes.ts +192 -0
  92. package/src/{serve → server}/openai-wire.ts +3 -2
  93. package/src/{serve → server}/pipeline.ts +1 -1
  94. package/src/server/prompt-builder.ts +13 -0
  95. package/src/server/prompt-contracts.ts +17 -0
  96. package/src/server/proxy-routes.ts +187 -0
  97. package/src/server/publishing-routes.ts +41 -0
  98. package/src/{serve → server}/request-plan.ts +8 -8
  99. package/src/{serve → server}/request-prep.ts +82 -68
  100. package/src/{serve → server}/responses-client.ts +30 -13
  101. package/src/{responses.ts → server/responses.ts} +12 -93
  102. package/src/server/routes.ts +205 -0
  103. package/src/server/start.ts +72 -0
  104. package/src/server/status-routes.ts +108 -0
  105. package/src/{serve → server}/text-completion-stage.ts +25 -4
  106. package/src/server/text-prompt.ts +12 -0
  107. package/src/{serve → server}/token-streams.ts +36 -38
  108. package/src/server/worker-routes.ts +311 -0
  109. package/src/storage/paths.ts +61 -0
  110. package/src/web/assets.ts +37 -0
  111. package/src/web/{src → browser}/api.ts +5 -23
  112. package/src/web/browser/chat-host.ts +17 -0
  113. package/src/web/browser/installed-panels.ts +6 -0
  114. package/src/web/browser/main.ts +55 -0
  115. package/src/web/browser/markdown.ts +279 -0
  116. package/src/web/browser/memory-panel.ts +9 -0
  117. package/src/web/browser/palette.ts +64 -0
  118. package/src/web/browser/protocol.ts +13 -0
  119. package/src/web/browser/shell.ts +467 -0
  120. package/src/web/{src → browser}/status.ts +32 -36
  121. package/src/web/build.ts +51 -0
  122. package/src/web/public/app.html +1054 -0
  123. package/src/web/{sw.js → public/sw.js} +4 -3
  124. package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
  125. package/src/web/{vendor → public/vendor}/hljs.js +2 -2
  126. package/THIRD_PARTY_LICENSES.md +0 -51
  127. package/docs/README.md +0 -94
  128. package/docs/archive/hlg-sampling.md +0 -350
  129. package/docs/archive/investigations/audio-input-plan.md +0 -356
  130. package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
  131. package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
  132. package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
  133. package/docs/archive/investigations/curve-bisector-routing.md +0 -123
  134. package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
  135. package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
  136. package/docs/archive/investigations/dspark-handoff.md +0 -224
  137. package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
  138. package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
  139. package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
  140. package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
  141. package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
  142. package/docs/archive/investigations/lab-build-journal.md +0 -157
  143. package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
  144. package/docs/archive/investigations/openjarvis-fit.md +0 -198
  145. package/docs/archive/investigations/openwiki-evaluation.md +0 -249
  146. package/docs/archive/investigations/optimization_plan.md +0 -346
  147. package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
  148. package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
  149. package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
  150. package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
  151. package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
  152. package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
  153. package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
  154. package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
  155. package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
  156. package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
  157. package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
  158. package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
  159. package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
  160. package/docs/archive/planning/chat-ui-vision.md +0 -50
  161. package/docs/archive/planning/chunk-task-roadmap.md +0 -77
  162. package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
  163. package/docs/archive/planning/journal.md +0 -316
  164. package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
  165. package/docs/archive/planning/pi-integration-review.md +0 -178
  166. package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
  167. package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
  168. package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
  169. package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
  170. package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
  171. package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
  172. package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
  173. package/docs/archive/planning/web-ui-pass-plan.md +0 -213
  174. package/docs/archive/planning/website-readme-pass-plan.md +0 -224
  175. package/docs/design/batching.md +0 -1841
  176. package/docs/design/decode-speed-program.md +0 -2349
  177. package/docs/design/dreaming-nightly-pipeline.md +0 -684
  178. package/docs/design/generic-model-support.md +0 -705
  179. package/docs/design/kv-cache.md +0 -824
  180. package/docs/design/orpo-training.md +0 -680
  181. package/docs/design/speculative-decoding.md +0 -1990
  182. package/docs/design/turboquant.md +0 -2212
  183. package/docs/design/unified-engine-frontier-plan.md +0 -1880
  184. package/docs/design/web-chat-redesign.md +0 -695
  185. package/docs/planning/PRODUCT_ROADMAP.md +0 -594
  186. package/docs/planning/ResearchTopics.md +0 -119
  187. package/docs/planning/release-notes-v0.4.0.md +0 -45
  188. package/docs/planning/video-series.md +0 -421
  189. package/docs/reference/benchmarks.md +0 -3613
  190. package/docs/reference/cli.md +0 -585
  191. package/docs/reference/distribution.md +0 -355
  192. package/docs/reference/environment.md +0 -398
  193. package/docs/reference/glossary.md +0 -39
  194. package/docs/reference/library-api.md +0 -674
  195. package/docs/reference/memory.md +0 -260
  196. package/docs/reference/models.md +0 -334
  197. package/docs/reference/server-api.md +0 -1775
  198. package/docs/reference/server-config.md +0 -846
  199. package/docs/reference/training.md +0 -462
  200. package/docs/reference/troubleshooting.md +0 -158
  201. package/src/admission-plan.ts +0 -18
  202. package/src/audio/conformer.ts +0 -647
  203. package/src/audio/decode.ts +0 -133
  204. package/src/audio/features.ts +0 -202
  205. package/src/audio/transcode.ts +0 -67
  206. package/src/backends/mlx/assistant-target.ts +0 -46
  207. package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
  208. package/src/backends/mlx/autoregressive.ts +0 -108
  209. package/src/backends/mlx/batch-group.ts +0 -1790
  210. package/src/backends/mlx/cache-attachment.ts +0 -67
  211. package/src/backends/mlx/cache-layout.ts +0 -31
  212. package/src/backends/mlx/checkpoint-state.ts +0 -53
  213. package/src/backends/mlx/continuation-persistence.ts +0 -106
  214. package/src/backends/mlx/continuation-request.ts +0 -43
  215. package/src/backends/mlx/continuation.ts +0 -77
  216. package/src/backends/mlx/diffusion.ts +0 -39
  217. package/src/backends/mlx/draft-target.ts +0 -32
  218. package/src/backends/mlx/evaluation.ts +0 -105
  219. package/src/backends/mlx/fill-append.ts +0 -45
  220. package/src/backends/mlx/full-kv-row-append.ts +0 -32
  221. package/src/backends/mlx/full-kv-row-donor.ts +0 -54
  222. package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
  223. package/src/backends/mlx/gateway-binding.ts +0 -130
  224. package/src/backends/mlx/graph.ts +0 -38
  225. package/src/backends/mlx/kv-codec.ts +0 -9
  226. package/src/backends/mlx/kv-maintenance.ts +0 -133
  227. package/src/backends/mlx/kv-tensor-rows.ts +0 -212
  228. package/src/backends/mlx/memory-client.ts +0 -189
  229. package/src/backends/mlx/methods.ts +0 -90
  230. package/src/backends/mlx/model-prompt.ts +0 -164
  231. package/src/backends/mlx/model-serving.ts +0 -79
  232. package/src/backends/mlx/prefill-cohort.ts +0 -89
  233. package/src/backends/mlx/prefill-rows.ts +0 -200
  234. package/src/backends/mlx/prefill.ts +0 -33
  235. package/src/backends/mlx/projected-context-rows.ts +0 -72
  236. package/src/backends/mlx/request-policy.ts +0 -20
  237. package/src/backends/mlx/request-state-policy.ts +0 -41
  238. package/src/backends/mlx/rollback.ts +0 -89
  239. package/src/backends/mlx/row-sampling.ts +0 -49
  240. package/src/backends/mlx/serial-executor.ts +0 -242
  241. package/src/backends/mlx/speculative-group.ts +0 -359
  242. package/src/backends/mlx/speculative-round.ts +0 -161
  243. package/src/backends/mlx/speculative.ts +0 -114
  244. package/src/backends/mlx/state-rows.ts +0 -104
  245. package/src/backends/mlx/state-views.ts +0 -46
  246. package/src/backends/mlx/target-layout-capability.ts +0 -35
  247. package/src/backends/mlx/text-engine.ts +0 -73
  248. package/src/backends/mlx/transitioning-kv-rows.ts +0 -165
  249. package/src/chat-template.ts +0 -382
  250. package/src/cli.ts +0 -3168
  251. package/src/config.ts +0 -399
  252. package/src/contracts/completion.ts +0 -16
  253. package/src/contracts/execution.ts +0 -60
  254. package/src/contracts/generation.ts +0 -103
  255. package/src/contracts/host.ts +0 -7
  256. package/src/contracts/pi-provider.ts +0 -30
  257. package/src/contracts/pi.ts +0 -259
  258. package/src/contracts/prefix-cache.ts +0 -23
  259. package/src/contracts/resources.ts +0 -25
  260. package/src/contracts/sampling.ts +0 -23
  261. package/src/contracts/scheduling.ts +0 -45
  262. package/src/contracts/tsconfig.json +0 -14
  263. package/src/dataset/generators.ts +0 -1087
  264. package/src/dataset/index.ts +0 -12
  265. package/src/dataset/job.ts +0 -81
  266. package/src/dataset/llm.ts +0 -71
  267. package/src/dataset/registry.ts +0 -528
  268. package/src/diffusion/diffusion-generate.ts +0 -507
  269. package/src/download.ts +0 -505
  270. package/src/embed.ts +0 -77
  271. package/src/engine/admission.ts +0 -59
  272. package/src/engine/cancellation.ts +0 -38
  273. package/src/engine/engine.ts +0 -56
  274. package/src/engine/execution-coordinator.ts +0 -70
  275. package/src/engine/execution-plan.ts +0 -53
  276. package/src/engine/resources.ts +0 -53
  277. package/src/engine/scheduler.ts +0 -72
  278. package/src/engine/session.ts +0 -247
  279. package/src/eval/README.md +0 -130
  280. package/src/eval/ifeval.ts +0 -81
  281. package/src/eval/kl-prompts.ts +0 -29
  282. package/src/eval/kl.ts +0 -590
  283. package/src/eval/perplexity.ts +0 -164
  284. package/src/eval/quality-db.ts +0 -92
  285. package/src/eval/reference-logits.ts +0 -207
  286. package/src/eval/runner.ts +0 -124
  287. package/src/eval/score.ts +0 -49
  288. package/src/eval/tasks/bfcl.ts +0 -535
  289. package/src/eval/tasks/gsm8k.ts +0 -98
  290. package/src/eval/tasks/hashhop.ts +0 -139
  291. package/src/eval/tasks/humaneval.ts +0 -275
  292. package/src/eval/tasks/ifeval.ts +0 -525
  293. package/src/eval/tasks/mmlu.ts +0 -157
  294. package/src/eval/ultrafeedback.ts +0 -60
  295. package/src/evaldb.ts +0 -199
  296. package/src/expert-io.ts +0 -393
  297. package/src/expert-offload-build.ts +0 -77
  298. package/src/expert-offload.ts +0 -76
  299. package/src/expert-residency.ts +0 -1134
  300. package/src/expert-usage.ts +0 -415
  301. package/src/fill/echo-index.ts +0 -188
  302. package/src/fill/fill-session.ts +0 -452
  303. package/src/fill/proposal.ts +0 -68
  304. package/src/fill/schema-rows.ts +0 -506
  305. package/src/fill/tool-boundary.ts +0 -103
  306. package/src/fit.ts +0 -249
  307. package/src/generate.ts +0 -1346
  308. package/src/grammar.ts +0 -591
  309. package/src/harness-pi.ts +0 -184
  310. package/src/hf-push.ts +0 -628
  311. package/src/hub-rest.ts +0 -242
  312. package/src/index.ts +0 -63
  313. package/src/inference/denoising.ts +0 -17
  314. package/src/inference/draft-acceptance.ts +0 -41
  315. package/src/inference/draft-output.ts +0 -49
  316. package/src/inference/graph.ts +0 -33
  317. package/src/inference/prefill.ts +0 -39
  318. package/src/inference/rollback.ts +0 -7
  319. package/src/jobs/index.ts +0 -31
  320. package/src/jobs/job-entry.ts +0 -107
  321. package/src/jobs/task-client.ts +0 -19
  322. package/src/jobs/types.ts +0 -2
  323. package/src/kv-scheme.ts +0 -257
  324. package/src/kv-store.ts +0 -1099
  325. package/src/lab/README.md +0 -18
  326. package/src/lab/curve/curve-designer.html +0 -303
  327. package/src/lab/curve/curve-sampler.ts +0 -122
  328. package/src/lab/expert-trace/expert-trace.ts +0 -69
  329. package/src/lab/paged-kv/paged-kv-rows.ts +0 -83
  330. package/src/lab/paged-kv/paged-kv.ts +0 -270
  331. package/src/library.ts +0 -40
  332. package/src/lora.ts +0 -408
  333. package/src/md-modules.d.ts +0 -49
  334. package/src/media-fetch.ts +0 -442
  335. package/src/memory/article.ts +0 -558
  336. package/src/memory/chunk-validation.ts +0 -150
  337. package/src/memory/chunk.ts +0 -318
  338. package/src/memory/cluster.ts +0 -351
  339. package/src/memory/coverage.ts +0 -134
  340. package/src/memory/crosslink.ts +0 -0
  341. package/src/memory/db.ts +0 -253
  342. package/src/memory/entity.ts +0 -203
  343. package/src/memory/gate.ts +0 -187
  344. package/src/memory/history.ts +0 -152
  345. package/src/memory/ingest.ts +0 -335
  346. package/src/memory/model.ts +0 -183
  347. package/src/memory/normalize.ts +0 -671
  348. package/src/memory/parse.ts +0 -22
  349. package/src/memory/pipeline.ts +0 -275
  350. package/src/memory/prompts.ts +0 -65
  351. package/src/memory/query.ts +0 -334
  352. package/src/memory/reconcile.ts +0 -734
  353. package/src/memory/redirect.ts +0 -108
  354. package/src/memory/reindex.ts +0 -325
  355. package/src/memory/resolve.ts +0 -307
  356. package/src/memory/rest.ts +0 -245
  357. package/src/memory/route.ts +0 -272
  358. package/src/memory/schedule.ts +0 -165
  359. package/src/memory/stages.ts +0 -777
  360. package/src/memory/synthesize.ts +0 -1638
  361. package/src/memory/tools.ts +0 -749
  362. package/src/memory/vault.ts +0 -1194
  363. package/src/memory/wikify.ts +0 -1064
  364. package/src/mlx/array.ts +0 -388
  365. package/src/mlx/autograd.ts +0 -304
  366. package/src/mlx/checkpoint.ts +0 -149
  367. package/src/mlx/compile.ts +0 -168
  368. package/src/mlx/custom-vjp.ts +0 -188
  369. package/src/mlx/ffi.ts +0 -512
  370. package/src/mlx/materialize.ts +0 -23
  371. package/src/mlx/metal-kernel.ts +0 -159
  372. package/src/mlx/ops.ts +0 -1021
  373. package/src/mlx/quantized-rows.ts +0 -36
  374. package/src/mlx/token-bitmask.ts +0 -52
  375. package/src/mlx/turboquant-kv-decode.ts +0 -82
  376. package/src/mlx/turboquant-kv-inverse.ts +0 -122
  377. package/src/mlx/turboquant-ops.ts +0 -861
  378. package/src/mlx/turboquant-tables.ts +0 -237
  379. package/src/mmap.ts +0 -90
  380. package/src/model/artifact-identity.ts +0 -73
  381. package/src/model/batched-kv.ts +0 -235
  382. package/src/model/batched-mask.ts +0 -279
  383. package/src/model/batched-quant.ts +0 -212
  384. package/src/model/batched-quantized-kv.ts +0 -144
  385. package/src/model/batched-rotating-quant.ts +0 -353
  386. package/src/model/batched-rotating-state.ts +0 -149
  387. package/src/model/batched-rotating.ts +0 -338
  388. package/src/model/batched-row-storage.ts +0 -179
  389. package/src/model/batched-ssm.ts +0 -52
  390. package/src/model/batched-turboquant-kv.ts +0 -93
  391. package/src/model/compiled-decode.ts +0 -712
  392. package/src/model/decoded-kv-donor.ts +0 -12
  393. package/src/model/delayed-quantized-kv.ts +0 -72
  394. package/src/model/delayed-rotating-quantized-kv.ts +0 -93
  395. package/src/model/delayed-turboquant-kv.ts +0 -55
  396. package/src/model/diffusion-gemma.ts +0 -1026
  397. package/src/model/factory.ts +0 -189
  398. package/src/model/fingerprint.ts +0 -33
  399. package/src/model/flash-attention.ts +0 -694
  400. package/src/model/full-prefill-padding.ts +0 -56
  401. package/src/model/full-prefill-row.ts +0 -105
  402. package/src/model/gemma4-base.ts +0 -2351
  403. package/src/model/gemma4.ts +0 -1315
  404. package/src/model/generated/gemma4-12b.ts +0 -429
  405. package/src/model/generated/gemma4-26b.ts +0 -448
  406. package/src/model/generated/gemma4-e4b.ts +0 -717
  407. package/src/model/generated/index.ts +0 -18
  408. package/src/model/glm52-atlas.ts +0 -511
  409. package/src/model/glm52-cache.ts +0 -792
  410. package/src/model/glm52-config.ts +0 -262
  411. package/src/model/glm52-container.ts +0 -305
  412. package/src/model/glm52-coupling.ts +0 -388
  413. package/src/model/glm52-dsa.ts +0 -606
  414. package/src/model/glm52-expert-layout.ts +0 -253
  415. package/src/model/glm52-layout.ts +0 -152
  416. package/src/model/glm52-memory.ts +0 -500
  417. package/src/model/glm52-mla.ts +0 -792
  418. package/src/model/glm52-moe.ts +0 -223
  419. package/src/model/glm52-pilot.ts +0 -537
  420. package/src/model/glm52-reference.ts +0 -445
  421. package/src/model/glm52-residency.ts +0 -557
  422. package/src/model/glm52-resident-weights.ts +0 -135
  423. package/src/model/glm52-streamed-experts.ts +0 -453
  424. package/src/model/glm52-streamed-metal.ts +0 -610
  425. package/src/model/glm52-weights.ts +0 -503
  426. package/src/model/glm52.ts +0 -1132
  427. package/src/model/implementation.ts +0 -69
  428. package/src/model/kv-attention-view.ts +0 -67
  429. package/src/model/minicpm5.ts +0 -376
  430. package/src/model/profile.ts +0 -443
  431. package/src/model/qwen-append.ts +0 -12
  432. package/src/model/qwen-conv.ts +0 -52
  433. package/src/model/qwen3-delta-state.ts +0 -74
  434. package/src/model/qwen3-delta.ts +0 -451
  435. package/src/model/qwen3-moe.ts +0 -404
  436. package/src/model/qwen3-mrope.ts +0 -252
  437. package/src/model/qwen3.ts +0 -277
  438. package/src/model/qwen3_5-checkpoint.ts +0 -164
  439. package/src/model/qwen3_5.ts +0 -819
  440. package/src/model/rotating-kv-donor.ts +0 -35
  441. package/src/model/rotating-kv-layout.ts +0 -262
  442. package/src/model/rotating-row-transaction.ts +0 -35
  443. package/src/model/speculative-rotating-kv.ts +0 -101
  444. package/src/model/ssm-prefill-padding.ts +0 -39
  445. package/src/model/support.ts +0 -102
  446. package/src/model/trellis-balanced-scatter.ts +0 -51
  447. package/src/model/trellis-linear.ts +0 -672
  448. package/src/model/trellis-shared-m.ts +0 -151
  449. package/src/model/trellis-shared-scatter.ts +0 -144
  450. package/src/model/trellis-splitk-prefill.ts +0 -158
  451. package/src/model/trellis-tiled-prefill.ts +0 -136
  452. package/src/model/trellis-vector-expand.ts +0 -62
  453. package/src/model/trellis-wide-prefill.ts +0 -94
  454. package/src/model/turboquant-codec.ts +0 -83
  455. package/src/model/universal/archs.ts +0 -332
  456. package/src/model/universal/dense.ts +0 -612
  457. package/src/model/universal/modules.ts +0 -170
  458. package/src/model/universal/rope.ts +0 -381
  459. package/src/native/expert_io.c +0 -523
  460. package/src/native/frame_extract.swift +0 -140
  461. package/src/native-pack.ts +0 -162
  462. package/src/pi-provider.ts +0 -53
  463. package/src/pi-session.ts +0 -81
  464. package/src/pi-terminal.ts +0 -274
  465. package/src/pi-web.ts +0 -1932
  466. package/src/preflight.ts +0 -158
  467. package/src/prompt-cache.ts +0 -460
  468. package/src/quantize/allocator.ts +0 -627
  469. package/src/quantize/atomic-output.ts +0 -38
  470. package/src/quantize/calibration.ts +0 -181
  471. package/src/quantize/config-writer.ts +0 -145
  472. package/src/quantize/index.ts +0 -84
  473. package/src/quantize/job.ts +0 -162
  474. package/src/quantize/quantizer.ts +0 -666
  475. package/src/quantize/rotate.ts +0 -603
  476. package/src/quantize/safetensors-writer.ts +0 -249
  477. package/src/quantize/sensitivity.ts +0 -216
  478. package/src/quantize/trellis.ts +0 -586
  479. package/src/quantize/weight-transform.ts +0 -292
  480. package/src/registry.ts +0 -602
  481. package/src/runtime-config.ts +0 -81
  482. package/src/safetensors.ts +0 -158
  483. package/src/sampler.ts +0 -1011
  484. package/src/serve/admin-routes.ts +0 -184
  485. package/src/serve/assets/curve-terrain.html +0 -105
  486. package/src/serve/assets/training-inference-map.html +0 -575
  487. package/src/serve/aux-routes.ts +0 -132
  488. package/src/serve/batch-scheduler.ts +0 -8
  489. package/src/serve/checkpoint-identity.ts +0 -64
  490. package/src/serve/generation-gateway.ts +0 -496
  491. package/src/serve/isolate.ts +0 -609
  492. package/src/serve/kv-budget.ts +0 -14
  493. package/src/serve/lab-routes.ts +0 -341
  494. package/src/serve/lane-registry.ts +0 -45
  495. package/src/serve/model-admin-routes.ts +0 -155
  496. package/src/serve/model-binding.ts +0 -43
  497. package/src/serve/model-host.ts +0 -511
  498. package/src/serve/parent-application.ts +0 -56
  499. package/src/serve/prompt-response-trace.ts +0 -147
  500. package/src/serve/response-lease.ts +0 -24
  501. package/src/serve/session-search.ts +0 -326
  502. package/src/serve/static-routes.ts +0 -101
  503. package/src/server.ts +0 -1314
  504. package/src/spec/assistant-rows.ts +0 -92
  505. package/src/spec/assistant-source.ts +0 -126
  506. package/src/spec/cache-identity.ts +0 -22
  507. package/src/spec/deepspec-rows.ts +0 -29
  508. package/src/spec/deepspec-source.ts +0 -145
  509. package/src/spec/dflash-source.ts +0 -149
  510. package/src/spec/drafter.ts +0 -304
  511. package/src/spec/dspark/ab-stats.ts +0 -136
  512. package/src/spec/dspark/calibration.ts +0 -112
  513. package/src/spec/dspark/data-dflash.ts +0 -156
  514. package/src/spec/dspark/data.ts +0 -201
  515. package/src/spec/dspark/deepspec-module.ts +0 -776
  516. package/src/spec/dspark/generate-dflash.ts +0 -245
  517. package/src/spec/dspark/generate.ts +0 -290
  518. package/src/spec/dspark/loader.ts +0 -23
  519. package/src/spec/dspark/loss.ts +0 -213
  520. package/src/spec/dspark/module-dflash.ts +0 -560
  521. package/src/spec/dspark/module.ts +0 -540
  522. package/src/spec/dspark/quantize-drafter.ts +0 -91
  523. package/src/spec/dspark/sample.ts +0 -112
  524. package/src/spec/generate.ts +0 -276
  525. package/src/spec/glm52-mtp-graph.ts +0 -29
  526. package/src/spec/glm52-mtp-rows.ts +0 -132
  527. package/src/spec/glm52-mtp-source.ts +0 -317
  528. package/src/spec/glm52-mtp-state.ts +0 -33
  529. package/src/spec/ngram-source.ts +0 -204
  530. package/src/spec/projected-draft-rows.ts +0 -73
  531. package/src/spec/qwen-mtp-module.ts +0 -171
  532. package/src/spec/qwen-mtp-rows.ts +0 -219
  533. package/src/spec/qwen-mtp-source.ts +0 -383
  534. package/src/spec/qwen-mtp-state.ts +0 -30
  535. package/src/spec/serve-loop.ts +0 -585
  536. package/src/spec/source.ts +0 -215
  537. package/src/spec/two-model-rows.ts +0 -61
  538. package/src/spec/two-model.ts +0 -113
  539. package/src/ssd-cache.ts +0 -372
  540. package/src/ssd-durability.ts +0 -211
  541. package/src/tokenizer.ts +0 -335
  542. package/src/tool-approvals.ts +0 -131
  543. package/src/tool-call.ts +0 -695
  544. package/src/train/dataset.ts +0 -484
  545. package/src/train/diffusion-lora.ts +0 -216
  546. package/src/train/export.ts +0 -32
  547. package/src/train/flash-cce.ts +0 -1234
  548. package/src/train/forward.ts +0 -266
  549. package/src/train/fuse.ts +0 -243
  550. package/src/train/index.ts +0 -46
  551. package/src/train/job.ts +0 -205
  552. package/src/train/lora-params.ts +0 -254
  553. package/src/train/loss.ts +0 -1591
  554. package/src/train/merge.ts +0 -195
  555. package/src/train/optimizer.ts +0 -183
  556. package/src/train/prefix-shared.ts +0 -490
  557. package/src/train/rank.ts +0 -137
  558. package/src/train/segmented.ts +0 -1828
  559. package/src/train/steel-qmm-header.ts +0 -1461
  560. package/src/train/trainer.ts +0 -1221
  561. package/src/train/watch.ts +0 -542
  562. package/src/vision/diffusion-vision.ts +0 -620
  563. package/src/vision/embedder.ts +0 -168
  564. package/src/vision/preprocess.ts +0 -247
  565. package/src/vision/prompt.ts +0 -373
  566. package/src/vision/qwen3vl-preprocess.ts +0 -409
  567. package/src/vision/qwen3vl-prompt.ts +0 -196
  568. package/src/vision/qwen3vl-tower.ts +0 -555
  569. package/src/vision/siglip.ts +0 -599
  570. package/src/vision/video-frames.ts +0 -148
  571. package/src/web/app.html +0 -2143
  572. package/src/web/app.js +0 -5703
  573. package/src/web/skills/memory/SKILL.md +0 -120
  574. package/src/web/skills/web-research/SKILL.md +0 -46
  575. package/src/web/skills.ts +0 -97
  576. package/src/web/src/adapters-panel.ts +0 -288
  577. package/src/web/src/assistant.ts +0 -451
  578. package/src/web/src/chat.ts +0 -1057
  579. package/src/web/src/composer.ts +0 -959
  580. package/src/web/src/dataset.ts +0 -132
  581. package/src/web/src/finetune.ts +0 -267
  582. package/src/web/src/hub.ts +0 -428
  583. package/src/web/src/main.ts +0 -98
  584. package/src/web/src/markdown.ts +0 -0
  585. package/src/web/src/memory-panel.ts +0 -572
  586. package/src/web/src/model-picker.ts +0 -159
  587. package/src/web/src/palette.ts +0 -346
  588. package/src/web/src/protocol.ts +0 -26
  589. package/src/web/src/quantize.ts +0 -167
  590. package/src/web/src/rag.ts +0 -274
  591. package/src/web/src/sessions.ts +0 -469
  592. package/src/web/src/shell.ts +0 -804
  593. package/src/web/src/ui-catalog.ts +0 -78
  594. package/src/web/vendor/README.md +0 -61
  595. package/src/web-assets.ts +0 -47
  596. package/src/web-tools.ts +0 -478
  597. package/src/weights.ts +0 -255
  598. /package/src/{serve → server}/completion-sink.ts +0 -0
  599. /package/src/web/{icon.svg → public/icon.svg} +0 -0
  600. /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
  601. /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
package/src/cli.ts DELETED
@@ -1,3168 +0,0 @@
1
- #!/usr/bin/env bun
2
- // mlx-bun CLI: the appliance interface.
3
- //
4
- // mlx-bun get <org/repo> [--revision main] resumable verified download
5
- // mlx-bun scan index the HF cache
6
- // mlx-bun ls [--vision] [--max-size 10GB] [query]
7
- // mlx-bun fit <query> [--ctx 8192] [--skus]
8
- // mlx-bun serve [query] [--port 8080] [--memory-budget GB]
9
- // mlx-bun evals recent benchmark runs
10
- // mlx-bun harness pi [--base-url <url>] [--remove] connect your own pi to the local server
11
-
12
- import { existsSync } from "node:fs";
13
- import { basename, join, resolve } from "node:path";
14
- import { Registry } from "./registry";
15
- import { loadModelConfig, parseTurboQuantScheme, type TurboQuantScheme } from "./config";
16
- import { fit, skuMatrix, thisMachine, type FitKvScheme } from "./fit";
17
- import { resolveKvScheme } from "./kv-scheme";
18
- import { EvalDB } from "./evaldb";
19
- import pkg from "../package.json" with { type: "json" };
20
- import { renderHelp } from "./tui";
21
- import { isGlm52Config, isSupportedModelRecord } from "./model/support";
22
- import {
23
- GLM52_G5_ASPIRATIONAL_DECODE_TPS,
24
- GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS,
25
- GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES,
26
- GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS,
27
- GLM52_G5_MEASURED_WARM_DECODE_TPS,
28
- planGlm52MemoryForArtifact,
29
- } from "./model/glm52-memory";
30
- import type { JobEvent } from "./jobs/types";
31
- import {
32
- configureRuntime,
33
- runtimeValue,
34
- type RuntimeKey,
35
- } from "./runtime-config";
36
-
37
- const argv = process.argv.slice(2);
38
- // Private self-exec protocol for managed jobs in the single-file binary.
39
- // Literal imports in job-entry keep the numerical runners inside the bundle.
40
- if (argv[0] === "__job") {
41
- const { runJobEntry } = await import("./jobs/job-entry");
42
- await runJobEntry(argv[1]);
43
- process.exit(0);
44
- }
45
-
46
- // The appliance path: naked `mlx-bun` (or only options, e.g.
47
- // `mlx-bun --port 9000`) runs `serve` — first run downloads a model and
48
- // opens the chat UI. `--help`/`--version` and explicit subcommands still win.
49
- const HELP_VERSION_FLAGS = ["--help", "-h", "--version", "-v"];
50
- const noSubcommand =
51
- !argv[0] || (argv[0].startsWith("-") && !HELP_VERSION_FLAGS.includes(argv[0]));
52
- const cmd = noSubcommand ? "serve" : argv[0];
53
-
54
- const OVERVIEW = `mlx-bun ${pkg.version} — local AI on Apple silicon. One binary, no Python.
55
-
56
- Usage: mlx-bun [command] [options] (no command = serve)
57
-
58
- Commands:
59
- pi Launch a pi coding-agent session on a local model (the appliance path)
60
- serve Start the OpenAI/Anthropic-compatible server + web app
61
- get Download a model from Hugging Face (resumable, verified)
62
- ls List downloaded models
63
- gc Reclaim superseded model snapshots + dead blobs from the HF cache
64
- fit Will a model fit this machine? Memory + speed assessment
65
- scan Re-index the Hugging Face cache
66
- harness Connect your own pi install to the local mlx-bun server
67
- train Fine-tune a LoRA adapter (SFT / DPO / ORPO) on your data
68
- train-watch Live dashboard for a training run (tails metrics.jsonl)
69
- fuse Merge a LoRA adapter into the base weights (standalone model)
70
- convert Quantize an HF model into a local MLX snapshot
71
- upload Push a local model directory to the Hugging Face Hub
72
- perplexity Perplexity of a model over a local text/JSONL dataset
73
- memory Set up + manage your local AI's personal wiki memory
74
- generate One-shot raw text generation from a local model (no server)
75
- benchmark Measure decode/prefill speed of OUR stack on this machine
76
- embed Text embeddings from a local embedding model (Qwen3-Embedding)
77
- evals Show recent benchmark runs (all stacks)
78
- help Show help for a command (also: mlx-bun <command> --help)
79
-
80
- Options:
81
- -h, --help Show help
82
- -v, --version Show version
83
-
84
- Examples:
85
- mlx-bun # download (first run) + serve + open the chat UI
86
- mlx-bun pi # start (download if needed) and chat with an agent
87
- mlx-bun serve 12B # serve the 12B; web app at http://localhost:8080/
88
- mlx-bun train e4b --data ./prefs # ORPO LoRA fine-tune on {prompt,chosen,rejected} data
89
- mlx-bun get mlx-community/gemma-4-12B-it-OptiQ-4bit`;
90
-
91
- const SERVER_FLAGS = `Server options:
92
- --model <path|query> Explicit model override (mlx_lm.server spelling):
93
- a directory containing config.json loads from that
94
- path; anything else resolves like the positional
95
- query. Wins over the positional and auto-pick.
96
- --host <addr> Interface to bind [default: 127.0.0.1, loopback
97
- only; use 0.0.0.0 to expose on your network]
98
- --port <n> Listen port [default: 8080]
99
- --memory-budget <GB> Admission-control memory budget; requests that
100
- cannot fit are rejected instead of crashing the
101
- GPU [default: machine RAM × 0.75, check-only]
102
- --kv-budget <GB> Aggregate KV budget across concurrent batch rows
103
- (decimal GB): joiners whose projected KV
104
- (prompt + max_tokens) would exceed it queue until
105
- rows finish; a request over the budget alone is
106
- rejected. Off unless set.
107
- --prompt-cache <GB> Prompt (KV) cache byte cap [default: 8 GB;
108
- --prompt-cache 0 disables the cache]
109
- --ssd-cache <dir> SSD cold tier for the prompt cache: prefix KV
110
- spills to disk on eviction and persists across
111
- restarts (long-context agent TTFT drops from a
112
- full re-prefill to a zero-copy mmap restore).
113
- Off unless set.
114
- --ssd-cache-max <GB> Optional SSD tier byte cap; 0 = unlimited
115
- [default: unlimited]
116
- --ssd-demote-idle <sec> Demote prompt-cache entries idle this long to the
117
- SSD tier, freeing their GPU memory (prefixes stay
118
- reachable via zero-copy restore). 0 disables.
119
- [default: 300 when --ssd-cache is on]
120
- --ssd-cache-verify Verify tensor hashes on every restore (reads all
121
- bytes eagerly — integrity paranoia only)
122
- --generation-checkpoint <tokens>
123
- Queue an in-flight generation snapshot to the SSD
124
- cache every N emitted tokens. Repeating the same
125
- request after restart replays and resumes it.
126
- Requires --ssd-cache; supports ordinary requests
127
- with shared cache layouts.
128
- --isolate Run the inference engine as a CHILD process and
129
- keep this process a pure UI/API proxy — the UI
130
- stays instant under GPU load and survives engine
131
- crashes (respawned). /ws/chat not proxied yet.
132
- Requests naming another model (exact /v1/models
133
- id) spawn/route to THAT model's engine child.
134
- --model-pool <n> With --isolate: max RESIDENT model engines
135
- [default: 1]. Over the cap, the least-recently-
136
- used engine drains, demotes its caches to the
137
- SSD tier, and exits — switching back respawns
138
- it with all state restored from disk.
139
- --unix <path> (internal) listen on a unix socket instead of a
140
- TCP port — the engine half of --isolate.
141
- --batch <n> Max concurrent requests decoded together
142
- [default: 8]. A lone scheduler request uses the
143
- parity- and latency-gated B=1 fast path; concurrent
144
- requests can form B=N (agent fan-out). 1 pins the
145
- preserved strict serial executor.
146
- --decode-concurrency is accepted as the
147
- mlx_lm.server alias (semantics differ; see
148
- docs/reference/server-config.md)
149
- --no-open Don't open the chat UI in your browser on start.
150
- By default an interactive terminal opens
151
- http://<host>:<port>/#/chat once the server is up.
152
- --allow-private-media Let image_url/audio_url content parts fetch from
153
- private/loopback/link-local hosts (NAS, another
154
- LAN box). Blocked by default: a request's URL is
155
- attacker-controlled input, and the default keeps
156
- the server from being steered at cloud metadata
157
- or LAN-internal services (SSRF). data: URLs are
158
- always fine; the 10 s timeout and 64 MB response
159
- cap apply either way.
160
-
161
- Model & quality:
162
- --adapter <dir> Mount a LoRA adapter at startup and use it as the
163
- default for requests that don't select one (a
164
- request's \`adapter\` field, incl. "none", wins).
165
- --adapter-path is accepted as the mlx_lm.server
166
- alias. Hot-swap via POST /v1/adapters unchanged.
167
- --draft-model <query> Speculative decoding: a drafter proposes tokens
168
- the main model verifies (exact results, faster
169
- decode when drafts land). The artifact's kind is
170
- auto-detected — a full same-tokenizer model
171
- (mlx_lm.server parity), a Gemma "-assistant"
172
- KV-borrowing drafter, or a DSpark checkpoint.
173
- Supported draft providers use shared batched
174
- execution; ordinary requests can batch too.
175
- --draft-kind <kind> Override draft-artifact detection:
176
- two-model | assistant | dspark | deepspec | mtp | ngram
177
- (deepspec = DeepSeek's released Gemma4DSparkModel
178
- drafters; mtp = a NATIVE multi-token-prediction
179
- head split from the target's own release, e.g.
180
- mlx-community/Qwen3.8-27B-MTP-bf16 — auto-detected
181
- from model_type *_mtp;
182
- ngram = MODEL-FREE prompt lookup — no
183
- --draft-model, drafts copied from the request's
184
- own context, lossless) [default: auto]
185
- --num-draft-tokens <n> Drafts per verify round [default: 3; ngram: 10;
186
- DSpark pins to its trained block width]
187
- --ngram-max <k> Prompt-lookup longest match window [default: 3]
188
- --ngram-min <k> Prompt-lookup shortest match window [default: 1]
189
- --mtp <on|off> GLM-5.2 checkpoint-native MTP. On by default;
190
- routes requests through the exact serial verify
191
- lane. off enables ordinary continuous batching.
192
- --context-length <n> GLM-5.2 resource-plan context reservation
193
- [default: 4096]
194
- --kv-quant <mode> KV cache quantization: config (per-layer
195
- kv_config.json when the model ships one), off
196
- (bf16), 4 / 8 (uniform bits), or turbo[:k<bits>v
197
- <bits>] (rotation-based TurboQuant — default
198
- k8v3; kBits in {2,4,5,8}, vBits in {2,3,4,5,8};
199
- docs/design/turboquant.md)
200
- [default: off — quantized KV measured 5-20% slower
201
- decode than bf16 at ≤16k (on mlx-lm too); it buys
202
- memory headroom, so opt in when context is tight.
203
- The batched engine (--batch N) is bf16-only: an
204
- explicit --kv-quant routes those requests serial.
205
- turbo is solo-only in v1 (full-attention layers
206
- only; sliding-window layers stay bf16)]
207
- --paged-kv OPTIONAL vLLM-style paged KV storage (block pool +
208
- gather before the stock SDPA; env MLX_BUN_PAGED_KV=1).
209
- Gemma4-family, bf16; supports --batch N.
210
- Refuses --kv-quant and --draft-model; bypasses the prompt
211
- cache; runs uncompiled decode. Bit-exact with the
212
- plain path; expect a small decode cost at batch=1
213
- (the gather copy). docs/design/kv-cache.md
214
- [default: off]
215
- --paged-kv-block-size <n> Tokens per KV block [default: 256 = the plain
216
- cache's growth step]
217
- --thinking <true|false> Default for the chat template's enable_thinking
218
- variable (CPM and other hybrid-reasoning models);
219
- a request's chat_template_kwargs overrides it
220
- [default: the model's own, false for CPM]
221
- --temperature <n> Server-wide sampling defaults; a per-request field
222
- --top-p <n> still overrides them, and the browser chat (which
223
- --top-k <n> sends none) inherits them. --temp is accepted as
224
- an alias for --temperature (mlx_lm.server compat;
225
- NOTE mlx_lm.server defaults temperature to 0.0 —
226
- pass --temp 0 for its unset-request behavior)
227
- [default: the model's generation_config.json]
228
- --max-tokens <n> Completion cap when a request omits max_tokens
229
- (mlx_lm.server flag; --max-tokens 512 reproduces
230
- its default) [default: 128 GLM-5.2 (memory-plan
231
- reservation); otherwise none — a defaulted
232
- request runs to EOS or the admitted context]
233
- --hlg-sampling on|off Piecewise tone-curve sampling (HLG): rolls off the
234
- top, boosts the mids, gentles the tail. Gain folds
235
- from --temperature. [default: off]
236
- See docs/archive/hlg-sampling.md
237
- --hlg-width <nats> HLG mid-region half-width [default: 4]
238
- --hlg-shoulder <nats> HLG highlight rolloff scale [default: 4]
239
- --hlg-toe <nats> HLG shadow rolloff scale [default: 6]
240
- --hlg-pivot-offset <nats> HLG pivot: nats below the top token [default: 6]
241
-
242
- Parity tier (an ALIAS for the per-fork flags below; each expands to the fastest
243
- set of forks that still holds the guarantee — a per-fork flag overrides one):
244
- --l1 Bit-for-bit IDENTICAL to mlx-lm (bf16 KV, compiled
245
- activations — the faithful @mx.compile kernels).
246
- No tier given = --l1 (decided 2026-07-05: the L1
247
- faithful kernels match mlx-lm 1.00x on every model;
248
- anything output-changing must beat that baseline in
249
- a paired A/B to earn a default).
250
- --l2 Bit-for-bit IDENTICAL to mlx-optiq (quantized KV +
251
- fused prefill SDPA, stock unfused decode — the
252
- optiq-golden composition)
253
-
254
- Kill switches (bit-exact A/B levers — each selects a slower same-parity path):
255
- --compiled-decode on|off Compiled decode graphs [default: on]
256
- --compiled-activations on|off
257
- Compiled geglu/swiglu (mlx-lm's @mx.compile kernel —
258
- bit-exact AND fewer dispatches). off = the uncompiled
259
- composition (same parity, slower) [default: on]
260
- --fused-sdpa on|off Fused SDPA path (quantized KV only) [default:
261
- follows --kv-quant — on for config (the optiq
262
- composition), off for uniform 4/8 (the mlx-lm
263
- composition) and bf16]
264
- --force-wire Wire weights into memory at load
265
- --expert-offload MoE only: serve experts from a page-aligned file
266
- mmap (built on first use) — keeps the model out of
267
- memory pressure (phys_footprint ≈ active params),
268
- bit-exact with the resident path`;
269
-
270
- const HELP: Record<string, string> = {
271
- pi: `mlx-bun pi — drop into a coding-agent session on a local model
272
-
273
- Usage: mlx-bun pi [options] [message...]
274
-
275
- Runs the pi coding agent's own terminal UI in-process against the local
276
- model — nothing to install, pi is built in. Reuses a healthy local
277
- server when one is running; otherwise serves e4b by default (override
278
- with --query) and starts a server for the session (the server ends with
279
- the session). Fresh install: downloads a small starter model first so
280
- you're chatting in minutes, then streams e4b in the background — it
281
- becomes the default on the next run.
282
-
283
- Model selection:
284
- -q, --query <q> Model to serve when starting a server (registry query)
285
-
286
- ${SERVER_FLAGS}
287
-
288
- Run modes:
289
- message... Pre-fill the first turn (interactive)
290
- -p, --print [msg] One-shot: print the reply and exit (also reads piped
291
- stdin); add --json for the full event stream
292
- --mode json|rpc Structured print (json) or JSONL RPC over stdio
293
-
294
- Already use pi? Connect your own pi to this local model instead:
295
- mlx-bun harness pi`,
296
-
297
- serve: `mlx-bun serve — OpenAI/Anthropic-compatible server for a local model
298
-
299
- Usage: mlx-bun serve [query] [options]
300
-
301
- [query] Registry query (e.g. "12B", "e4b", a repo substring).
302
- --query <q> is accepted for the positional. Omitted:
303
- serves e4b (the default), else the largest downloaded
304
- model that fits. Fresh install: downloads a small
305
- starter model first, then e4b in the background
306
- (default on next run).
307
-
308
- ${SERVER_FLAGS}
309
-
310
- Endpoints: /v1/chat/completions, /v1/completions, /v1/messages, /v1/responses,
311
- /v1/models, /v1/adapters, /health, /stats, /fit, /library, /downloads —
312
- web app at / (lands on chat) — /chat, /status, /quantize, /finetune, /dataset deep-link its views`,
313
-
314
- get: `mlx-bun get — download a model from Hugging Face
315
-
316
- Usage: mlx-bun get <org/repo | substring> [options]
317
-
318
- <org/repo> Full repo id (e.g. mlx-community/gemma-4-12B-it-OptiQ-4bit)
319
- <substring> No "/" = a registry query over already-downloaded repos
320
- (e.g. \`mlx-bun get 12B\` re-gets/refreshes the match)
321
-
322
- Options:
323
- --revision <rev> Git revision [default: main]
324
-
325
- Resumable (Range requests against partial blobs) and verified (sha256
326
- for LFS blobs). Re-running after an interruption continues where it
327
- stopped. Before payload transfer, complete blobs and partial prefixes are
328
- credited and the target volume must fit the remainder plus a 1 GiB reserve.
329
- Uses HF_TOKEN / hf auth login credentials when present.
330
- When upstream pushed a new revision, the previous snapshot stays on
331
- disk — \`mlx-bun gc\` reclaims it.`,
332
-
333
- ls: `mlx-bun ls — list downloaded models
334
-
335
- Usage: mlx-bun ls [query] [options]
336
-
337
- One row per repo (the canonical revision: refs/main). The HF cache keeps
338
- a snapshot dir per downloaded revision — superseded ones are hidden here
339
- and reclaimed by \`mlx-bun gc\`.
340
-
341
- Options:
342
- --vision Only vision-capable models (SigLIP sidecar or the
343
- unified encoder-free tower)
344
- --max-size <size> Filter by weight size (e.g. 10GB, 800MB)
345
- --all-revisions One row per cached snapshot (canonical marked *)`,
346
-
347
- gc: `mlx-bun gc — reclaim superseded snapshots + dead blobs
348
-
349
- Usage: mlx-bun gc [options]
350
-
351
- Every \`get\` that follows an upstream push creates a NEW
352
- snapshots/<commit> dir; the old one (and any blobs only it references)
353
- stays on disk forever. gc keeps the snapshots refs/* point at, deletes
354
- the rest, then deletes blobs no surviving snapshot links to.
355
-
356
- Without --yes it only prints what would be deleted (per-repo reclaim).
357
-
358
- Options:
359
- --yes Actually delete (destructive; default prints only)
360
- --dry-run Print the plan and never delete, even with --yes
361
- --force Also prune superseded snapshots that contain files
362
- the canonical revision lacks (normally skipped with
363
- a warning — deleting them loses the only copy)`,
364
-
365
- fit: `mlx-bun fit — will this model fit on this machine?
366
-
367
- Usage: mlx-bun fit <query> [options]
368
-
369
- Options:
370
- --ctx <tokens> Context size to assess [default: 8192]
371
- --kv-quant <mode> Bill the KV cache quantized: 4 | 8 | config | off
372
- (mirrors serve --kv-quant; default bf16)
373
- --skus Also print the Apple silicon SKU matrix
374
-
375
- Solves weights + KV growth + prefill transient against wired memory and
376
- predicts decode speed from memory bandwidth.`,
377
-
378
- scan: `mlx-bun scan — re-index the Hugging Face cache
379
-
380
- Usage: mlx-bun scan
381
-
382
- Reads config.json + safetensors headers (never tensor bytes) for every
383
- snapshot in ~/.cache/huggingface/hub and refreshes the model registry.`,
384
-
385
- harness: `mlx-bun harness pi — connect your own pi to the local model
386
-
387
- Usage: mlx-bun harness pi [options]
388
-
389
- For people who already use pi: this points your existing pi install at
390
- the local mlx-bun server so you can run pi the way you already do, on a
391
- local model. (To just chat now with no setup, use \`mlx-bun pi\` — the
392
- built-in agent.)
393
-
394
- Options:
395
- --base-url <url> Server base URL [default: http://127.0.0.1:8080/v1]
396
- --remove Disconnect (delete the extension)
397
-
398
- Installs a small discovery extension into ~/.pi/agent/extensions that
399
- registers mlx-bun as a pi provider (models discovered live from
400
- /v1/models at pi startup). One file, reversible with --remove; never
401
- touches your existing pi config. Then: pi --provider mlx-bun`,
402
-
403
- benchmark: `mlx-bun benchmark — measure OUR stack on this machine
404
-
405
- Usage: mlx-bun benchmark [query] [options]
406
-
407
- Runs mlx-bun by itself (no other systems required) and records the
408
- result to the eval DB. For quotable, cross-stack numbers use
409
- bun scripts/bench-serve.ts all from the repo — it preflight-gates on an idle machine
410
- and runs the mlx-lm/optiq comparison legs.
411
-
412
- [query] Model to benchmark (--query <q> is accepted for the
413
- positional) [default: auto-pick]
414
-
415
- Options:
416
- --model <path|query> Explicit model override: a directory containing
417
- config.json loads from that path; anything else
418
- resolves like the positional query. Wins over both.
419
- --tokens <n> Tokens to decode per run [default: 256]
420
- --runs <n> Runs (median reported) [default: 3]
421
- --prompt-tokens <n> Pad the prompt to ~n tokens (long-context decode)
422
- --kv-quant <mode> config | off | 4 | 8 | turbo[:k<bits>v<bits>]
423
- [default: off, the historical baseline]
424
-
425
- Kill switches (--compiled-decode, --compiled-activations, --fused-sdpa)
426
- apply to the run — A/B by running twice.`,
427
-
428
- embed: `mlx-bun embed — text embeddings from a local embedding model
429
-
430
- Usage: mlx-bun embed [query] --text "…" [options]
431
- mlx-bun embed [query] "…" (text as a positional)
432
- echo -e "line one\\nline two" | mlx-bun embed [query]
433
-
434
- Embeds text with a local Qwen3-Embedding model (last-token pooled,
435
- L2-normalized — bit-exact vs mlx-lm). One text per --text, or one per
436
- line on stdin. Prints one JSON array (the vector) per input line.
437
-
438
- [query] Embedding model (--query <q> is accepted for the
439
- positional) [default: the first downloaded
440
- embedding model; errors with an mlx-bun get
441
- suggestion when none is downloaded]
442
-
443
- Options:
444
- --text "…" Text to embed (omit to read lines from stdin)
445
- --instruct "…" Query instruction (Qwen3-Embedding query format);
446
- steers WHICH similarity axis the geometry reflects.
447
- Omit for plain document embeddings.
448
- --json Emit one OpenAI-style {object:"list",data:[…]} object
449
- instead of bare vector arrays
450
-
451
- For a server endpoint use \`mlx-bun serve <embedding-model>\` then
452
- POST /v1/embeddings.`,
453
-
454
- generate: `mlx-bun generate — one-shot raw text generation (no server)
455
-
456
- Usage: mlx-bun generate [query] --prompt "…" [options]
457
- mlx-bun generate [query] "…" (prompt as a positional)
458
-
459
- A direct one-shot entry point: load the model, generate, print, exit —
460
- no server, no chat UI. By default the prompt is rendered through the
461
- model's chat template; --raw feeds it verbatim.
462
-
463
- [query] Model (registry query; --query is accepted for the
464
- positional) [default: auto-pick]
465
-
466
- Options:
467
- --prompt "…" The prompt (or pass it as the positional)
468
- --raw Skip the chat template (verbatim prompt)
469
- --max-tokens <n> Completion cap [default: 256]
470
- --temperature <n> Sampling temperature (--temp is accepted as an alias)
471
- --top-p <n> Nucleus sampling
472
- --top-k <n> Top-k sampling
473
- --seed <n> Sampler seed (reproducible runs)
474
-
475
- Decode-path levers mirror serve: --l1/--l2 and the per-fork flags
476
- (--fused-sdpa/--compiled-decode/--compiled-activations/--kv-quant).
477
- The tier's KV scheme applies here too (--l2 = the model's kv_config,
478
- same tokens as serve); with no tier/--kv-quant the KV cache stays bf16
479
- (the bit-exact mlx-lm greedy path). mlx-lm compat = --l1.
480
-
481
- Aliases: mlx-bun gen`,
482
-
483
- evals: `mlx-bun evals — recent benchmark runs
484
-
485
- Usage: mlx-bun evals [options]
486
-
487
- Options:
488
- --limit <n> Rows to show [default: 20]
489
- --raw Full records as JSON lines (for scripts/jq)
490
- --clear Archive the eval DB (timestamped backup) and start fresh
491
-
492
- Table view: when, model, bench kind, KV mode, decode tok/s, TTFT,
493
- peak memory, commit. Runs are written by bun scripts/bench-serve.ts all.`,
494
-
495
- train: `mlx-bun train — fine-tune a LoRA adapter on your data
496
-
497
- Usage: mlx-bun train <model> --data <dir> [options]
498
-
499
- <model> Model to fine-tune (query or snapshot path; auto-picks
500
- the default model if omitted)
501
-
502
- Data: <dir> must contain train.jsonl (+ optional valid.jsonl). Rows are
503
- {prompt, chosen, rejected} for dpo/orpo, or {messages|text} for sft.
504
-
505
- Options:
506
- --data <dir> Dataset dir with train.jsonl (required)
507
- --method <m> sft | dpo | orpo [default: orpo]
508
- --adapter <dir> Output adapter dir
509
- [default: ~/.cache/mlx-bun/mlx-bun-finetunes/<method>-<model>]
510
- --iters <n> Training iterations [default: 100]
511
- --lr <f> Learning rate [default: orpo 1e-5 · dpo 5e-5 · sft 2e-4]
512
- --rank <n> LoRA rank [default: orpo 16 · else 8]
513
- --scale <f> LoRA scale [default: orpo 2.0 · else 1.0]
514
- --seq <n> Max sequence length [default: gemma 8192 · else 4096]
515
- --batch <n> Batch size [default: 1]
516
- --grad-accum <n> Gradient accumulation steps (effective batch =
517
- batch × grad-accum at batch-size-1 memory) [default: 1]
518
- --grad-clip <f> Gradient-norm clip (0 = off) [default: 1.0]
519
- --seed <n> Data-shuffle / init seed [default: 0]
520
- --val-size <n> Max validation examples per eval [default: 256]
521
- --lambda <f> ORPO odds-ratio weight [default: 0.1]
522
- --sft-scope <s> ORPO chosen-NLL scope: full (paper/TRL-faithful,
523
- prompt+response) | response (pre-2026-07 runs,
524
- bit-exact) [default: full]
525
- --seg <n> Layers per segment (segmented backward; orpo default 2)
526
- --save-every <n> Crash-safe mountable checkpoint every n steps
527
- --resume <dir> Warm-start LoRA weights from a checkpoint/adapter dir
528
- --no-flash Disable the flash-CCE Metal head (use the MLX fused head)
529
- --no-prefix Disable prefix-sharing (two-forward branches)
530
- --no-segment Disable the segmented backward (hold all activations)
531
- --dry-run Inspect the dataset + print the resolved plan, don't train
532
-
533
- ORPO defaults run the full stack: the [M,vocab]-free flash-CCE head +
534
- prefix-sharing + segmented backward (each falls back + logs if a row's
535
- preconditions aren't met). Gemma/e4b sets its required training env flags
536
- automatically. Adapters are mountable directly (mlx-bun serve --adapter).
537
- Run long jobs detached from your own shell: nohup mlx-bun train … &
538
- Watch a run live from another tab: mlx-bun train-watch <adapter-dir>`,
539
-
540
- "train-watch": `mlx-bun train-watch — live dashboard for a training run
541
-
542
- Usage: mlx-bun train-watch [adapter-dir]
543
-
544
- Tails <adapter-dir>/metrics.jsonl (written by mlx-bun train and the web
545
- fine-tune tile) and renders a live loss/speed/memory dashboard. Point it
546
- at the same --adapter directory the training run uses.
547
-
548
- [adapter-dir] Adapter directory to watch; --adapter <dir> is
549
- accepted for the positional
550
- [default: ~/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5]`,
551
- fuse: `mlx-bun fuse — merge a LoRA adapter into the base weights
552
-
553
- Usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [options]
554
-
555
- Folds the adapter's low-rank deltas into the base weights and writes a
556
- standalone, loadable model snapshot (weights + config + tokenizer). A
557
- quantized base keeps its exact quantization layout: fused modules are
558
- dequantized, updated, and re-quantized with their own per-module spec.
559
-
560
- <model> Base model (registry query or a snapshot path);
561
- --model is accepted as the mlx_lm.fuse spelling
562
-
563
- Options:
564
- --adapter <dir> Adapter directory (adapters.safetensors +
565
- adapter_config.json). --adapter-path is accepted
566
- as the mlx_lm.fuse alias [default: adapters]
567
- --save-path <dir> Output model directory [default: fused_model]
568
-
569
- Not supported (mlx_lm.fuse flags we don't implement — the command exits
570
- with an error rather than guessing): --de-quantize, --export-gguf,
571
- --gguf-path, --upload-repo.
572
-
573
- Serve the result: mlx-bun serve <save-path>`,
574
-
575
- convert: `mlx-bun convert — quantize an HF model into a local MLX snapshot
576
-
577
- Usage: mlx-bun convert --hf-path <repo-or-path> -q [options]
578
- mlx-bun convert <repo-or-path> -q [options] (positional source)
579
-
580
- Wraps mlx-bun's native quantize pipeline (the same engine as the web
581
- /api/quantize). The source may be a local model directory, a downloaded
582
- model (registry query), or a Hugging Face repo id — an un-downloaded
583
- repo is fetched first (resumable, verified; can be many GB).
584
-
585
- Options:
586
- --hf-path <src> Source model: local path, downloaded model, or HF
587
- repo id (--model and the bare positional are
588
- accepted as aliases)
589
- --mlx-path <dir> Output directory [default: mlx_model; must not
590
- already exist, matching mlx_lm.convert]
591
- -q, --quantize Quantize the model (uniform affine)
592
- --q-bits <n> Bits per weight: 4 or 8 [default: 4]
593
- --q-group-size <n> Quantization group size: 32 or 64 [default: 64]
594
- --upload-repo <id> Push the converted model to this Hugging Face repo
595
- after converting (mlx_lm.convert parity; same engine
596
- as \`mlx-bun upload\`, needs a write token)
597
-
598
- Mixed precision (the mlx-bun differentiator — OptiQ sensitivity sweep +
599
- knapsack per-layer bit allocation; implies quantization, no -q needed):
600
- --target-bpw <f> Target bits-per-weight (e.g. 4.5). Runs
601
- calibration → per-layer KL sensitivity → greedy
602
- knapsack; writes per-module bits into config.json
603
- --candidate-bits <l> Comma list the knapsack may pick from [default: 4,8]
604
- --calibration-mix <m> "optiq" or a JSONL path [default: optiq]
605
- --n-calibration <n> Calibration samples [default: 2]
606
- --rotate-weights Fold the model's offline TurboQuant rotation before
607
- quantization (auto-detects Llama/Qwen3.5/Qwen MTP)
608
- --rotation-seed <n> Deterministic rotation seed [default: 42]
609
-
610
- Not supported (mlx_lm.convert flags we don't implement — the command
611
- exits with an error rather than guessing): --dtype, -d/--dequantize,
612
- --q-mode other than affine, and the mlx-lm --quant-predicate recipes
613
- (use --target-bpw instead). Plain non-quantizing conversion is also not
614
- supported: pass -q or --target-bpw.`,
615
-
616
- upload: `mlx-bun upload — push a local model directory to the Hugging Face Hub
617
-
618
- Usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [options]
619
-
620
- Native push-to-hub, no Python (src/hf-push.ts — the same engine as the
621
- web UI's push buttons): creates the repo if needed (idempotent), uploads
622
- weights via the git-LFS batch protocol, and commits. Flag names match
623
- mlx_lm.upload.
624
-
625
- Options:
626
- --path <dir> Local model directory to upload [default: mlx_model]
627
- --upload-repo <id> Hub repo id, org/name or bare name (required)
628
- --private Create the repo as private (mlx-bun extension)
629
-
630
- Auth: needs a Hugging Face WRITE token, resolved from ~/.mlx-bun/hf.json
631
- (web UI Settings → Hugging Face), $HF_TOKEN, or ~/.cache/huggingface/token
632
- (hf auth login). Errors before uploading anything if none is found.
633
-
634
- Convert + push in one step: mlx-bun convert … --upload-repo <org/repo>`,
635
-
636
- perplexity: `mlx-bun perplexity — perplexity over a local text/JSONL dataset
637
-
638
- Usage: mlx-bun perplexity <model-query-or-path> --data-path <file> [options]
639
-
640
- mlx_lm.perplexity methodology, exactly: samples are visited in a seeded
641
- random order, tokenized, concatenated, and cut into NON-OVERLAPPING rows
642
- of --sequence-length tokens; per batch the model scores rows[:, :-1]
643
- against rows[:, 1:] in f32 (every position counts); reported as
644
- ppl = exp(mean CE) ± the delta-method standard error. The one deliberate
645
- difference: the data source is a LOCAL file (never a Hugging Face
646
- dataset download).
647
-
648
- <model> Model (registry query or snapshot path);
649
- --model (mlx_lm spelling) and --query are accepted
650
- for the positional
651
-
652
- Options:
653
- --data-path <file> .jsonl ({"text": …} rows) or plain .txt (required)
654
- --sequence-length <n> Tokens per row [default: 512]
655
- --num-samples <n> Rows to score (-1 = all available) [default: 256]
656
- --batch-size <n> Rows per forward [default: 8]
657
- --seed <n> Sample-shuffle seed [default: 123]`,
658
-
659
- memory: `mlx-bun memory — your local AI's personal wiki
660
-
661
- Usage: mlx-bun memory <subcommand>
662
-
663
- A local, durable memory for the assistant: a wiki of Markdown articles
664
- (~/.mlx-bun/wiki) it reads to remember your projects, people, and history
665
- across sessions. It's yours — git-tracked, editable in any tool (Obsidian
666
- opens it as a vault), and it never leaves the machine. A read-only Reference/
667
- folder symlinks mlx-bun's own docs into memory on day one. Once set up, it
668
- loads automatically into every \`mlx-bun pi\` and \`mlx-bun serve\` session.
669
-
670
- Subcommands:
671
- init, setup Create the wiki + walk through setup (idempotent);
672
- offers to import an existing vault and install the
673
- nightly synthesis job
674
- status Path, article count, git + schedule state
675
- open, browse [article]
676
- Open the wiki, or a specific article, in Obsidian
677
- (falls back to Finder / the default Markdown app)
678
- list List article titles + read-only Reference docs
679
- search <query> Search articles from the terminal
680
- toc <article> Print an article's headings + anchors
681
- section <article> <anchor>
682
- Print one article section
683
- links <article> Show resolved outbound + inbound wikilinks
684
- read <article> Print an article (stem, e.g. Archie_Project)
685
- synthesize Run the FULL synthesis DAG now (--since, --model,
686
- --dry-run); also: pipeline, all
687
- segment | extract | route | synthesize-stage
688
- Run ONE decomposed stage worker (--limit N —
689
- segment/extract/synthesize-stage only; --convs a,b).
690
- Each pulls its eligible work from the DB by state, walks
691
- oldest-conversation-first, persists, and exits — resumable,
692
- and runnable as separate concurrent processes on slices.
693
- link Deterministic cross-linking stage: inline-link first
694
- mentions + rebuild ## See also (--limit N; no model)
695
- schedule Install the nightly launchd job (--at HH:MM [03:00])
696
- unschedule Remove the nightly launchd job
697
-
698
- The read path is live: the assistant reads a wiki you set up by hand or
699
- import during \`memory init\`. Ask the assistant to "open my memory" or run
700
- \`mlx-bun memory open\` to browse it in Obsidian/Finder, or
701
- \`mlx-bun memory open <article>\` to jump to a specific page. Synthesis
702
- (conversations → articles) runs the full local pipeline via
703
- \`mlx-bun memory synthesize\` (or per-stage: segment/extract/route/
704
- synthesize-stage/link); the nightly job runs it on a schedule.`,
705
- };
706
-
707
- HELP.bench = HELP.benchmark!;
708
- HELP.gen = HELP.generate!;
709
- HELP.setup = HELP.memory!;
710
-
711
- function printHelp(topic?: string): never {
712
- if (topic && HELP[topic]) console.log(renderHelp(HELP[topic]));
713
- else if (topic) {
714
- console.error(`unknown command: ${topic}\n`);
715
- console.log(renderHelp(OVERVIEW));
716
- process.exit(1);
717
- } else console.log(renderHelp(OVERVIEW));
718
- process.exit(0);
719
- }
720
-
721
- if (cmd === "--help" || cmd === "-h") printHelp();
722
- if (cmd === "--version" || cmd === "-v" || cmd === "version") {
723
- console.log(`mlx-bun ${pkg.version}`);
724
- process.exit(0);
725
- }
726
- if (cmd === "help") printHelp(argv[1]);
727
- // Per-command --help/-h — except `pi`, whose extra args pass through
728
- // (use `mlx-bun help pi` for our help, `pi --help` for pi's).
729
- if (cmd !== "pi" && (argv.includes("--help") || argv.includes("-h"))) printHelp(cmd);
730
- if (cmd === "pi" && (argv[1] === "--help" || argv[1] === "-h") && argv.length === 2) printHelp("pi");
731
-
732
- const flag = (name: string): boolean => argv.includes(`--${name}`);
733
- const opt = (name: string, dflt: string | null = null): string | null => {
734
- const i = argv.indexOf(`--${name}`);
735
- return i > -1 && argv[i + 1] ? argv[i + 1]! : dflt;
736
- };
737
- /** Single-dash value flag (e.g. -q), the short-alias counterpart to opt(). */
738
- const optShort = (short: string, dflt: string | null = null): string | null => {
739
- const i = argv.indexOf(`-${short}`);
740
- return i > -1 && argv[i + 1] ? argv[i + 1]! : dflt;
741
- };
742
- const positional = (n: number): string | undefined =>
743
- argv.filter((a, i) => !a.startsWith("--") && (i === 0 || !argv[i - 1]!.startsWith("--"))).slice(1)[n];
744
-
745
- const gb = (bytes: number) => `${(bytes / 2 ** 30).toFixed(2)} GB`;
746
-
747
- /** First-run: fetch the MLX native runtime pack if no copy is resolvable
748
- * (env / beside the binary / cache / homebrew). ~52 MB, resumable,
749
- * sha256-verified. Must run BEFORE importing anything that dlopens
750
- * (./server, ./generate — they load libmlxc at module scope). */
751
- async function ensureNative(s?: import("./tui").Step): Promise<void> {
752
- const { nativeRuntimeDir, ensureNativeRuntime, NATIVE_PACK_VERSION } = await import("./native-pack");
753
- if (nativeRuntimeDir()) return;
754
- s?.update(`downloading the MLX native runtime (v${NATIVE_PACK_VERSION}, ~52 MB)`);
755
- await ensureNativeRuntime({
756
- onProgress: (received, total) => {
757
- const pct = total ? Math.floor((received / total) * 100) : 0;
758
- const line = `MLX native runtime ${gb(received)} / ${gb(total)} (${pct}%)`;
759
- if (s) s.update(line); else process.stdout.write(`\r ${line} `);
760
- },
761
- });
762
- if (!s) process.stdout.write("\n");
763
- }
764
-
765
- /** AppleScript that focuses the first tab whose URL contains `match` in a
766
- * literal-named browser, returning "ok"/"miss". The app name MUST be a literal
767
- * (not a variable): AppleScript loads an app's scripting dictionary by literal
768
- * name at COMPILE time, so `tell application someVar` can't resolve app-specific
769
- * terms like `active tab index` (fails with -2740). Referencing an uninstalled
770
- * app by literal name is also a compile error a `try` can't catch — hence we
771
- * only ever run a browser's script when pgrep says it's running (= installed). */
772
- function focusTabScript(app: string, kind: "chromium" | "safari", match: string): string {
773
- const m = JSON.stringify(match); // safe AppleScript string literal
774
- if (kind === "safari") {
775
- return `tell application "${app}"
776
- repeat with w in windows
777
- repeat with t in tabs of w
778
- if URL of t contains ${m} then
779
- set current tab of w to t
780
- activate
781
- return "ok"
782
- end if
783
- end repeat
784
- end repeat
785
- end tell
786
- return "miss"`;
787
- }
788
- return `tell application "${app}"
789
- repeat with w in windows
790
- set k to 0
791
- repeat with t in tabs of w
792
- set k to k + 1
793
- if URL of t contains ${m} then
794
- set active tab index of w to k
795
- set index of w to 1
796
- activate
797
- return "ok"
798
- end if
799
- end repeat
800
- end repeat
801
- end tell
802
- return "miss"`;
803
- }
804
-
805
- /** Open the chat UI, reusing an already-open tab on this host:port instead of
806
- * spawning a duplicate. Each browser is scripted via its own literal-named
807
- * AppleScript, run ONLY when pgrep confirms it's running (so the literal `tell`
808
- * always compiles). Falls back to a plain `open` (new tab) when no running
809
- * browser has the tab, for browsers we don't script (e.g. Firefox), or if
810
- * AppleScript is blocked. Best-effort and fire-and-forget: any failure just
811
- * means a fresh tab, so it never regresses the plain-open path. The first focus
812
- * may trigger a one-time macOS "control your browser" permission prompt;
813
- * declining it just falls back to opening a new tab. */
814
- function openChatUi(url: string, hostPort: string): void {
815
- const browsers: Array<{ proc: string; kind: "chromium" | "safari" }> = [
816
- { proc: "Google Chrome", kind: "chromium" },
817
- { proc: "Arc", kind: "chromium" },
818
- { proc: "Brave Browser", kind: "chromium" },
819
- { proc: "Microsoft Edge", kind: "chromium" },
820
- { proc: "Safari", kind: "safari" },
821
- ];
822
- void (async () => {
823
- for (const b of browsers) {
824
- try {
825
- const running = Bun.spawn(["pgrep", "-x", b.proc], { stdout: "ignore", stderr: "ignore" });
826
- if ((await running.exited) !== 0) continue; // not running → skip (and don't compile its tell)
827
- const script = focusTabScript(b.proc, b.kind, hostPort);
828
- const p = Bun.spawn(["osascript", "-e", script], { stdout: "pipe", stderr: "ignore", timeout: 8000 });
829
- const out = await new Response(p.stdout).text();
830
- if ((await p.exited) === 0 && out.trim() === "ok") return; // focused an existing tab
831
- } catch { /* try the next browser */ }
832
- }
833
- try { Bun.spawn(["open", url], { stdout: "ignore", stderr: "ignore" }); } catch { /* best-effort */ }
834
- })();
835
- }
836
-
837
- /** Server/runtime flags shared by every mode that loads a model
838
- * (serve, pi). Runtime overrides are installed here before the
839
- * generate/compiled-decode modules read them. */
840
- /** Resolve the decode ROUTE: a tier alias (--l1/--l2) sets the whole route,
841
- * and an explicit per-fork flag (--kv-quant/--fused-sdpa/…) overrides the
842
- * alias. Installs decode runtime options and returns the kv-quant mode. See
843
- * docs/design/unified-engine-frontier-plan.md + unified-engine-frontier-plan.md.
844
- * Each tier is a GUARANTEE about which reference you reproduce bit-for-bit:
845
- * --l1 bit-for-bit IDENTICAL to mlx-lm — drop-in replacement for mlx-lm
846
- * (bf16 KV; THE naked default since 2026-07-05)
847
- * --l2 bit-for-bit IDENTICAL to mlx-optiq — drop-in replacement for
848
- * mlx-optiq: quantized KV per kv_config.json + fused N-tiled SDPA for
849
- * L>1 + STOCK unfused L=1 decode (the optiq-golden composition,
850
- * scripts/regen.ts kvq + regen-mixed-kv-goldens.ts).
851
- * There is no --l3: output-changing experiments live in the Lab (env flags
852
- * with a bench + expiry) until one beats the L1 baseline in a paired A/B —
853
- * docs/design/unified-engine-frontier-plan.md §6-7. */
854
- function applyDecodeRoute(): { kvQuant?: "off" | "config" | number; turboQuant?: TurboQuantScheme } {
855
- const onOff = (name: string): boolean | null => {
856
- const v = opt(name); if (v == null) return null;
857
- if (v === "on" || v === "1" || v === "true") return true;
858
- if (v === "off" || v === "0" || v === "false") return false;
859
- console.error(`--${name} expects on|off (got "${v}")`); process.exit(1);
860
- };
861
- if (flag("l3")) {
862
- console.error(
863
- "--l3 was removed 2026-07-05: no output-changing kernel beat the L1 baseline " +
864
- "in a paired A/B, so the tier had no content. Use --l2 for the optiq route, " +
865
- "or the Lab (docs/design/unified-engine-frontier-plan.md) for experiments.",
866
- );
867
- process.exit(1);
868
- }
869
- type Preset = { kv: "off" | "config"; fusedSdpa: boolean; compiled: boolean; compiledAct: boolean };
870
- const TIERS: Record<string, Preset> = {
871
- // A tier is the GUARANTEE; within it we default to the FAST kernel that still
872
- // holds it, and expose the slow one as an opt-in. compiled-decode is BIT-EXACT
873
- // with uncompiled (compiled-decode.test: on==off) → free speed, so it's ON in
874
- // EVERY tier (--compiled-decode off is the slow, same-tier opt-in). compiled
875
- // ACTIVATIONS (geglu/swiglu) go through the SAME libmlx as mlx-lm's `@mx.compile`
876
- // → same kernel, bit-exact AND faster → ON in every tier (--compiled-activations
877
- // off is the slower same-parity opt-in). fused-sdpa matches OPTIQ bit-for-bit
878
- // (tier-a goldens, fused-sdpa.test) and is quantized-KV only → it's the L2 bridge,
879
- // a no-op on L1's bf16.
880
- l1: { kv: "off", fusedSdpa: false, compiled: true, compiledAct: true }, // = mlx-lm bit-for-bit (bf16)
881
- l2: { kv: "config", fusedSdpa: true, compiled: true, compiledAct: true }, // = mlx-optiq bit-for-bit
882
- };
883
- // NAKED DEFAULT = L1 (decided 2026-07-05): with no tier flag you get the
884
- // mlx-lm bit-for-bit route. The 2026-07-05 benchmark pass showed the L1
885
- // faithful kernel set matches mlx-lm 1.00× on every model while every
886
- // output-changing lever failed to beat it in a paired A/B (the losers were
887
- // deleted the same day — Phase 1 of unified-engine-frontier-plan.md).
888
- // Quantized KV costs 5–20% decode at ≤16k on BOTH stacks and pays only in
889
- // memory/context headroom → explicit opt-in (--kv-quant / --l2).
890
- const tier = flag("l2") ? "l2" : "l1";
891
- const p = TIERS[tier]!;
892
- const kv = opt("kv-quant"); // explicit --kv-quant overrides the tier
893
- const pick = (name: string, base: boolean | undefined): boolean | null => {
894
- const ex = onOff(name); return ex !== null ? ex : (base ?? null); // explicit flag wins, else tier preset
895
- };
896
- const set = (name: RuntimeKey, val: boolean | null, invert = false) => {
897
- if (val !== null)
898
- configureRuntime({ [name]: (invert ? !val : val) ? "1" : "0" });
899
- };
900
- // Explicit --kv-quant picks the composition its ORACLE uses: config (the
901
- // optiq mixed scheme) runs optiq's fused SDPA (the L2 golden composition);
902
- // uniform 4/8 runs UNFUSED (mlx-lm's quantized_scaled_dot_product_attention
903
- // — the bit-exact L1-eligible scheme). --fused-sdpa still overrides either.
904
- const fusedSdpaBase = kv === "config" ? true : kv != null ? false : p.fusedSdpa;
905
- set("MLX_BUN_COMPILED_DECODE", pick("compiled-decode", p.compiled));
906
- set("MLX_BUN_NO_FUSED_SDPA", pick("fused-sdpa", fusedSdpaBase), true); // inverted env
907
- // Compiled activations (the faithful geglu/swiglu — mlx-lm's @mx.compile kernel).
908
- // One fork drives both runtime options; qwen3/qwen3.5/universal compile unconditionally,
909
- // so this toggles gemma geglu + minicpm5 swiglu (the only unfused-capable sites).
910
- set("MLX_BUN_COMPILED_GEGLU", pick("compiled-activations", p.compiledAct));
911
- set("MLX_BUN_COMPILED_SWIGLU", pick("compiled-activations", p.compiledAct));
912
- if (kv === "off") return { kvQuant: "off" };
913
- if (kv === "config") return { kvQuant: "config" };
914
- if (kv) {
915
- // TurboQuant (docs/design/turboquant.md): a distinct axis from the
916
- // affine kvQuant above — dequantize-on-fetch, stock unfused ops.sdpa,
917
- // no kvQuant value at all (mutually exclusive with kvBits/kvConfig).
918
- let turboQuant: TurboQuantScheme | null;
919
- try {
920
- turboQuant = parseTurboQuantScheme(kv);
921
- } catch (e) {
922
- console.error((e as Error).message);
923
- process.exit(1);
924
- }
925
- if (turboQuant) return { turboQuant };
926
- const bits = Number(kv);
927
- if (![4, 8].includes(bits)) { console.error(`--kv-quant expects config|off|4|8|turbo (got "${kv}")`); process.exit(1); }
928
- return { kvQuant: bits };
929
- }
930
- return { kvQuant: p.kv };
931
- }
932
-
933
- function serverRuntimeFlags(): { port: number; serverOptions: import("./server").ServerOptions } {
934
- const onOff = (name: string): boolean | null => {
935
- const v = opt(name);
936
- if (v === null) return null;
937
- if (v === "on" || v === "1" || v === "true") return true;
938
- if (v === "off" || v === "0" || v === "false") return false;
939
- console.error(`--${name} expects on|off (got "${v}")`);
940
- process.exit(1);
941
- };
942
- const route = applyDecodeRoute(); // --l1/--l2 tier alias, with per-fork flags overriding
943
- if (flag("force-wire")) configureRuntime({ MLX_BUN_FORCE_WIRE: "1" });
944
- // Remote image_url/audio_url media fetches block private/loopback/link-
945
- // local destinations by default (SSRF — src/media-fetch.ts); this is the
946
- // LAN-hosts escape hatch.
947
- if (flag("allow-private-media"))
948
- configureRuntime({ MLX_BUN_ALLOW_PRIVATE_MEDIA: "1" });
949
-
950
- const serverOptions: import("./server").ServerOptions = {};
951
- const budgetGB = Number(opt("memory-budget", "0"));
952
- if (budgetGB > 0) serverOptions.memoryBudgetBytes = budgetGB * 1e9;
953
- // Aggregate KV budget across batch rows (batching-perf-path P3 admission).
954
- const kvBudgetGB = Number(opt("kv-budget", "0"));
955
- if (kvBudgetGB > 0) serverOptions.kvBudgetBytes = kvBudgetGB * 1e9;
956
- const pcRaw = opt("prompt-cache"); // null if absent; an explicit value (incl. 0) wins → `--prompt-cache 0` DISABLES the cache
957
- if (pcRaw !== null) serverOptions.promptCacheBytes = Math.max(0, Number(pcRaw)) * 2 ** 30;
958
- // Engine-child mode (docs/reference/server-config.md): listen on a unix socket
959
- // instead of TCP — the isolated parent proxies here. Internal flag,
960
- // spawned by --isolate; usable directly for socket-level integration.
961
- const unixRaw = opt("unix");
962
- if (unixRaw !== null) serverOptions.unixSocket = unixRaw;
963
- // SSD cold tier (docs/design/kv-cache.md) — off unless a dir is given.
964
- const ssdDir = opt("ssd-cache");
965
- if (ssdDir !== null) {
966
- serverOptions.ssdCacheDir = ssdDir;
967
- const maxRaw = opt("ssd-cache-max");
968
- if (maxRaw !== null) {
969
- const maxGb = Number(maxRaw);
970
- if (!Number.isFinite(maxGb) || maxGb < 0) {
971
- console.error(`--ssd-cache-max expects a number >= 0 (got "${maxRaw}")`);
972
- process.exit(1);
973
- }
974
- serverOptions.ssdCacheMaxBytes = maxGb === 0
975
- ? Number.POSITIVE_INFINITY
976
- : maxGb * 2 ** 30;
977
- }
978
- if (flag("ssd-cache-verify")) serverOptions.ssdCacheVerify = true;
979
- const demoteRaw = opt("ssd-demote-idle");
980
- if (demoteRaw !== null) serverOptions.ssdDemoteIdleSec = Math.max(0, Number(demoteRaw));
981
- const checkpointRaw = opt("generation-checkpoint");
982
- if (checkpointRaw !== null) {
983
- const n = Number(checkpointRaw);
984
- if (!Number.isInteger(n) || n < 1) {
985
- console.error(`--generation-checkpoint expects an integer >= 1 (got "${checkpointRaw}")`);
986
- process.exit(1);
987
- }
988
- serverOptions.generationCheckpointTokens = n;
989
- }
990
- } else {
991
- // Refuse-loudly house rule: sub-flags of an absent tier must not vanish.
992
- for (const f of ["ssd-cache-max", "ssd-demote-idle", "generation-checkpoint"] as const)
993
- if (opt(f) !== null) console.warn(`--${f} has no effect without --ssd-cache — ignored`);
994
- if (flag("ssd-cache-verify"))
995
- console.warn("--ssd-cache-verify has no effect without --ssd-cache — ignored");
996
- }
997
- // --batch N: max concurrent requests batched through the mlx-lm-parity
998
- // engine (N=1 pins the strict serial executor). --decode-concurrency is accepted for
999
- // drop-in compatibility with mlx_lm.server, but the semantics differ: there
1000
- // it caps per-BatchGenerator decode parallelism (default 32); in mlx-bun it
1001
- // caps continuous batching (default 8; `--batch 1` pins the serial path —
1002
- // the unified engine is bit-parity with it and costs ~nothing at B=1).
1003
- const batchRaw = opt("batch") ?? opt("decode-concurrency");
1004
- if (batchRaw !== null) {
1005
- const n = Number(batchRaw);
1006
- if (!Number.isInteger(n) || n < 1) {
1007
- console.error(`--batch expects an integer >= 1 (got "${batchRaw}")`);
1008
- process.exit(1);
1009
- }
1010
- serverOptions.batch = n;
1011
- }
1012
- if (route.kvQuant !== undefined) serverOptions.kvQuant = route.kvQuant;
1013
- if (route.turboQuant !== undefined) serverOptions.turboQuant = route.turboQuant;
1014
- // --paged-kv (env: MLX_BUN_PAGED_KV=1): OPTIONAL vLLM-style block-pool KV
1015
- // storage, default off (docs/design/kv-cache.md). The state policy owns
1016
- // block pools and cache reuse; paging preserves the selected batch size.
1017
- if (flag("paged-kv") || runtimeValue("MLX_BUN_PAGED_KV") === "1") {
1018
- const bsRaw = opt("paged-kv-block-size");
1019
- serverOptions.pagedKv = bsRaw !== null ? { blockSize: Number(bsRaw) } : {};
1020
- }
1021
- // Bind loopback unless asked otherwise (mlx_lm.server parity); --host
1022
- // 0.0.0.0 is the explicit opt-in for LAN exposure. The chat-UI open and
1023
- // `mlx-bun pi` attach both go through localhost, so loopback-only is
1024
- // transparent to them.
1025
- serverOptions.hostname = opt("host", "127.0.0.1")!;
1026
- // Server-wide default for the chat template's enable_thinking variable
1027
- // (CPM/MiniCPM5 and other hybrid-reasoning models). Unset ⇒ the model's
1028
- // own default (false for MiniCPM5). onOff accepts true|false|on|off|1|0.
1029
- const thinking = onOff("thinking");
1030
- if (thinking !== null) serverOptions.defaultThinking = thinking;
1031
- // Server-wide sampling defaults. Unset ⇒ the model's generation_config.json
1032
- // (MiniCPM5 ships 0.9 / 0.95 — its think recipe). A per-request field still
1033
- // overrides these; the browser chat sends none, so this is its only lever.
1034
- const numFlag = (name: string, lo: number, hi: number): number | null => {
1035
- const v = opt(name);
1036
- if (v === null) return null;
1037
- const n = Number(v);
1038
- if (!Number.isFinite(n) || n < lo || n > hi) {
1039
- console.error(`--${name} expects a number in [${lo}, ${hi}] (got "${v}")`);
1040
- process.exit(1);
1041
- }
1042
- return n;
1043
- };
1044
- // --temp is the mlx_lm.server spelling; explicit --temperature wins.
1045
- const temp = numFlag("temperature", 0, 5) ?? numFlag("temp", 0, 5);
1046
- if (temp !== null) serverOptions.defaultTemperature = temp;
1047
- const topP = numFlag("top-p", 0, 1);
1048
- if (topP !== null) serverOptions.defaultTopP = topP;
1049
- const topK = numFlag("top-k", 0, 1_000_000);
1050
- if (topK !== null) serverOptions.defaultTopK = topK;
1051
- // mlx_lm.server's --max-tokens: the completion cap when a request omits
1052
- // max_tokens. Our built-in default stays 65,536 (chat) / 512 (raw text
1053
- // completion); `--max-tokens 512` reproduces mlx-lm's behavior exactly.
1054
- const maxTok = numFlag("max-tokens", 1, 10_000_000);
1055
- if (maxTok !== null) serverOptions.defaultMaxTokens = Math.floor(maxTok);
1056
- // HLG sampling — piecewise tone curve on the logprobs (default off). Knobs in
1057
- // nats; the mid gain folds from --temperature. docs/archive/hlg-sampling.md.
1058
- if (onOff("hlg-sampling") === true) {
1059
- serverOptions.hlg = {
1060
- enabled: true,
1061
- width: numFlag("hlg-width", 0, 100) ?? 4,
1062
- shoulder: numFlag("hlg-shoulder", 0, 100) ?? 4,
1063
- toe: numFlag("hlg-toe", 0, 100) ?? 6,
1064
- pivotOffset: numFlag("hlg-pivot-offset", 0, 100) ?? 6,
1065
- pivot: "top",
1066
- };
1067
- }
1068
- return { port: Number(opt("port", "8080")), serverOptions };
1069
- }
1070
-
1071
- /** Direct Colibri runtime defaults shared by `serve` and the embedded `pi`
1072
- * server. Other model families ignore this load-only option block. */
1073
- function glm52RuntimeFlags(
1074
- serverOptions: import("./server").ServerOptions,
1075
- ): import("./model/factory").Glm52RuntimeOpenOptions {
1076
- const mtpRaw = opt("mtp");
1077
- let enableMtp: boolean | undefined;
1078
- if (mtpRaw !== null) {
1079
- if (["on", "1", "true"].includes(mtpRaw)) enableMtp = true;
1080
- else if (["off", "0", "false"].includes(mtpRaw)) enableMtp = false;
1081
- else {
1082
- console.error(`--mtp expects on|off (got "${mtpRaw}")`);
1083
- process.exit(1);
1084
- }
1085
- }
1086
- const contextRaw = opt("context-length");
1087
- let contextTokens: number | undefined;
1088
- if (contextRaw !== null) {
1089
- contextTokens = Number(contextRaw);
1090
- if (!Number.isSafeInteger(contextTokens) || contextTokens < 1) {
1091
- console.error(`--context-length expects an integer >= 1 (got "${contextRaw}")`);
1092
- process.exit(1);
1093
- }
1094
- }
1095
- return {
1096
- memoryBudgetBytes: serverOptions.memoryBudgetBytes,
1097
- batchSize: serverOptions.batch ?? 8,
1098
- maxGenerationTokens: serverOptions.defaultMaxTokens ?? 128,
1099
- ...(contextTokens !== undefined ? { contextTokens } : {}),
1100
- ...(enableMtp !== undefined ? { enableMtp } : {}),
1101
- };
1102
- }
1103
-
1104
- /** `--adapter <dir>` (alias `--adapter-path`, mlx_lm.server's spelling):
1105
- * mount a LoRA adapter at startup — the same machinery as POST /v1/adapters —
1106
- * and make it the default for requests that don't select one (a request's
1107
- * explicit `adapter`, including "none", still wins; hot-swap unchanged).
1108
- * This is the flag `mlx-bun train`'s completion message points at. Shared by
1109
- * `serve` and the `pi` session server. Exits on a bad adapter — better a
1110
- * loud startup failure than silently serving the base model. */
1111
- async function mountStartupAdapter(
1112
- ctx: import("./server").ServerContext,
1113
- serverOptions: import("./server").ServerOptions,
1114
- ): Promise<void> {
1115
- const adapterDir = opt("adapter") ?? opt("adapter-path");
1116
- if (!adapterDir) return;
1117
- const { step, style } = await import("./tui");
1118
- const sAdp = step("mounting adapter");
1119
- try {
1120
- const adapterId = adapterDir.replace(/\/+$/, "").split("/").pop()!;
1121
- const info = await ctx.adapters.mount(adapterId, adapterDir);
1122
- serverOptions.defaultAdapter = info.id;
1123
- sAdp.done(`adapter ${style.bold(info.id)} ${style.dim(`· ${info.mountedLayers} layers · default for requests (select others via \`adapter\`)`)}`);
1124
- } catch (e) {
1125
- sAdp.fail(`adapter mount failed: ${(e as Error).message}`);
1126
- process.exit(1);
1127
- }
1128
- }
1129
-
1130
- /** One-line summary of the active runtime levers for the ready card. */
1131
- function runtimeSummary(o: import("./server").ServerOptions): string {
1132
- const kv = o.turboQuant ? `turbo-k${o.turboQuant.kBits}v${o.turboQuant.vBits}`
1133
- : o.kvQuant === "off" ? "off" : typeof o.kvQuant === "number" ? `kv${o.kvQuant}` : "config";
1134
- const lever = (name: RuntimeKey, dflt: string) => runtimeValue(name) ?? dflt;
1135
- return `kv-quant ${kv} · compiled-decode ${lever("MLX_BUN_COMPILED_DECODE", "1") === "1" ? "on" : "off"}` +
1136
- (lever("MLX_BUN_COMPILED_GEGLU", "1") === "0" ? " · compiled-activations off" : "") +
1137
- (o.batch && o.batch > 1 ? ` · batch ${o.batch}` : "") +
1138
- (o.defaultThinking !== undefined ? ` · thinking ${o.defaultThinking ? "on" : "off"}` : "") +
1139
- (o.defaultTemperature !== undefined ? ` · temp ${o.defaultTemperature}` : "") +
1140
- (o.defaultTopP !== undefined ? ` · top-p ${o.defaultTopP}` : "") +
1141
- (o.defaultTopK !== undefined ? ` · top-k ${o.defaultTopK}` : "");
1142
- }
1143
-
1144
- /** Shared model resolution: explicit query wins; otherwise e4b (the default
1145
- * model everywhere) if it's downloaded, else the largest supported model that
1146
- * fits this machine. Downloads the recommended model first on a fresh install.
1147
- * A query that is a DIRECTORY containing config.json loads directly via
1148
- * scanSnapshot (no registry dependency) — the `--model <path>` bench/CI shape. */
1149
- async function resolveModelAuto(query: string | null): Promise<{ m: import("./registry").ModelRecord; picked: boolean }> {
1150
- if (query && existsSync(join(query, "config.json"))) {
1151
- const { scanSnapshot } = await import("./registry");
1152
- const dir = resolve(query);
1153
- // Recover the repo id from an HF-cache path (…/models--org--name/snapshots/<rev>);
1154
- // otherwise the directory basename is the display id.
1155
- const hf = /models--([^/]+)--([^/]+)\/snapshots\//.exec(dir);
1156
- const repoId = hf ? `${hf[1]}/${hf[2]}` : basename(dir);
1157
- const m = await scanSnapshot(dir, repoId);
1158
- if (!m) throw new Error(`${dir} has config.json but no weights (*.safetensors)`);
1159
- return { m, picked: false };
1160
- }
1161
- const reg = new Registry();
1162
- if (reg.list().length === 0) await reg.scan();
1163
- if (query) return { m: reg.resolve(query), picked: false };
1164
- const { recommendedRepoId } = await import("./fit");
1165
- let candidates = reg.list().filter((r) => isSupportedModelRecord(r.modelType, r.repoId));
1166
- if (candidates.length === 0) {
1167
- // First run: starter-model flow (PRODUCT_ROADMAP / PLAN Phase 16).
1168
- // Download the sub-GB starter first so the user is chatting quickly,
1169
- // then stream the recommended Gemma for this Mac in the background —
1170
- // it becomes the auto-pick next run.
1171
- const starterRepo = "mlx-community/MiniCPM5-1B-OptiQ-4bit";
1172
- const recRepo = recommendedRepoId();
1173
- const { step, style } = await import("./tui");
1174
- const { downloadModel } = await import("./download");
1175
- const s = step(`downloading starter model ${starterRepo}`);
1176
- await downloadModel(starterRepo, {
1177
- onProgress: (file, received, total) => {
1178
- const pct = total ? Math.floor((received / total) * 100) : 0;
1179
- s.update(`starter model ${starterRepo} — ${file} ${gb(received)} / ${gb(total)} (${pct}%)`);
1180
- },
1181
- });
1182
- s.done(`starter model ready ${style.dim(`· ${starterRepo}`)}`);
1183
- await reg.scan();
1184
- if (recRepo !== starterRepo) {
1185
- console.log(` ${style.dim(`downloading ${recRepo} (recommended for this Mac) in the background —`)}`);
1186
- console.log(` ${style.dim("it becomes the default next run · progress in the web app's Status view · resumable")}`);
1187
- downloadModel(recRepo)
1188
- .then(async () => {
1189
- await new Registry().scan();
1190
- console.log(` ${style.dim(`background download complete: ${recRepo} (used on next run)`)}`);
1191
- })
1192
- .catch(() => { /* resumable — the next run continues it */ });
1193
- }
1194
- candidates = reg.list().filter((r) => isSupportedModelRecord(r.modelType, r.repoId));
1195
- }
1196
- // No query: e4b by default; otherwise the largest model that still leaves
1197
- // the machine usable for other apps. Never auto-grab the 26B "dedicate the
1198
- // machine" model on a small-RAM Mac — that's an explicit --query choice —
1199
- // but allow it as a last resort if it's all the user has downloaded.
1200
- const { DEFAULT_REPO_ID, COEXIST_FRACTION, chooseAutoModel, thisMachine } = await import("./fit");
1201
- const machine = thisMachine();
1202
- const fitsFull = new Map<string, boolean>();
1203
- const fitsCoexist = new Map<string, boolean>();
1204
- for (const r of candidates) {
1205
- const config = await loadModelConfig(r.path);
1206
- fitsFull.set(r.repoId, fit(config, r.sizeBytes, 8192, machine, undefined, r.expertsBytes).fits);
1207
- fitsCoexist.set(
1208
- r.repoId,
1209
- fit(config, r.sizeBytes, 8192, machine, undefined, r.expertsBytes, machine.ramBytes * COEXIST_FRACTION).fits,
1210
- );
1211
- }
1212
- const chosen = chooseAutoModel(
1213
- candidates,
1214
- DEFAULT_REPO_ID,
1215
- (c) => fitsFull.get(c.repoId) ?? false,
1216
- (c) => fitsCoexist.get(c.repoId) ?? false,
1217
- );
1218
- if (chosen) return { m: chosen, picked: true };
1219
- console.error("no downloaded supported model fits this machine — pick one explicitly (mlx-bun ls)");
1220
- process.exit(1);
1221
- }
1222
-
1223
- function parseSize(s: string): number {
1224
- const m = /^([\d.]+)\s*(GB|MB|G|M)?$/i.exec(s.trim());
1225
- if (!m) throw new Error(`bad size: ${s}`);
1226
- const mult = /^m/i.test(m[2] ?? "G") ? 2 ** 20 : 2 ** 30;
1227
- return Number(m[1]) * mult;
1228
- }
1229
-
1230
- switch (cmd) {
1231
- case "get": {
1232
- let repoId = positional(0);
1233
- if (!repoId) {
1234
- console.error("usage: mlx-bun get <org/repo | substring> [--revision main]");
1235
- process.exit(1);
1236
- }
1237
- if (!repoId.includes("/")) {
1238
- // No org/name — treat it as a registry query over already-downloaded
1239
- // repos (re-get/refresh, e.g. `mlx-bun get 12B` after an upstream push).
1240
- try {
1241
- const regQ = new Registry();
1242
- if (regQ.list().length === 0) await regQ.scan();
1243
- repoId = regQ.resolve(repoId).repoId;
1244
- } catch (e) {
1245
- console.error((e as Error).message);
1246
- console.error(`usage: mlx-bun get <org/repo> [--revision main] — or a substring of a downloaded repo (try \`mlx-bun ls ${repoId}\`)`);
1247
- process.exit(1);
1248
- }
1249
- }
1250
- const { downloadModel } = await import("./download");
1251
- const { step, style } = await import("./tui");
1252
- const { join: joinPath, basename } = await import("node:path");
1253
- const { readdirSync, existsSync } = await import("node:fs");
1254
- const { DEFAULT_HUB, planRepoGc } = await import("./registry");
1255
- const repoDir = joinPath(DEFAULT_HUB, `models--${repoId.replaceAll("/", "--")}`);
1256
- const priorSnapshots = existsSync(joinPath(repoDir, "snapshots"))
1257
- ? readdirSync(joinPath(repoDir, "snapshots"))
1258
- : [];
1259
- const s = step(`downloading ${repoId}`);
1260
- const snap = await downloadModel(repoId, {
1261
- revision: opt("revision", "main")!,
1262
- onProgress: (file, received, total) => {
1263
- const pct = total ? Math.floor((received / total) * 100) : 0;
1264
- s.update(`${style.bold(repoId)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
1265
- },
1266
- });
1267
- s.done(`${style.bold(repoId)} ${style.dim("downloaded · verified")}`);
1268
- const sScan = step("updating registry");
1269
- const reg = new Registry();
1270
- await reg.scan();
1271
- sScan.done(`registry updated ${style.dim(`· ${snap}`)}`);
1272
- // Upstream pushed a new revision → this get created a NEW snapshots/<sha>
1273
- // dir; the old one stays on disk (that's how ~25 GB of dead blobs pile
1274
- // up). Say so, with what `gc` would get back.
1275
- if (priorSnapshots.length > 0 && !priorSnapshots.includes(basename(snap))) {
1276
- const plan = planRepoGc(repoDir);
1277
- const note = plan.skippedSnapshots.length > 0
1278
- ? " (a previous snapshot has files this revision lacks — gc will warn; --force to prune it)"
1279
- : "";
1280
- console.log(style.dim(
1281
- ` previous revision kept on disk — \`mlx-bun gc\` reclaims ~${gb(plan.reclaimBytes)}${note}`,
1282
- ));
1283
- }
1284
- break;
1285
- }
1286
-
1287
- case "scan": {
1288
- const { step } = await import("./tui");
1289
- const s = step("scanning the Hugging Face cache");
1290
- const reg = new Registry();
1291
- const n = await reg.scan();
1292
- s.done(`indexed ${n} model snapshot(s)`);
1293
- break;
1294
- }
1295
-
1296
- case "ls": {
1297
- const reg = new Registry();
1298
- if ((reg.list().length ?? 0) === 0) await reg.scan();
1299
- const maxSize = opt("max-size");
1300
- const allRevisions = flag("all-revisions");
1301
- const lsFilter = {
1302
- vision: flag("vision") ? true : undefined,
1303
- maxBytes: maxSize ? parseSize(maxSize) : undefined,
1304
- query: positional(0),
1305
- };
1306
- // One row per repo by default: the HF cache keeps a snapshots/<commit>
1307
- // dir per downloaded revision, and re-getting after an upstream push
1308
- // strands the old one — those are duplicates, not separate models.
1309
- // --all-revisions shows the per-snapshot truth (canonical marked *).
1310
- const models = allRevisions ? reg.list(lsFilter) : reg.listCanonical(lsFilter);
1311
- if (models.length === 0) {
1312
- console.log("no models match (try `mlx-bun scan`)");
1313
- break;
1314
- }
1315
- const { visionCapable } = await import("./registry");
1316
- const { supportTier } = await import("./model/support");
1317
- const canonicalPaths = new Set(reg.listCanonical(lsFilter).map((m) => m.path));
1318
- const capabilities = (m: (typeof models)[number]) => {
1319
- const tier = supportTier(m.modelType, m.repoId);
1320
- return [
1321
- tier ? `supported (${tier})` : `unsupported (${m.modelType})`,
1322
- visionCapable(m) ? "vision" : null,
1323
- m.hasToolTemplate ? "tools" : null,
1324
- m.hasKvConfig ? "kv-quant" : null,
1325
- ].filter(Boolean).join(" · ");
1326
- };
1327
- const { table, style, h1 } = await import("./tui");
1328
- h1("library");
1329
- console.log();
1330
- table(
1331
- [
1332
- { header: "model", paint: (c) => style.bold(c) },
1333
- ...(allRevisions ? [{ header: "revision" }] : []),
1334
- { header: "size", align: "right" as const },
1335
- { header: "params", align: "right" as const },
1336
- { header: "quant" },
1337
- { header: "license", paint: (c: string) => style.dim(c) },
1338
- { header: "capabilities", paint: (c: string) => style.dim(c) },
1339
- ],
1340
- models.map((m) => [
1341
- m.repoId,
1342
- ...(allRevisions
1343
- ? [`${(m.path.split("/snapshots/")[1] ?? "").slice(0, 12)}${canonicalPaths.has(m.path) ? " *" : ""}`]
1344
- : []),
1345
- gb(m.sizeBytes),
1346
- m.paramCount ? `${(m.paramCount / 1e9).toFixed(1)}B` : "?",
1347
- m.quantBits ? `${m.quantBits}-bit g${m.quantGroupSize}` : "full",
1348
- m.license ?? "?",
1349
- capabilities(m),
1350
- ]),
1351
- );
1352
- console.log();
1353
- if (allRevisions) {
1354
- console.log(style.dim(
1355
- ` ${models.length} snapshot(s) · * = canonical (refs/main) · superseded snapshots: \`mlx-bun gc\``,
1356
- ));
1357
- } else {
1358
- console.log(style.dim(` ${models.length} model(s) · mlx-bun fit <query> for a memory assessment`));
1359
- }
1360
- break;
1361
- }
1362
-
1363
- case "gc": {
1364
- const { planGc, executeGc, DEFAULT_HUB } = await import("./registry");
1365
- const { table, style, h1 } = await import("./tui");
1366
- const force = flag("force");
1367
- const plans = planGc(DEFAULT_HUB, { force }).filter(
1368
- (p) => p.pruneSnapshots.length || p.skippedSnapshots.length || p.deadBlobs.length,
1369
- );
1370
- h1("gc — superseded snapshots + dead blobs");
1371
- console.log();
1372
- if (plans.length === 0) {
1373
- console.log(" nothing to reclaim — every snapshot is referenced by refs/*");
1374
- break;
1375
- }
1376
- table(
1377
- [
1378
- { header: "repo", paint: (c) => style.bold(c) },
1379
- { header: "keep", align: "right" },
1380
- { header: "prune", align: "right" },
1381
- { header: "skip", align: "right" },
1382
- { header: "reclaims", align: "right" },
1383
- ],
1384
- plans.map((p) => [
1385
- p.repoId,
1386
- String(p.keepSnapshots.length),
1387
- String(p.pruneSnapshots.length),
1388
- String(p.skippedSnapshots.length),
1389
- gb(p.reclaimBytes),
1390
- ]),
1391
- );
1392
- console.log();
1393
- for (const p of plans) {
1394
- for (const s of p.skippedSnapshots) {
1395
- const rev = (s.path.split("/snapshots/")[1] ?? s.path).slice(0, 12);
1396
- console.log(style.accent(
1397
- ` WARNING ${p.repoId}@${rev}: superseded snapshot has files the canonical revision lacks — skipped.`,
1398
- ));
1399
- console.log(style.dim(` only copy of: ${s.extraFiles.join(", ")}`));
1400
- console.log(style.dim(" deleting it loses those files — rerun with --force if that's intended."));
1401
- }
1402
- }
1403
- const totalReclaim = plans.reduce((a, p) => a + p.reclaimBytes, 0);
1404
- const totalSnaps = plans.reduce((a, p) => a + p.pruneSnapshots.length, 0);
1405
- const totalBlobs = plans.reduce((a, p) => a + p.deadBlobs.length, 0);
1406
- console.log(
1407
- ` total: ${totalSnaps} snapshot(s), ${totalBlobs} blob(s), ${style.bold(gb(totalReclaim))}`,
1408
- );
1409
- if (flag("dry-run") || !flag("yes")) {
1410
- console.log(style.dim(" nothing deleted — rerun with --yes to delete (destructive)."));
1411
- break;
1412
- }
1413
- const res = executeGc(plans);
1414
- console.log(
1415
- ` deleted ${res.snapshots} snapshot(s) + ${res.blobs} blob(s) — reclaimed ${style.bold(gb(res.reclaimedBytes))}`,
1416
- );
1417
- const reg = new Registry();
1418
- await reg.scan(); // reap deleted snapshots from the registry
1419
- break;
1420
- }
1421
-
1422
- case "fit": {
1423
- const query = positional(0);
1424
- if (!query) { console.error("usage: mlx-bun fit <query> [--ctx N] [--kv-quant 4|8|config] [--skus]"); process.exit(1); }
1425
- const reg = new Registry();
1426
- if (reg.list().length === 0) await reg.scan();
1427
- const m = reg.resolve(query);
1428
- const config = await loadModelConfig(m.path);
1429
- const { box, table, style, h1, gradient } = await import("./tui");
1430
- const glm = isGlm52Config(config);
1431
- const ctx = Number(opt("ctx", glm ? "4096" : "8192"));
1432
- if (glm) {
1433
- const machine = thisMachine();
1434
- let plan;
1435
- try {
1436
- plan = await planGlm52MemoryForArtifact(m.path, {
1437
- machineBytes: machine.ramBytes,
1438
- processLimitBytes: Math.min(
1439
- GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES,
1440
- machine.ramBytes,
1441
- ),
1442
- contextTokens: ctx,
1443
- maxGenerationTokens: Math.min(
1444
- ctx,
1445
- GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS,
1446
- ),
1447
- batchSize: 1,
1448
- enableMtp: true,
1449
- });
1450
- } catch (error) {
1451
- console.error(
1452
- `GLM-5.2 resource plan refused: ` +
1453
- `${error instanceof Error ? error.message : String(error)}`,
1454
- );
1455
- process.exitCode = 1;
1456
- break;
1457
- }
1458
- const gib = (bytes: number): string => `${(bytes / 2 ** 30).toFixed(2)} GiB`;
1459
- const li = plan.lineItems;
1460
- const kvBytes = li.targetKvBytes + li.mtpKvBytes;
1461
- const otherBytes = plan.plannedProcessBytes - li.residentWeightsBytes -
1462
- li.mainExpertSlabBytes - li.mtpExpertSlabBytes - kvBytes;
1463
- h1("will it fit?");
1464
- console.log(` ${style.bold(m.repoId)} ${style.dim(`@ ${ctx.toLocaleString()} context · streamed GLM plan`)}`);
1465
- console.log();
1466
- box([
1467
- `artifact on disk ${gib(m.sizeBytes).padStart(10)} ${style.dim("(streamed, not resident)")}`,
1468
- `resident weights ${gib(li.residentWeightsBytes).padStart(10)}`,
1469
- `main expert slab ${gib(li.mainExpertSlabBytes).padStart(10)}`,
1470
- `MTP expert slab ${gib(li.mtpExpertSlabBytes).padStart(10)}`,
1471
- `target + MTP KV ${gib(kvBytes).padStart(10)}`,
1472
- `runtime reserves ${gib(otherBytes).padStart(10)}`,
1473
- `process plan ${gib(plan.plannedProcessBytes).padStart(10)} ${style.dim(`of ${gib(plan.processLimitBytes)}`)} ${style.green(style.bold("FITS"))}`,
1474
- `macOS headroom ${gib(plan.machineHeadroomBytes).padStart(10)}`,
1475
- "",
1476
- `max safe context ${style.bold(plan.contextTokens.toLocaleString())} tokens`,
1477
- `max generation ${style.bold(plan.maxGenerationTokens.toLocaleString())} tokens`,
1478
- `measured warm ${gradient(`${GLM52_G5_MEASURED_WARM_DECODE_TPS.toFixed(3)} tok/s`)} ${style.dim("(quality-preserving, MTP on)")}`,
1479
- `direct oracle ${GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS.toFixed(2)} tok/s`,
1480
- `aspirational ${GLM52_G5_ASPIRATIONAL_DECODE_TPS.toFixed(1)} tok/s ${style.dim("(not a fit prediction or release gate)")}`,
1481
- ]);
1482
- if (flag("skus")) {
1483
- console.log(style.dim(
1484
- " GLM-5.2's direct-container preset is validated on the M1 Max 32 GB; " +
1485
- "the generic resident-weight SKU projection does not apply.",
1486
- ));
1487
- }
1488
- console.log();
1489
- break;
1490
- }
1491
- // --kv-quant mirrors serve: bill the quantized cache's true bytes so
1492
- // the reported window matches what serving with the same flag admits.
1493
- const kvQuantOpt = opt("kv-quant");
1494
- let fitKvScheme: FitKvScheme | undefined;
1495
- if (kvQuantOpt === "4" || kvQuantOpt === "8") {
1496
- fitKvScheme = resolveKvScheme({ override: Number(kvQuantOpt) }).fitOptions;
1497
- } else if (kvQuantOpt === "config") {
1498
- if (!config.kvQuant?.length) {
1499
- console.error(`${m.repoId} ships no per-layer kv-quant config — use --kv-quant 4|8`);
1500
- process.exit(1);
1501
- }
1502
- fitKvScheme = resolveKvScheme({
1503
- override: "config",
1504
- config: config.kvQuant,
1505
- missingConfig: "error",
1506
- }).fitOptions;
1507
- } else if (kvQuantOpt != null && kvQuantOpt !== "off") {
1508
- console.error(`--kv-quant ${kvQuantOpt}: expected 4, 8, config, or off`);
1509
- process.exit(1);
1510
- }
1511
- const r = fit(config, m.sizeBytes, ctx, thisMachine(), undefined, m.expertsBytes, undefined, fitKvScheme);
1512
- h1("will it fit?");
1513
- const kvNote = fitKvScheme
1514
- ? style.dim(` · kv-quant ${fitKvScheme.kvBits ? `${fitKvScheme.kvBits}-bit` : "config"}`)
1515
- : "";
1516
- console.log(` ${style.bold(m.repoId)} ${style.dim(`@ ${ctx.toLocaleString()} context · this machine`)}${kvNote}`);
1517
- console.log();
1518
- const expertsNote = config.text.enableMoeBlock && m.expertsBytes > 0
1519
- ? style.dim(` (experts ${gb(m.expertsBytes)}; top ${config.text.topKExperts}/${config.text.numExperts} read per token)`)
1520
- : "";
1521
- box([
1522
- `weights ${gb(r.weightsBytes).padStart(9)}${expertsNote}`,
1523
- ...(m.sidecarBytes > 0
1524
- ? [style.dim(` + vision sidecar ${gb(m.sidecarBytes)} (bf16, loads only for vision)`)] : []),
1525
- `kv cache ${gb(r.kvBytes).padStart(9)}`,
1526
- `transient ${gb(r.transientBytes).padStart(9)}`,
1527
- `total ${gb(r.totalBytes).padStart(9)} ${style.dim(`of ${gb(r.usableBytes)} usable`)} ${r.fits ? style.green(style.bold("FITS")) : style.bold("DOES NOT FIT")}`,
1528
- "",
1529
- `max safe context ${style.bold(r.maxSafeContext.toLocaleString())} tokens`,
1530
- `predicted decode ${gradient(`${r.predictedDecodeTps.toFixed(1)} tok/s`)}`,
1531
- ]);
1532
- if (flag("skus")) {
1533
- h1("apple silicon matrix");
1534
- console.log();
1535
- table(
1536
- [
1537
- { header: "chip" }, { header: "ram", align: "right" },
1538
- { header: "fits", paint: (c) => (c.includes("fits") ? style.green(c) : style.dim(c)) },
1539
- { header: "max context", align: "right" }, { header: "decode", align: "right" },
1540
- ],
1541
- skuMatrix(config, m.sizeBytes, ctx, m.expertsBytes, fitKvScheme).map((row) => [
1542
- row.sku, `${row.ramGB} GB`, row.fits ? "fits" : "—",
1543
- row.fits ? row.maxContext.toLocaleString() : "—",
1544
- row.fits ? `~${row.decodeTps.toFixed(0)} tok/s` : "—",
1545
- ]),
1546
- );
1547
- }
1548
- console.log();
1549
- break;
1550
- }
1551
-
1552
- case "serve": {
1553
- const { banner, step, box, style } = await import("./tui");
1554
- banner(pkg.version);
1555
- if (noSubcommand)
1556
- console.log(style.dim(" no command given — starting the server · mlx-bun --help for all commands"));
1557
- const rt = serverRuntimeFlags();
1558
- // Friendly collision check before loading gigabytes of weights.
1559
- if (!rt.serverOptions.unixSocket) {
1560
- const { probeServer } = await import("./harness-pi");
1561
- const running = await probeServer(`http://localhost:${rt.port}/v1`);
1562
- if (running) {
1563
- console.error(`port ${rt.port} is already serving ${running.map((m) => m.id).join(", ")}.`);
1564
- console.error("reuse it (mlx-bun pi attaches automatically), stop it, or pick --port <other>.");
1565
- console.error("NOTE: a second server is a second model in memory — check `mlx-bun fit` first.");
1566
- process.exit(1);
1567
- }
1568
- }
1569
- // Runtime isolation (--isolate, docs/reference/server-config.md): THIS process
1570
- // becomes a pure reactor — it binds the TCP port and reverse-proxies to
1571
- // an engine CHILD (the full server on a unix socket, spawned from the
1572
- // same argv minus parent-only flags). Zero MLX calls in the parent: the
1573
- // UI stays instant under any GPU load, and an engine crash respawns
1574
- // without taking the session down.
1575
- if (!flag("isolate") && opt("model-pool") !== null)
1576
- console.warn("--model-pool has no effect without --isolate (child-per-model pool) — ignored");
1577
- if (flag("isolate")) {
1578
- const { startProxyServer, engineArgvForModel, defaultSocketPath } = await import("./serve/isolate");
1579
- const sock = defaultSocketPath();
1580
- // Re-spawn ourselves: compiled binary = execPath alone (argv[1] is a
1581
- // $bunfs virtual path); dev = bun + the script path.
1582
- const selfArgv = process.argv[1] && !process.argv[1].includes("$bunfs")
1583
- ? [process.execPath, process.argv[1]]
1584
- : [process.execPath];
1585
- const rawArgs = process.argv.slice(2);
1586
- // Resolve the DEFAULT model in the parent (registry lookup — no
1587
- // weights load) so the pool has its key and the child gets a pinned
1588
- // --model path instead of re-running fuzzy resolution.
1589
- const { m: m0 } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
1590
- // STRICT resolver for the pool (P2): only EXACT /v1/models ids
1591
- // switch models. Fuzzy matching here would hijack drop-in clients
1592
- // that send model:"gpt-4"-style strings (which must keep today's
1593
- // ignored-field semantics and ride the default engine).
1594
- const { Registry } = await import("./registry");
1595
- const poolResolve = (query: string): { repoId: string; path: string } | null => {
1596
- if (query === m0.repoId) return { repoId: m0.repoId, path: m0.path };
1597
- try {
1598
- const reg = new Registry();
1599
- try {
1600
- const hit = reg.listCanonical().find((r) => r.repoId === query);
1601
- return hit ? { repoId: hit.repoId, path: hit.path } : null;
1602
- } finally {
1603
- reg.close();
1604
- }
1605
- } catch {
1606
- return null;
1607
- }
1608
- };
1609
- const poolMaxRaw = opt("model-pool");
1610
- const poolMax = Math.max(1, Number(poolMaxRaw ?? 1) || 1);
1611
- const { engine, close } = startProxyServer({
1612
- port: rt.port,
1613
- ...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
1614
- engine: {
1615
- argv: [...selfArgv, ...engineArgvForModel(rawArgs, sock, m0.path)],
1616
- socketPath: sock,
1617
- },
1618
- pool: {
1619
- rawArgs, selfArgv, poolMax,
1620
- resolve: poolResolve,
1621
- defaultKey: m0.repoId,
1622
- },
1623
- });
1624
- const shownHost0 = rt.serverOptions.hostname ?? "localhost";
1625
- console.log(style.dim(` isolated mode: engine child starting (socket ${sock})`));
1626
- console.log(` ${style.green("●")} ${style.bold("proxy up")} http://${shownHost0}:${rt.port} ${style.dim(`· ${m0.repoId} loading behind it · model pool ${poolMax}`)}`);
1627
- const onExit = () => { void close().finally(() => process.exit(0)); };
1628
- process.on("SIGINT", onExit);
1629
- process.on("SIGTERM", onExit);
1630
- await engine.ready;
1631
- console.log(` ${style.green("●")} ${style.bold("engine ready")} ${style.dim(`pid ${engine.pid}`)}`);
1632
- await new Promise(() => {}); // serve until killed
1633
- }
1634
- // --model <path-or-query> is the explicit override (mlx_lm.server's
1635
- // spelling — previously accepted-but-IGNORED here, which silently
1636
- // auto-picked the wrong model). A directory with config.json loads
1637
- // straight from that path; anything else resolves like the positional
1638
- // fuzzy query. Precedence: --model > positional > --query > auto-pick.
1639
- const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
1640
- const sFit = step("assessing fit");
1641
- const config = await loadModelConfig(m.path);
1642
- const report = fit(config, m.sizeBytes, 8192, undefined, undefined, m.expertsBytes);
1643
- sFit.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)} · ~${report.predictedDecodeTps.toFixed(0)} tok/s predicted`)}${picked ? style.dim(" · auto-picked (override: mlx-bun serve <query>)") : ""}`);
1644
- const sNative = step("native runtime");
1645
- await ensureNative(sNative);
1646
- sNative.done("native runtime ready");
1647
- const { createServer, loadContext, shutdownServer } = await import("./server");
1648
- if (flag("expert-offload")) {
1649
- if (m.expertsBytes === 0) {
1650
- console.error(style.dim("--expert-offload ignored: this model has no experts (dense)"));
1651
- } else {
1652
- const sOff = step("expert offload");
1653
- const { ensureOffloadFile } = await import("./expert-offload-build");
1654
- const { activateExpertOffload } = await import("./expert-offload");
1655
- // builds <model>/.mlx-bun-offload on first use (~one-time), then reuses
1656
- const dir = await ensureOffloadFile(m.path, (msg) => sOff.update(msg));
1657
- activateExpertOffload(dir);
1658
- sOff.done(`experts mmap'd ${style.dim(dir)} ${style.dim("· phys_footprint ≈ core")}`);
1659
- }
1660
- }
1661
- // Speculative decoding (mlx_lm.server parity): --draft-model <path|query>
1662
- // resolves like the main model; --num-draft-tokens defaults to 3 (the
1663
- // server's default upstream — mlx_lm.generate's is 2). Qualified providers
1664
- // use shared execution; ordinary requests retain their own placement.
1665
- const draftQuery = opt("draft-model");
1666
- let draftModelDir: string | undefined;
1667
- if (draftQuery !== null) {
1668
- const { m: dm } = await resolveModelAuto(draftQuery);
1669
- draftModelDir = dm.path;
1670
- }
1671
- const numDraftRaw = opt("num-draft-tokens");
1672
- const numDraftTokens = numDraftRaw !== null ? Number(numDraftRaw) : undefined;
1673
- if (numDraftTokens !== undefined && (!Number.isInteger(numDraftTokens) || numDraftTokens < 1)) {
1674
- console.error(`--num-draft-tokens expects an integer >= 1 (got "${numDraftRaw}")`);
1675
- process.exit(1);
1676
- }
1677
- const draftKindRaw = opt("draft-kind");
1678
- const draftKind = (draftKindRaw ?? undefined) as
1679
- | "dspark" | "deepspec" | "assistant" | "two-model" | "ngram" | "mtp" | undefined;
1680
- if (draftKind !== undefined && !["dspark", "deepspec", "assistant", "two-model", "ngram", "mtp"].includes(draftKind)) {
1681
- console.error(`--draft-kind expects two-model|assistant|dspark|deepspec|mtp|ngram (got "${draftKindRaw}")`);
1682
- process.exit(1);
1683
- }
1684
- // Prompt-lookup window bounds (ngram kind only; validated here so a typo
1685
- // fails the launch, not the first request).
1686
- const ngramInt = (name: string): number | undefined => {
1687
- const raw = opt(name);
1688
- if (raw === null) return undefined;
1689
- const v = Number(raw);
1690
- if (!Number.isInteger(v) || v < 1) {
1691
- console.error(`--${name} expects an integer >= 1 (got "${raw}")`);
1692
- process.exit(1);
1693
- }
1694
- return v;
1695
- };
1696
- const ngramMax = ngramInt("ngram-max");
1697
- const ngramMin = ngramInt("ngram-min");
1698
- if (ngramMax !== undefined && ngramMin !== undefined && ngramMin > ngramMax) {
1699
- console.error(`--ngram-min (${ngramMin}) must be <= --ngram-max (${ngramMax})`);
1700
- process.exit(1);
1701
- }
1702
- if ((ngramMax !== undefined || ngramMin !== undefined) && draftKind !== "ngram")
1703
- console.warn("--ngram-max/--ngram-min only apply with --draft-kind ngram — ignored");
1704
- const sLoad = step("loading weights");
1705
- const t0 = performance.now();
1706
- const ctx = await loadContext(m.path, m.repoId, {
1707
- memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
1708
- glm: glm52RuntimeFlags(rt.serverOptions),
1709
- // ngram is model-free; mtp resolves the bundled <model>/mtp/ companion
1710
- // in loadContext when no --draft-model is given (cli.md contract —
1711
- // this gate silently dropped that case until 2026-08-20).
1712
- ...(draftModelDir || draftKind === "ngram" || draftKind === "mtp"
1713
- ? {
1714
- ...(draftModelDir ? { draftModelDir } : {}),
1715
- numDraftTokens,
1716
- ...(draftKind ? { draftKind } : {}),
1717
- ...(ngramMax !== undefined ? { ngramMax } : {}),
1718
- ...(ngramMin !== undefined ? { ngramMin } : {}),
1719
- }
1720
- : {}),
1721
- });
1722
- sLoad.done(`weights loaded ${style.dim(`in ${(performance.now() - t0).toFixed(0)} ms`)}${draftModelDir ? style.dim(` · draft: ${draftModelDir.split("/").filter(Boolean).at(-1)}`) : draftKind === "ngram" ? style.dim(" · draft: ngram (prompt lookup)") : ""}`);
1723
- await mountStartupAdapter(ctx, rt.serverOptions);
1724
- const server = createServer(ctx, rt.port, { ...rt.serverOptions, owner: "serve" });
1725
- let shutdownStarted = false;
1726
- const onServeSignal = (signal: "SIGINT" | "SIGTERM"): void => {
1727
- if (shutdownStarted) return;
1728
- shutdownStarted = true;
1729
- const rawTimeout = Number(runtimeValue("MLX_BUN_SHUTDOWN_TIMEOUT_MS"));
1730
- const timeoutMs = Number.isFinite(rawTimeout) && rawTimeout > 0 ? rawTimeout : 120_000;
1731
- console.log(`\n[server] ${signal}: flushing prompt-cache durability`);
1732
- void shutdownServer(server, timeoutMs).then((result) => {
1733
- if (!result.durability.durable) {
1734
- const d = result.durability;
1735
- console.warn(
1736
- `[server] cache flush incomplete after ${Math.round(d.elapsedMs)} ms: ` +
1737
- `${d.pendingSnapshots} snapshots, ${d.pendingSpills} spills, ` +
1738
- `${d.droppedSpills} dropped, ${d.failedSpills} failed`,
1739
- );
1740
- }
1741
- process.exit(0);
1742
- }).catch((error) => {
1743
- console.warn(`[server] shutdown flush failed: ${(error as Error).message}`);
1744
- process.exit(0);
1745
- });
1746
- };
1747
- process.once("SIGINT", () => onServeSignal("SIGINT"));
1748
- process.once("SIGTERM", () => onServeSignal("SIGTERM"));
1749
- const shownHost = rt.serverOptions.hostname ?? "localhost";
1750
- console.log();
1751
- box([
1752
- `${style.green("●")} ${style.bold("serving")} ${m.repoId}`,
1753
- "",
1754
- `API ${style.url(`http://${shownHost}:${server.port}/v1`)} ${style.dim("(OpenAI · Anthropic · Responses)")}`,
1755
- `App ${style.url(`http://${shownHost}:${server.port}/`)} ${style.dim("(chat · quantize · finetune · dataset · status)")}`,
1756
- ...(rt.serverOptions.memoryBudgetBytes
1757
- ? [`Mem ${gb(rt.serverOptions.memoryBudgetBytes)} budget ${style.dim("(admission control on)")}`] : []),
1758
- `Perf ${style.dim(runtimeSummary(rt.serverOptions))}`,
1759
- "",
1760
- style.dim("agent session: mlx-bun pi stop: Ctrl+C"),
1761
- ]);
1762
- // Open the chat UI for interactive runs (--no-open, or a non-TTY such as
1763
- // a piped/headless/benchmark run, skips it). 0.0.0.0/:: bind → localhost
1764
- // for the browser. Reuses an existing tab on this host:port when possible
1765
- // (see openChatUi); the URL is printed above either way.
1766
- if (!flag("no-open") && process.stdout.isTTY) {
1767
- const browserHost = shownHost === "0.0.0.0" || shownHost === "::" ? "localhost" : shownHost;
1768
- openChatUi(`http://${browserHost}:${server.port}/#/chat`, `${browserHost}:${server.port}`);
1769
- }
1770
- break;
1771
- }
1772
-
1773
- case "gen":
1774
- case "generate": {
1775
- // RAW generate — a direct one-shot entry point (no server, no chat UI). The
1776
- // three entry points (raw / OpenAI API / chat UI) differ only in how params
1777
- // are POPULATED; here every param is explicit. Decode-path levers mirror
1778
- // serve so you can pin the route (mlx-lm compat = --l1).
1779
- // --l1/--l2 + per-fork overrides set the decode runtime options; the
1780
- // returned KV scheme is applied below so `generate --l2` runs the same
1781
- // quantized-KV route (and produces the same tokens) as `serve --l2`.
1782
- const route = applyDecodeRoute();
1783
-
1784
- const prompt = opt("prompt") ?? positional(1);
1785
- if (!prompt) {
1786
- console.error('usage: mlx-bun generate [query] --prompt "…" [--raw] [--max-tokens N]');
1787
- console.error(' sampling: --temperature N (alias --temp) --top-p N --top-k N --seed N');
1788
- console.error(' decode path: --l1 (= mlx-lm compat, the default) / --l2, or per-fork flags');
1789
- process.exit(1);
1790
- }
1791
- const { loadTaskModel, generateText } = await import("./eval/runner");
1792
- const maxTokens = Number(opt("max-tokens", "256"));
1793
- const tm = await loadTaskModel(
1794
- positional(0) ?? opt("query") ?? "",
1795
- undefined,
1796
- { maxGenerationTokens: maxTokens, enableMtp: false },
1797
- );
1798
- // Resolve the tier/--kv-quant KV scheme exactly the way serve does
1799
- // (server.ts kvScheme): "config" = the model's kv_config.json (absent →
1800
- // bf16, same silent fallback as serve), N = uniform bits from decode
1801
- // start. Unset (no tier, no --kv-quant) keeps generate's historical
1802
- // default: bf16 bit-exact greedy (generateText's parity path).
1803
- const kvScheme = route.turboQuant || route.kvQuant !== undefined
1804
- ? resolveKvScheme({
1805
- override: route.kvQuant,
1806
- turboQuant: route.turboQuant,
1807
- config: tm.config.kvQuant,
1808
- }).generationOptions
1809
- : undefined;
1810
- const num = (n: string): number | undefined => { const v = opt(n); return v == null ? undefined : Number(v); };
1811
- const text = await generateText(tm, prompt, {
1812
- maxTokens,
1813
- useChat: !flag("raw"),
1814
- ...(kvScheme !== undefined ? { kvScheme } : {}),
1815
- sampler: { temperature: num("temperature") ?? num("temp"), topP: num("top-p"), topK: num("top-k"), seed: num("seed") },
1816
- });
1817
- process.stdout.write(text.endsWith("\n") ? text : text + "\n");
1818
- break;
1819
- }
1820
-
1821
- case "embed": {
1822
- // Raw text embeddings — one-shot, no server. Text from --text, or one per
1823
- // line on stdin. Uses the same embedPooled path the parity test verifies.
1824
- const { Weights } = await import("./weights");
1825
- const { createModel } = await import("./model/factory");
1826
- const { loadTokenizer } = await import("./tokenizer");
1827
- const { embedMany, isEmbeddingModel } = await import("./embed");
1828
-
1829
- let texts: string[];
1830
- const oneText = opt("text") ?? positional(1);
1831
- if (oneText != null) {
1832
- texts = [oneText];
1833
- } else if (!process.stdin.isTTY) {
1834
- texts = (await Bun.stdin.text()).split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
1835
- } else {
1836
- texts = [];
1837
- }
1838
- if (texts.length === 0) {
1839
- console.error('usage: mlx-bun embed [query] --text "…" (or pipe text, one per line)');
1840
- console.error(' --instruct "…" query instruction · --json OpenAI-style output');
1841
- process.exit(1);
1842
- }
1843
-
1844
- // Model resolution: an explicit query resolves normally; the no-query
1845
- // path auto-picks the first DOWNLOADED embedding model (plain Qwen3 —
1846
- // the record-level equivalent of isEmbeddingModel, which the factory
1847
- // maps to Qwen3Model). Never the chat default, and never the fresh-
1848
- // install starter download — a chat model can't embed, so downloading
1849
- // one here only to fail the isEmbeddingModel check would be a trap.
1850
- const embedQuery = positional(0) ?? opt("query");
1851
- let m: import("./registry").ModelRecord;
1852
- if (embedQuery) {
1853
- ({ m } = await resolveModelAuto(embedQuery));
1854
- } else {
1855
- const reg = new Registry();
1856
- if (reg.list().length === 0) await reg.scan();
1857
- const embedder = reg.list().find((r) => r.modelType === "qwen3");
1858
- if (!embedder) {
1859
- console.error("no embedding model downloaded — try: mlx-bun get mlx-community/Qwen3-Embedding-4B-4bit-DWQ");
1860
- process.exit(1);
1861
- }
1862
- m = embedder;
1863
- }
1864
- const config = await loadModelConfig(m.path);
1865
- const model = createModel(await Weights.open(m.path), config);
1866
- if (!isEmbeddingModel(model)) {
1867
- console.error(`"${m.repoId}" is not an embedding model (need plain Qwen3, e.g. Qwen3-Embedding).`);
1868
- process.exit(1);
1869
- }
1870
- const tok = await loadTokenizer(m.path);
1871
- const results = embedMany(model, tok, texts, opt("instruct") ?? undefined);
1872
-
1873
- if (flag("json")) {
1874
- let total = 0;
1875
- const data = results.map((r, index) => { total += r.tokens; return { object: "embedding", index, embedding: Array.from(r.vector) }; });
1876
- process.stdout.write(JSON.stringify({
1877
- object: "list", data, model: m.repoId,
1878
- usage: { prompt_tokens: total, total_tokens: total },
1879
- }) + "\n");
1880
- } else {
1881
- for (const r of results) process.stdout.write(JSON.stringify(Array.from(r.vector)) + "\n");
1882
- }
1883
- break;
1884
- }
1885
-
1886
- case "bench":
1887
- case "benchmark": {
1888
- const { banner, step, box, style } = await import("./tui");
1889
- banner(pkg.version);
1890
- serverRuntimeFlags(); // decode options (--compiled-decode, --fused-sdpa, ...) apply to the run
1891
- const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
1892
- const tokens = Number(opt("tokens", "256"));
1893
- const runs = Number(opt("runs", "3"));
1894
- const promptTokens = Number(opt("prompt-tokens", "0"));
1895
- const kvMode = opt("kv-quant", "off")!;
1896
- console.log(` ${style.dim("measures THIS machine, OUR stack only — no other systems needed.")}`);
1897
- console.log(` ${style.dim("numbers on a loaded machine are not quotable; bun scripts/bench-serve.ts all is the preflight-gated harness.")}`);
1898
- const sNative = step("native runtime");
1899
- await ensureNative(sNative);
1900
- sNative.done("native runtime ready");
1901
- const sLoad = step(`loading ${m.repoId}${picked ? " (auto-picked)" : ""}`);
1902
- const { Weights } = await import("./weights");
1903
- const { createModel } = await import("./model/factory");
1904
- const { generate } = await import("./generate");
1905
- const { ChatTemplate } = await import("./chat-template");
1906
- const { loadTokenizer } = await import("./tokenizer");
1907
- const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
1908
- const config = await loadModelConfig(m.path);
1909
- const weights = await Weights.open(m.path);
1910
- const model = createModel(weights, config);
1911
- const tok = await loadTokenizer(m.path);
1912
- const template = await ChatTemplate.load(m.path);
1913
- sLoad.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)}`)}`);
1914
-
1915
- let userMsg = "Write a detailed essay about the history of computing, starting with mechanical calculators.";
1916
- if (promptTokens > 0) {
1917
- const filler = "Background context: the history of computation spans mechanical " +
1918
- "calculators, electromechanical relays, vacuum tubes, transistors, " +
1919
- "integrated circuits, and modern accelerators. ";
1920
- while (tok.encode(userMsg).length < promptTokens - 24) userMsg = filler + userMsg;
1921
- }
1922
- const rendered = template.render([{ role: "user", content: userMsg }]);
1923
- const ids = tok.encode(rendered);
1924
- const promptIds = ids[0] === ids[1] && ids[0] === tok.bosTokenId ? ids.slice(1) : ids;
1925
- let turboScheme: TurboQuantScheme | null;
1926
- try {
1927
- turboScheme = parseTurboQuantScheme(kvMode);
1928
- } catch (e) {
1929
- console.error((e as Error).message);
1930
- process.exit(1);
1931
- }
1932
- const kvOptions = turboScheme
1933
- ? { turboQuant: turboScheme, quantizedKvStart: 0 }
1934
- : kvMode === "config"
1935
- ? (() => {
1936
- if (!config.kvQuant?.length) { console.error("model has no kv_config.json (--kv-quant config)"); process.exit(1); }
1937
- return { kvConfig: config.kvQuant };
1938
- })()
1939
- : kvMode !== "off" ? { kvBits: Number(kvMode), quantizedKvStart: 0 } : {};
1940
-
1941
- // Warmup materializes weights so prefill timing measures prefill,
1942
- // not lazy page-in (same protocol as bench.ts (deleted 2026-08-23; git history)).
1943
- const sWarm = step("warmup");
1944
- {
1945
- const wCache = model.makeCache();
1946
- const wGen = generate(model, promptIds.slice(0, Math.min(8, promptIds.length - 1)), {
1947
- maxTokens: 1, temperature: 0, cache: wCache,
1948
- });
1949
- for await (const _ of wGen) { /* discard */ }
1950
- for (const c of wCache) c.dispose();
1951
- }
1952
- resetPeakMemory();
1953
- sWarm.done("warmup complete");
1954
-
1955
- const decodes: number[] = [];
1956
- const prefills: number[] = [];
1957
- let lastStats: { promptTokens: number; generatedTokens: number } = { promptTokens: 0, generatedTokens: 0 };
1958
- for (let i = 0; i < runs; i++) {
1959
- const sRun = step(`run ${i + 1}/${runs} · ${tokens} tokens`);
1960
- const gen = generate(model, promptIds, { maxTokens: tokens, temperature: 0, ...kvOptions });
1961
- for await (const _ of gen) { /* timing only */ }
1962
- const s = gen.stats!;
1963
- decodes.push(s.decodeTps);
1964
- prefills.push(s.prefillTps);
1965
- lastStats = s;
1966
- sRun.done(`run ${i + 1}/${runs} ${style.dim("·")} ${style.green(style.bold(`${s.decodeTps.toFixed(1)} tok/s`))} ${style.dim(`decode · prefill ${s.prefillTps.toFixed(0)} tok/s`)}`);
1967
- }
1968
- const median = (a: number[]) => [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)]!;
1969
- const peak = peakMemory();
1970
-
1971
- const { EvalDB: DB, gitCommit } = await import("./evaldb");
1972
- new DB().record({
1973
- modelPath: m.path,
1974
- commitSha: gitCommit() ?? undefined,
1975
- promptTokens: lastStats.promptTokens, cachedTokens: 0,
1976
- generatedTokens: lastStats.generatedTokens,
1977
- prefillTps: median(prefills), decodeTps: median(decodes),
1978
- peakBytes: peak,
1979
- notes: `cli-bench median-of-${runs} ${tokens}tok kv=${kvMode}${promptTokens ? ` ctx=${lastStats.promptTokens}` : ""} decode[${decodes.map((d) => d.toFixed(1)).join(",")}]`,
1980
- });
1981
- console.log();
1982
- box([
1983
- `${style.green("●")} ${style.bold("benchmark complete")} ${style.dim(`· ${m.repoId}`)}`,
1984
- "",
1985
- `decode ${style.green(style.bold(`${median(decodes).toFixed(1)} tok/s`))} ${style.dim(`median of ${runs} · [${decodes.map((d) => d.toFixed(1)).join(", ")}]`)}`,
1986
- `prefill ${style.bold(`${median(prefills).toFixed(0)} tok/s`)} ${style.dim(`· ${lastStats.promptTokens} prompt tokens`)}`,
1987
- `peak mem ${style.bold(gb(peak))} ${style.dim("(generation only)")}`,
1988
- `levers ${style.dim(runtimeSummary(turboScheme ? { turboQuant: turboScheme } : { kvQuant: kvMode === "off" ? "off" : kvMode === "config" ? undefined : Number(kvMode) }))}`,
1989
- "",
1990
- style.dim("recorded to the eval DB — see mlx-bun evals"),
1991
- ]);
1992
- break;
1993
- }
1994
-
1995
- case "evals": {
1996
- if (flag("clear")) {
1997
- // Eval rows are real data — archive, never destroy.
1998
- const { DEFAULT_EVAL_DB } = await import("./evaldb");
1999
- const { existsSync, renameSync } = await import("node:fs");
2000
- if (!existsSync(DEFAULT_EVAL_DB)) { console.log("no eval DB to clear"); break; }
2001
- const stamp = new Date().toISOString().replace(/[:.]/g, "-");
2002
- const backup = DEFAULT_EVAL_DB.replace(/\.sqlite$/, `-${stamp}.sqlite`);
2003
- renameSync(DEFAULT_EVAL_DB, backup);
2004
- console.log(`eval DB archived to ${backup}`);
2005
- console.log("fresh DB starts with the next mlx-bun benchmark run");
2006
- break;
2007
- }
2008
- const db = new EvalDB();
2009
- const limit = Number(opt("limit", "20"));
2010
- const rows = db.recent(limit);
2011
- if (flag("raw")) {
2012
- for (const r of rows) console.log(JSON.stringify(r));
2013
- break;
2014
- }
2015
- const { style } = await import("./tui");
2016
- if (rows.length === 0) {
2017
- console.log("no eval runs recorded yet — bun scripts/bench-serve.ts all writes them");
2018
- break;
2019
- }
2020
- const ago = (ts: number) => {
2021
- const s = (Date.now() - ts) / 1000;
2022
- if (s < 90) return `${Math.round(s)}s ago`;
2023
- if (s < 5400) return `${Math.round(s / 60)}m ago`;
2024
- if (s < 129600) return `${Math.round(s / 3600)}h ago`;
2025
- return `${Math.round(s / 86400)}d ago`;
2026
- };
2027
- const modelName = (p: string) =>
2028
- /models--[^/]+--([^/]+)/.exec(p)?.[1] ?? p.split("/").at(-1) ?? p;
2029
- const note = (r: Record<string, unknown>, key: string) =>
2030
- new RegExp(`${key}=([\\w.]+)`).exec((r.notes as string) ?? "")?.[1] ?? "";
2031
- const BENCH_LABELS: Record<string, string> = {
2032
- "h2h-server": "server e2e",
2033
- "h2h-direct": "direct decode",
2034
- "bench.ts": "decode script",
2035
- "cli-bench": "cli benchmark",
2036
- };
2037
- const table = rows.map((r) => ({
2038
- when: ago(r.ts as number),
2039
- model: modelName(r.model_path as string).replace(/-OptiQ-4bit$/, "").replace(/^gemma-4-/, "g4-"),
2040
- bench: BENCH_LABELS[((r.notes as string) ?? "").split(" ")[0] ?? ""] ?? ((r.notes as string) ?? "").split(" ")[0] ?? "",
2041
- kv: note(r, "kv"),
2042
- stack: (r.stack as string) ?? "mlx-bun",
2043
- prefill: (r.prefill_tps as number) > 0 ? `${(r.prefill_tps as number).toFixed(0)}` : "",
2044
- decode: (r.decode_tps as number).toFixed(1),
2045
- ttft: note(r, "ttft_ms") ? `${note(r, "ttft_ms")}ms` : "",
2046
- peak: gb(r.peak_bytes as number),
2047
- commit: (r.commit_sha as string) ?? "",
2048
- }));
2049
- type Row = (typeof table)[number];
2050
- const cols: Array<[string, keyof Row, "left" | "right"]> = [
2051
- ["WHEN", "when", "right"], ["STACK", "stack", "left"], ["MODEL", "model", "left"],
2052
- ["BENCH", "bench", "left"], ["KV", "kv", "left"],
2053
- ["PREFILL", "prefill", "right"], ["DECODE", "decode", "right"], ["TTFT", "ttft", "right"],
2054
- ["PEAK", "peak", "right"], ["COMMIT", "commit", "left"],
2055
- ];
2056
- const widths = cols.map(([h, k]) => Math.max(h.length, ...table.map((t) => t[k].length)));
2057
- console.log();
2058
- console.log(" " + cols.map(([h], i) => style.dim(h.padEnd(widths[i]!))).join(" "));
2059
- for (const t of table) {
2060
- console.log(" " + cols.map(([, k, align], i) => {
2061
- const cell = align === "right" ? t[k].padStart(widths[i]!) : t[k].padEnd(widths[i]!);
2062
- if (k === "decode") return style.green(style.bold(cell));
2063
- if (k === "model") return style.bold(cell);
2064
- if (k === "stack") return cell.trimEnd() === "mlx-bun" ? style.accent(cell) : style.dim(cell);
2065
- if (k === "commit" || k === "when") return style.dim(cell);
2066
- return cell;
2067
- }).join(" "));
2068
- }
2069
- console.log();
2070
- console.log(style.dim(" STACK engine that produced the number (mlx-bun = us; mlx-lm/optiq = Python references)"));
2071
- console.log(style.dim(" BENCH server e2e = full HTTP round-trip · direct decode = engine only, no server"));
2072
- console.log(style.dim(" decode script / cli benchmark = single-stack decode measurement"));
2073
- console.log(style.dim(" KV KV-cache quantization (off = bf16, config = per-layer kv_config.json)"));
2074
- console.log(style.dim(" PREFILL prompt ingestion tok/s (in-process benches) · DECODE generation tok/s"));
2075
- console.log(style.dim(" TTFT request → first token, incl. server overhead (server benches)"));
2076
- console.log(style.dim(" PEAK max memory during generation"));
2077
- console.log();
2078
- console.log(style.dim(` ${rows.length} run(s) · --limit <n> for more · --raw for full records · mlx-bun benchmark to add a run`));
2079
- break;
2080
- }
2081
-
2082
- case "pi": {
2083
- // mlx-bun consumes its own flags; the rest describe how the built-in
2084
- // agent runs. `mlx-bun pi` always drives pi's own TUI in-process
2085
- // (src/pi-terminal.ts) — pi is bundled, nothing to install. Users who
2086
- // already run their own pi connect it with `mlx-bun harness pi`.
2087
- const OURS_VAL = new Set([
2088
- "--query", "-q", "--model", "--port", "--host", "--memory-budget", "--kv-budget", "--prompt-cache",
2089
- "--ssd-cache", "--ssd-cache-max", "--ssd-demote-idle", "--generation-checkpoint", "--kv-quant", "--unix",
2090
- "--batch", "--decode-concurrency", "--adapter", "--adapter-path",
2091
- "--draft-model", "--num-draft-tokens", "--draft-kind", "--ngram-max", "--ngram-min",
2092
- "--compiled-decode", "--compiled-activations", "--fused-sdpa", "--thinking",
2093
- "--temperature", "--temp", "--top-p", "--top-k", "--max-tokens",
2094
- "--mtp", "--context-length",
2095
- "--hlg-sampling", "--hlg-width", "--hlg-shoulder", "--hlg-toe", "--hlg-pivot-offset",
2096
- "--paged-kv-block-size",
2097
- ]);
2098
- const OURS_BOOL = new Set(["--force-wire", "--expert-offload", "--no-open", "--ssd-cache-verify", "--l1", "--l2", "--l3", "--isolate", "--paged-kv"]);
2099
- const passthrough: string[] = [];
2100
- for (let i = 1; i < argv.length; i++) {
2101
- const a = argv[i]!;
2102
- if (OURS_VAL.has(a)) { i++; continue; }
2103
- if (OURS_BOOL.has(a)) continue;
2104
- passthrough.push(a);
2105
- }
2106
-
2107
- const { parsePiArgs } = await import("./pi-terminal");
2108
- const stdinPiped = !process.stdin.isTTY;
2109
- const parsed = parsePiArgs(passthrough, stdinPiped);
2110
-
2111
- // Headless machine-readable runs (`-p`, `--mode json`, `--mode rpc`) must
2112
- // keep stdout pristine for the response/JSONL stream — route our bring-up
2113
- // logging to stderr. Interactive keeps the pretty banner on stdout.
2114
- const quiet = parsed.mode !== "interactive";
2115
- const origStdoutWrite = process.stdout.write.bind(process.stdout);
2116
- if (quiet) {
2117
- (process.stdout as unknown as { write: typeof process.stderr.write }).write =
2118
- process.stderr.write.bind(process.stderr);
2119
- }
2120
- const restoreStdout = () => {
2121
- if (quiet) (process.stdout as unknown as { write: typeof origStdoutWrite }).write = origStdoutWrite;
2122
- };
2123
-
2124
- const rt = serverRuntimeFlags();
2125
- const port = rt.port;
2126
- const baseUrl = `http://127.0.0.1:${port}/v1`;
2127
- const { probeServer } = await import("./harness-pi");
2128
- let models = await probeServer(baseUrl);
2129
- let startedServer = false;
2130
- if (models) {
2131
- console.log(`reusing running server at ${baseUrl} (${models.map((m) => m.id).join(", ")})`);
2132
- // Server-shaping flags can't apply to a server we didn't start —
2133
- // drop them loudly rather than silently or fatally.
2134
- const ignored = [...OURS_VAL, ...OURS_BOOL].filter((f) => f !== "--port" && argv.includes(f));
2135
- if (ignored.length > 0) {
2136
- const { style } = await import("./tui");
2137
- console.log(style.dim(` ignoring ${ignored.join(", ")} — the running server keeps its own configuration`));
2138
- console.log(style.dim(" (stop it or use --port <other> to start a fresh server with these flags)"));
2139
- }
2140
- try {
2141
- const stats = await (await fetch(`http://127.0.0.1:${port}/stats`)).json() as
2142
- { server?: { owner?: string } };
2143
- if (stats.server?.owner === "pi-session") {
2144
- console.log("note: that server belongs to another `mlx-bun pi` session and ends when it does —");
2145
- console.log(" run `mlx-bun serve` for a persistent server.");
2146
- }
2147
- } catch {}
2148
- } else {
2149
- const { banner, step, style } = await import("./tui");
2150
- banner(pkg.version);
2151
- const { m, picked } = await resolveModelAuto(opt("query") ?? optShort("q"));
2152
- if (picked)
2153
- console.log(` ${style.dim(`auto-picked ${m.repoId} (largest supported model that fits; override with --query)`)}`);
2154
- const sNative = step("native runtime");
2155
- await ensureNative(sNative);
2156
- sNative.done("native runtime ready");
2157
- const { createServer, loadContext } = await import("./server");
2158
- const sLoad = step(`loading ${m.repoId}`);
2159
- const t0 = performance.now();
2160
- const ctx = await loadContext(m.path, m.repoId, {
2161
- memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
2162
- glm: glm52RuntimeFlags(rt.serverOptions),
2163
- });
2164
- await mountStartupAdapter(ctx, rt.serverOptions);
2165
- createServer(ctx, port, { ...rt.serverOptions, owner: "pi-session" });
2166
- startedServer = true;
2167
- sLoad.done(`serving ${style.bold(m.repoId)} ${style.dim(`at ${baseUrl} · ready in ${(performance.now() - t0).toFixed(0)} ms`)}`);
2168
- models = await probeServer(baseUrl);
2169
- if (!models) { restoreStdout(); console.error("server started but /v1/models probe failed"); process.exit(1); }
2170
- }
2171
- const exitNote = () => {
2172
- // The server we started lives exactly as long as the pi session — any
2173
- // exit tears it down. A standalone server belongs to `mlx-bun serve`
2174
- // (which we reuse and never stop).
2175
- if (startedServer && parsed.mode === "interactive")
2176
- console.log("pi exited — shutting down (use `mlx-bun serve` for a persistent server)");
2177
- };
2178
-
2179
- // pi's own TUI embedded in-process — nothing to install.
2180
- const { runEmbeddedPi } = await import("./pi-terminal");
2181
- let message = parsed.message;
2182
- if (parsed.mode === "print" && stdinPiped) {
2183
- const piped = (await Bun.stdin.text()).trim();
2184
- if (piped) message = message ? `${message}\n\n${piped}` : piped;
2185
- }
2186
- if (parsed.mode === "interactive")
2187
- console.log(`launching mlx-bun pi (model ${models[0]!.id}) — /help for commands, double-Ctrl+C to exit`);
2188
- restoreStdout();
2189
- if (parsed.ignored.length) {
2190
- const { style } = await import("./tui");
2191
- console.log(style.dim(` ignoring pi flags: ${parsed.ignored.join(", ")}`));
2192
- }
2193
- const code = await runEmbeddedPi({
2194
- baseUrl,
2195
- modelLabel: models[0]!.id,
2196
- contextWindow: models[0]!.contextWindow,
2197
- reasoning: models[0]!.reasoning,
2198
- vision: models[0]!.vision,
2199
- verbose: parsed.verbose,
2200
- mode: parsed.mode,
2201
- printFormat: parsed.printFormat,
2202
- initialMessage: message,
2203
- });
2204
- exitNote();
2205
- process.exit(code);
2206
- }
2207
-
2208
- case "harness": {
2209
- const target = positional(0);
2210
- if (target !== "pi") {
2211
- console.error("usage: mlx-bun harness pi [--base-url <url>] [--remove]");
2212
- process.exit(1);
2213
- }
2214
- const { detectPi, installPiExtension, removePiExtension, DEFAULT_BASE_URL } =
2215
- await import("./harness-pi");
2216
- if (flag("remove")) {
2217
- console.log(removePiExtension() ? "mlx-bun provider extension removed" : "nothing to remove");
2218
- break;
2219
- }
2220
- const pi = detectPi();
2221
- if (!pi.found) {
2222
- console.error("pi not found on PATH. Install it first:");
2223
- console.error(" curl -fsSL https://pi.dev/install.sh | sh");
2224
- console.error("then re-run: mlx-bun harness pi");
2225
- process.exit(1);
2226
- }
2227
- const { step, box, style } = await import("./tui");
2228
- const sPi = step("detecting pi");
2229
- sPi.done(`pi ${style.dim(`${pi.binPath}${pi.version ? ` · v${pi.version}` : ""}`)}`);
2230
- const baseUrl = opt("base-url", DEFAULT_BASE_URL)!;
2231
- const sExt = step("installing provider extension");
2232
- const result = await installPiExtension(baseUrl);
2233
- sExt.done(`provider extension installed ${style.dim(`· ${result.path}`)}`);
2234
- const sProbe = step("probing server");
2235
- if (result.serverReachable) sProbe.done(`server live ${style.dim(`· ${result.bakedModels.join(", ")}`)}`);
2236
- else sProbe.done(`server not running ${style.dim(`· models discovered when pi starts against ${baseUrl}`)}`);
2237
- console.log();
2238
- box([
2239
- `${style.green("●")} ${style.bold("your pi is connected to mlx-bun")}`,
2240
- "",
2241
- `launch ${style.accent("pi --provider mlx-bun")}`,
2242
- `select ${style.dim("/model inside pi · or scope cycling:")} ${style.accent('pi --models "mlx-bun/*"')}`,
2243
- `undo ${style.dim("mlx-bun harness pi --remove")}`,
2244
- ]);
2245
- break;
2246
- }
2247
-
2248
- case "train": {
2249
- const { banner, step, box, style } = await import("./tui");
2250
- const query = positional(0) ?? opt("query");
2251
- const dataDir = opt("data");
2252
- if (!dataDir) {
2253
- console.error("usage: mlx-bun train <model> --data <dir> (see: mlx-bun help train)");
2254
- process.exit(1);
2255
- }
2256
- if (!(await Bun.file(`${dataDir}/train.jsonl`).exists())) {
2257
- console.error(`no train.jsonl in ${dataDir}`);
2258
- process.exit(1);
2259
- }
2260
- const method = opt("method", "orpo")!;
2261
- if (method !== "sft" && method !== "dpo" && method !== "orpo") {
2262
- console.error(`--method must be sft | dpo | orpo (got "${method}")`);
2263
- process.exit(1);
2264
- }
2265
- const isOrpo = method === "orpo";
2266
- // ORPO chosen-NLL scope (trainer sftScope; job.ts sft_scope). "full" =
2267
- // paper/TRL-faithful prompt+response CE (the default); "response" =
2268
- // response-only, bit-exact to pre-2026-07 runs. Validated here so junk
2269
- // fails fast, before any model resolution or weight loading.
2270
- const sftScope = opt("sft-scope");
2271
- if (sftScope !== null && sftScope !== "full" && sftScope !== "response") {
2272
- console.error(`--sft-scope must be full | response (got "${sftScope}")`);
2273
- process.exit(1);
2274
- }
2275
- // Validated numeric flag: default if absent, hard-exit on a non-number.
2276
- const numFlag = (name: string, dflt: number): number => {
2277
- const v = opt(name);
2278
- if (v == null) return dflt;
2279
- const n = Number(v);
2280
- if (!Number.isFinite(n)) { console.error(`--${name} expects a number (got "${v}")`); process.exit(1); }
2281
- return n;
2282
- };
2283
-
2284
- banner(pkg.version);
2285
- const sNative = step("native runtime");
2286
- await ensureNative(sNative);
2287
- sNative.done("native runtime ready");
2288
-
2289
- const { m, picked } = await resolveModelAuto(query);
2290
-
2291
- const isGemma = (await Bun.file(`${m.path}/config.json`).text()).toLowerCase().includes("gemma");
2292
-
2293
- const modelTag = isGemma ? "e4b" : "cpm5";
2294
- const adapter = opt("adapter") ?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/${method}-${modelTag}`;
2295
- const iters = numFlag("iters", 100);
2296
- const seq = numFlag("seq", isGemma ? 8192 : 4096);
2297
- const seg = flag("no-segment") ? 0 : numFlag("seg", isOrpo ? 2 : 0);
2298
- const saveEvery = numFlag("save-every", 0);
2299
- const flashOn = !flag("no-flash");
2300
- const prefixOn = !flag("no-prefix");
2301
- const resume = opt("resume") ?? "";
2302
-
2303
- // Build the snake_case submit record — the exact shape the server hands to
2304
- // the finetune job runner (src/train/job.ts parseConfig).
2305
- const cfg: Record<string, unknown> = {
2306
- model_dir: m.path,
2307
- data_dir: dataDir,
2308
- adapter_path: adapter,
2309
- method,
2310
- rank: numFlag("rank", isOrpo ? 16 : 8),
2311
- scale: numFlag("scale", isOrpo ? 2.0 : 1.0),
2312
- rank_scaling: "by_bits",
2313
- num_layers: -1,
2314
- iters,
2315
- learning_rate: numFlag("lr", isOrpo ? 1e-5 : method === "dpo" ? 5e-5 : 2e-4),
2316
- max_seq_length: seq,
2317
- batch_size: numFlag("batch", 1),
2318
- grad_accumulation_steps: numFlag("grad-accum", 1),
2319
- seed: numFlag("seed", 0),
2320
- steps_per_report: 1,
2321
- steps_per_eval: saveEvery > 0 ? saveEvery : 1_000_000,
2322
- save_checkpoints: saveEvery > 0,
2323
- segment_size: seg,
2324
- grad_clip_norm: numFlag("grad-clip", 1.0),
2325
- val_max_examples: numFlag("val-size", 256),
2326
- warm_start_adapter: resume,
2327
- ...(sftScope ? { sft_scope: sftScope } : {}),
2328
- ...(isOrpo ? {
2329
- orpo_lambda: numFlag("lambda", 0.1),
2330
- orpo_lr_schedule: "cosine",
2331
- orpo_warmup_iters: Math.min(10, Math.floor(iters / 10)),
2332
- orpo_chunk_size: 512,
2333
- orpo_flash_ce: flashOn,
2334
- orpo_fused_ce: !flashOn,
2335
- orpo_prefix_shared: prefixOn,
2336
- } : {}),
2337
- };
2338
-
2339
- // Pre-flight: dataset counts + detected format (bail before loading the model).
2340
- const { inspectDataset } = await import("./train/job");
2341
- const ds = await inspectDataset(dataDir);
2342
- if (!ds.ok) { console.error(`dataset: ${ds.error}`); process.exit(1); }
2343
-
2344
- const planLines = [
2345
- `${style.green("●")} ${style.bold(`train ${method}`)} ${style.dim(`· ${m.repoId}${picked ? " (auto-picked)" : ""}${isGemma ? " · e4b defaults" : ""}`)}`,
2346
- "",
2347
- `data ${style.bold(`${ds.n_train} train`)}${ds.n_valid ? ` · ${ds.n_valid} valid` : ""} ${style.dim(`· format ${ds.format}`)}`,
2348
- `loop ${style.dim(`iters ${iters} · lr ${cfg.learning_rate} · rank ${cfg.rank} · scale ${cfg.scale} · seq ${seq} · batch ${cfg.batch_size}`)}`,
2349
- ];
2350
- if (isOrpo)
2351
- planLines.push(
2352
- `head ${style.dim(flashOn ? "flash-CCE Metal ([M,vocab]-free)" : "MLX fused linear-CE")}`,
2353
- `stack ${style.dim(`prefix-share ${prefixOn ? "on" : "off"} · segmented ${seg > 0 ? `${seg}/seg` : "off"} · λ ${cfg.orpo_lambda}`)}`,
2354
- `stability ${style.dim(`grad-clip ${cfg.grad_clip_norm || "off"} · val-size ${cfg.val_max_examples}${(cfg.grad_accumulation_steps as number) > 1 ? ` · grad-accum ${cfg.grad_accumulation_steps} (eff batch ${(cfg.batch_size as number) * (cfg.grad_accumulation_steps as number)})` : ""}`)}`,
2355
- );
2356
- else planLines.push(`stack ${style.dim(`segmented ${seg > 0 ? `${seg}/seg` : "off"}`)}`);
2357
- if (resume) planLines.push(`warm-start ${style.dim(`from ${resume} (weights only)`)}`);
2358
- if (saveEvery > 0) planLines.push(`checkpoint ${style.dim(`every ${saveEvery} steps`)}`);
2359
- planLines.push("", `adapter ${style.dim(adapter)}`);
2360
- console.log();
2361
- box(planLines);
2362
- console.log();
2363
- console.log(` ${style.dim("watch live (other tab):")} ${style.accent(`mlx-bun train-watch ${adapter}`)}`);
2364
- console.log();
2365
-
2366
- if (flag("dry-run")) { console.log(` ${style.dim("dry run — not training.")}`); break; }
2367
-
2368
- // Run the finetune job runner IN-PROCESS (foreground), streaming metrics to
2369
- // the terminal — the same runner the server drives via submitSubprocess.
2370
- // Run long jobs detached from your own shell: nohup mlx-bun train … &
2371
- const { finetuneRunner } = await import("./train/job");
2372
- const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
2373
- const losses: number[] = [];
2374
- const stepMs: number[] = [];
2375
- let lastStepT = Date.now();
2376
- resetPeakMemory();
2377
- const emit = (e: JobEvent) => {
2378
- if (e.type === "stage" && e.message) console.log(` ${style.dim("·")} ${e.message}`);
2379
- else if (e.type === "metric" && e.kind === "train") {
2380
- const now = Date.now(); stepMs.push(now - lastStepT); lastStepT = now;
2381
- losses.push(e.loss);
2382
- const n = losses.length;
2383
- if (n <= 3 || n % 10 === 0)
2384
- console.log(` step ${n}/${iters}: loss ${style.bold(e.loss.toFixed(4))} ${style.dim(`(${(stepMs[stepMs.length - 1]! / 1000).toFixed(1)}s/step · peak ${gb(peakMemory())})`)}`);
2385
- }
2386
- };
2387
- try {
2388
- await finetuneRunner(emit, cfg);
2389
- } catch (err) {
2390
- console.error(`\n training failed: ${err instanceof Error ? err.message : String(err)}`);
2391
- process.exit(1);
2392
- }
2393
- const finite = losses.every((l) => Number.isFinite(l));
2394
- const sorted = stepMs.slice(1).sort((a, b) => a - b);
2395
- const med = sorted[Math.floor(sorted.length / 2)] ?? stepMs[0] ?? 0;
2396
- console.log();
2397
- box([
2398
- `${style.green("●")} ${style.bold("training complete")} ${style.dim(`· ${losses.length} steps`)}`,
2399
- "",
2400
- `loss ${style.bold(`${losses[0]?.toFixed(4) ?? "—"} → ${losses[losses.length - 1]?.toFixed(4) ?? "—"}`)}${finite ? "" : " (NON-FINITE!)"}`,
2401
- `speed ${style.dim(`${(med / 1000).toFixed(1)}s/step median · peak ${gb(peakMemory())}`)}`,
2402
- "",
2403
- `adapter ${style.bold(adapter)}`,
2404
- `serve it ${style.accent(`mlx-bun serve ${m.repoId} --adapter ${adapter}`)}`,
2405
- ]);
2406
- break;
2407
- }
2408
-
2409
- case "train-watch": {
2410
- // Live mactop-style dashboard for a training run: tails <adapter>/metrics.jsonl.
2411
- const dir = positional(0) ?? opt("adapter")
2412
- ?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5`;
2413
- const { runWatch } = await import("./train/watch");
2414
- await runWatch(dir);
2415
- break;
2416
- }
2417
-
2418
- case "fuse": {
2419
- // mlx_lm.fuse counterpart over our native machinery (src/train/fuse.ts):
2420
- // fold a LoRA adapter into the base weights → standalone snapshot.
2421
- const unsupported = ["--de-quantize", "--dequantize", "--export-gguf", "--gguf-path", "--upload-repo"]
2422
- .filter((f) => argv.includes(f));
2423
- if (unsupported.length > 0) {
2424
- console.error(`${unsupported.join(", ")}: not supported (see: mlx-bun help fuse)`);
2425
- process.exit(1);
2426
- }
2427
- const modelArg = positional(0) ?? opt("model");
2428
- if (!modelArg) {
2429
- console.error("usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [--save-path <dir>]");
2430
- process.exit(1);
2431
- }
2432
- const adapterDir = opt("adapter") ?? opt("adapter-path") ?? "adapters";
2433
- const savePath = opt("save-path") ?? "fused_model";
2434
- const { existsSync } = await import("node:fs");
2435
- if (!existsSync(adapterDir)) {
2436
- console.error(`adapter dir not found: ${adapterDir}`);
2437
- process.exit(1);
2438
- }
2439
- let modelDir = modelArg;
2440
- if (!existsSync(`${modelArg}/config.json`)) {
2441
- const reg = new Registry();
2442
- if (reg.list().length === 0) await reg.scan();
2443
- modelDir = reg.resolve(modelArg).path;
2444
- }
2445
- const { banner, step, box, style } = await import("./tui");
2446
- banner(pkg.version);
2447
- const sNative = step("native runtime");
2448
- await ensureNative(sNative);
2449
- sNative.done("native runtime ready");
2450
- const s = step(`fusing ${adapterDir} into ${modelDir}`);
2451
- const { fuseAdapter } = await import("./train");
2452
- try {
2453
- const stats = await fuseAdapter(modelDir, adapterDir, savePath, (e) => s.update(e.message));
2454
- s.done(`fused ${stats.fusedModules} module(s) ${style.dim(`· ${stats.totalTensors} tensors written`)}`);
2455
- console.log();
2456
- box([
2457
- `${style.green("●")} ${style.bold("fuse complete")}`,
2458
- "",
2459
- `base ${style.dim(modelDir)}`,
2460
- `adapter ${style.dim(adapterDir)}`,
2461
- `model ${style.bold(stats.outDir)}`,
2462
- ...(stats.skippedAdapterTensors > 0
2463
- ? [`skipped ${style.dim(`${stats.skippedAdapterTensors} adapter tensor(s) with no matching base weight`)}`] : []),
2464
- "",
2465
- `serve it ${style.accent(`mlx-bun serve ${stats.outDir}`)}`,
2466
- ]);
2467
- } catch (e) {
2468
- s.fail(`fuse failed: ${e instanceof Error ? e.message : String(e)}`);
2469
- process.exit(1);
2470
- }
2471
- break;
2472
- }
2473
-
2474
- case "convert": {
2475
- // mlx_lm.convert counterpart over our native quantize pipeline (the same
2476
- // engine as the web /api/quantize): uniform affine 4/8-bit, or the OptiQ
2477
- // mixed-precision path via --target-bpw.
2478
- // --upload-repo (mlx_lm.convert parity): push the converted model to the
2479
- // Hub afterwards, via the same engine as `mlx-bun upload`. The token is
2480
- // resolved BEFORE any conversion work — failing after minutes of
2481
- // quantizing because auth is missing would be cruel.
2482
- const uploadRepo = ((): string | null => {
2483
- if (!argv.includes("--upload-repo")) return null;
2484
- const v = opt("upload-repo");
2485
- if (!v || v.startsWith("-")) {
2486
- console.error("--upload-repo expects a repo id (org/name)");
2487
- process.exit(1);
2488
- }
2489
- return v;
2490
- })();
2491
- let uploadToken: string | null = null;
2492
- if (uploadRepo) {
2493
- const { getHfToken } = await import("./hf-push");
2494
- uploadToken = getHfToken();
2495
- if (!uploadToken) {
2496
- console.error("--upload-repo needs a Hugging Face WRITE token and none was found —");
2497
- console.error("run `hf auth login`, export HF_TOKEN, or save one in the web UI (Settings → Hugging Face).");
2498
- process.exit(1);
2499
- }
2500
- }
2501
- const unsupported = ["--dtype", "-d", "--dequantize", "--quant-predicate"]
2502
- .filter((f) => argv.includes(f));
2503
- if (unsupported.length > 0) {
2504
- console.error(`${unsupported.join(", ")}: not supported (mixed precision: --target-bpw; see: mlx-bun help convert)`);
2505
- process.exit(1);
2506
- }
2507
- const qMode = opt("q-mode", "affine")!;
2508
- if (qMode !== "affine") {
2509
- console.error(`--q-mode ${qMode}: only "affine" is supported`);
2510
- process.exit(1);
2511
- }
2512
- const hfPath = opt("hf-path") ?? opt("model") ?? positional(0);
2513
- if (!hfPath) {
2514
- console.error("usage: mlx-bun convert --hf-path <repo-or-path> -q [--q-bits N] [--q-group-size N] [--mlx-path <dir>] [--target-bpw F]");
2515
- process.exit(1);
2516
- }
2517
- const targetBpwRaw = opt("target-bpw");
2518
- if (!argv.includes("-q") && !argv.includes("--quantize") && targetBpwRaw === null) {
2519
- console.error("plain (non-quantizing) conversion is not supported yet — pass -q or --target-bpw");
2520
- process.exit(1);
2521
- }
2522
- const targetBpw = targetBpwRaw !== null ? Number(targetBpwRaw) : undefined;
2523
- if (targetBpwRaw !== null && (!Number.isFinite(targetBpw!) || targetBpw! <= 0)) {
2524
- console.error(`--target-bpw expects a positive number (got "${targetBpwRaw}")`);
2525
- process.exit(1);
2526
- }
2527
- const qBits = Number(opt("q-bits", "4"));
2528
- if (qBits !== 4 && qBits !== 8) {
2529
- console.error(`--q-bits must be 4 or 8 (got "${opt("q-bits")}")`);
2530
- process.exit(1);
2531
- }
2532
- const qGroup = Number(opt("q-group-size", "64"));
2533
- if (qGroup !== 32 && qGroup !== 64) {
2534
- console.error(`--q-group-size must be 32 or 64 (got "${opt("q-group-size")}")`);
2535
- process.exit(1);
2536
- }
2537
- const candidateBits = opt("candidate-bits")?.split(",").map((s) => Number(s.trim()));
2538
- if (candidateBits && candidateBits.some((b) => !Number.isInteger(b) || b < 2 || b > 8)) {
2539
- console.error(`--candidate-bits expects a comma list of integers in [2, 8] (got "${opt("candidate-bits")}")`);
2540
- process.exit(1);
2541
- }
2542
- const rotateWeights = flag("rotate-weights");
2543
- const rotationSeed = Number(opt("rotation-seed", "42"));
2544
- if (!Number.isInteger(rotationSeed)) {
2545
- console.error(`--rotation-seed expects an integer (got "${opt("rotation-seed")}")`);
2546
- process.exit(1);
2547
- }
2548
- const mlxPath = opt("mlx-path", "mlx_model")!;
2549
- const { existsSync } = await import("node:fs");
2550
- if (existsSync(mlxPath)) {
2551
- console.error(`Cannot save to the path ${mlxPath} as it already exists — delete it or pass a fresh --mlx-path.`);
2552
- process.exit(1);
2553
- }
2554
- const { banner, step, box, style } = await import("./tui");
2555
- banner(pkg.version);
2556
- const sNative = step("native runtime");
2557
- await ensureNative(sNative);
2558
- sNative.done("native runtime ready");
2559
- // Resolve the source: local dir → itself; downloaded model → registry;
2560
- // otherwise an HF repo id, downloaded first (the point of the verb).
2561
- let srcDir = hfPath;
2562
- if (!existsSync(`${hfPath}/config.json`)) {
2563
- const reg = new Registry();
2564
- if (reg.list().length === 0) await reg.scan();
2565
- try {
2566
- srcDir = reg.resolve(hfPath).path;
2567
- } catch (e) {
2568
- if (!/^[\w.-]+\/[\w.-]+$/.test(hfPath)) {
2569
- console.error(e instanceof Error ? e.message : String(e));
2570
- process.exit(1);
2571
- }
2572
- const { downloadModel } = await import("./download");
2573
- const sDl = step(`downloading ${hfPath}`);
2574
- srcDir = await downloadModel(hfPath, {
2575
- onProgress: (file, received, total) => {
2576
- const pct = total ? Math.floor((received / total) * 100) : 0;
2577
- sDl.update(`${style.bold(hfPath)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
2578
- },
2579
- });
2580
- sDl.done(`${style.bold(hfPath)} ${style.dim("downloaded · verified")}`);
2581
- await reg.scan();
2582
- }
2583
- }
2584
- const sQ = step(
2585
- targetBpw !== undefined
2586
- ? `quantizing (mixed, target ${targetBpw} bpw — sensitivity sweep, ~minutes)`
2587
- : `quantizing (${qBits}-bit, group ${qGroup})`,
2588
- );
2589
- const { quantizeModelDir, automaticRotationWeightTransform } = await import("./quantize");
2590
- try {
2591
- const r = await quantizeModelDir(
2592
- srcDir,
2593
- mlxPath,
2594
- {
2595
- bits: qBits as 4 | 8,
2596
- groupSize: qGroup as 32 | 64,
2597
- mode: "affine",
2598
- ...(targetBpw !== undefined ? { targetBpw } : {}),
2599
- ...(candidateBits ? { candidateBits } : {}),
2600
- ...(opt("calibration-mix") ? { calibrationMix: opt("calibration-mix")! } : {}),
2601
- ...(opt("n-calibration") ? { nCalibration: Number(opt("n-calibration")) } : {}),
2602
- ...(rotateWeights
2603
- ? { weightTransform: automaticRotationWeightTransform({ seed: rotationSeed }) }
2604
- : {}),
2605
- },
2606
- (e) => sQ.update(e.message),
2607
- );
2608
- sQ.done(`quantized ${r.nQuantized} module(s) ${style.dim(`· ${r.achievedBpw.toFixed(2)} bpw achieved`)}`);
2609
- console.log();
2610
- box([
2611
- `${style.green("●")} ${style.bold("convert complete")}`,
2612
- "",
2613
- `source ${style.dim(srcDir)}`,
2614
- `model ${style.bold(r.outDir)} ${style.dim(`· ${gb(r.write.totalSize)}`)}`,
2615
- `quant ${style.dim(targetBpw !== undefined ? `mixed ${r.achievedBpw.toFixed(2)} bpw (target ${targetBpw})` : `${qBits}-bit g${qGroup} affine`)}`,
2616
- ...(rotateWeights ? [`transform ${style.dim(`TurboQuant rotation seed ${rotationSeed}`)}`] : []),
2617
- "",
2618
- `serve it ${style.accent(`mlx-bun serve ${r.outDir}`)}`,
2619
- ]);
2620
- if (uploadRepo) {
2621
- const sUp = step(`uploading ${r.outDir} → ${uploadRepo}`);
2622
- try {
2623
- const { uploadFolder } = await import("./hf-push");
2624
- const up = await uploadFolder(r.outDir, uploadRepo, {
2625
- repoType: "model",
2626
- token: uploadToken,
2627
- commitMessage: "Upload with mlx-bun convert",
2628
- onProgress: (file, sent, total) =>
2629
- sUp.update(`${style.bold(uploadRepo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
2630
- });
2631
- sUp.done(`uploaded ${style.bold(up.url)}`);
2632
- } catch (e) {
2633
- sUp.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
2634
- console.error(`the converted model is intact at ${r.outDir} — retry with: mlx-bun upload --path ${r.outDir} --upload-repo ${uploadRepo}`);
2635
- process.exit(1);
2636
- }
2637
- }
2638
- } catch (e) {
2639
- sQ.fail(`convert failed: ${e instanceof Error ? e.message : String(e)}`);
2640
- process.exit(1);
2641
- }
2642
- break;
2643
- }
2644
-
2645
- case "upload": {
2646
- // mlx_lm.upload counterpart (same flag names: --path, --upload-repo) over
2647
- // our native push-to-hub (src/hf-push.ts — the exact engine the web
2648
- // /api/{quantize,finetune,dataset}/push routes call). Model-free until
2649
- // the network: everything before uploadFolder is arg + token validation.
2650
- const dir = opt("path", "mlx_model")!; // mlx_lm.upload's --path default
2651
- const repo = ((): string | null => {
2652
- const v = opt("upload-repo");
2653
- return v && !v.startsWith("-") ? v : null;
2654
- })();
2655
- if (!repo) {
2656
- console.error("usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [--private]");
2657
- process.exit(1);
2658
- }
2659
- const { existsSync, statSync } = await import("node:fs");
2660
- if (!existsSync(dir) || !statSync(dir).isDirectory()) {
2661
- console.error(`not a directory: ${dir} (pass the model directory via --path)`);
2662
- process.exit(1);
2663
- }
2664
- const { getHfToken, uploadFolder } = await import("./hf-push");
2665
- const token = getHfToken();
2666
- if (!token) {
2667
- console.error("no Hugging Face token found — uploading needs a WRITE token from one of:");
2668
- console.error(" hf auth login (~/.cache/huggingface/token)");
2669
- console.error(" export HF_TOKEN=hf_…");
2670
- console.error(" web UI Settings → Hugging Face (~/.mlx-bun/hf.json)");
2671
- process.exit(1);
2672
- }
2673
- const { banner, step, box, style } = await import("./tui");
2674
- banner(pkg.version);
2675
- const s = step(`uploading ${dir} → ${repo}`);
2676
- try {
2677
- const r = await uploadFolder(dir, repo, {
2678
- repoType: "model",
2679
- private: flag("private"),
2680
- token,
2681
- commitMessage: "Upload with mlx-bun",
2682
- onProgress: (file, sent, total) =>
2683
- s.update(`${style.bold(repo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
2684
- });
2685
- s.done(`uploaded ${style.bold(repo)}`);
2686
- console.log();
2687
- box([
2688
- `${style.green("●")} ${style.bold("upload complete")}`,
2689
- "",
2690
- `source ${style.dim(dir)}`,
2691
- `repo ${style.url(r.url)}${flag("private") ? style.dim(" · private") : ""}`,
2692
- "",
2693
- `get it back anywhere: ${style.accent(`mlx-bun get ${repo}`)}`,
2694
- ]);
2695
- } catch (e) {
2696
- s.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
2697
- process.exit(1);
2698
- }
2699
- break;
2700
- }
2701
-
2702
- case "perplexity": {
2703
- // mlx_lm.perplexity counterpart (methodology reproduced exactly — see
2704
- // src/eval/perplexity.ts) over a LOCAL dataset file; never downloads data.
2705
- const modelArg = positional(0) ?? opt("model") ?? opt("query");
2706
- const dataPath = opt("data-path");
2707
- if (!modelArg || !dataPath) {
2708
- console.error("usage: mlx-bun perplexity <model-query-or-path> --data-path <file.txt|file.jsonl> [--sequence-length 512] [--num-samples 256] [--batch-size 8] [--seed 123]");
2709
- process.exit(1);
2710
- }
2711
- const { existsSync } = await import("node:fs");
2712
- if (!existsSync(dataPath)) {
2713
- console.error(`data file not found: ${dataPath} (a local .txt or .jsonl — HF datasets are not downloaded)`);
2714
- process.exit(1);
2715
- }
2716
- const intFlag = (name: string, dflt: number, lo: number): number => {
2717
- const v = Number(opt(name, String(dflt)));
2718
- if (!Number.isInteger(v) || (v < lo && v !== -1)) {
2719
- console.error(`--${name} expects an integer >= ${lo} (got "${opt(name)}")`);
2720
- process.exit(1);
2721
- }
2722
- return v;
2723
- };
2724
- const seqLen = intFlag("sequence-length", 512, 2);
2725
- const numSamples = intFlag("num-samples", 256, 1); // -1 = all
2726
- const batchSize = intFlag("batch-size", 8, 1);
2727
- const seed = intFlag("seed", 123, 0);
2728
- const { banner, step, box, style } = await import("./tui");
2729
- banner(pkg.version);
2730
- const sNative = step("native runtime");
2731
- await ensureNative(sNative);
2732
- sNative.done("native runtime ready");
2733
- let modelDir = modelArg;
2734
- if (!existsSync(`${modelArg}/config.json`)) {
2735
- const reg = new Registry();
2736
- if (reg.list().length === 0) await reg.scan();
2737
- modelDir = reg.resolve(modelArg).path;
2738
- }
2739
- const sLoad = step(`loading ${modelDir}`);
2740
- const { Weights } = await import("./weights");
2741
- const { createModel } = await import("./model/factory");
2742
- const { loadTokenizer } = await import("./tokenizer");
2743
- const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
2744
- const config = await loadModelConfig(modelDir);
2745
- const model = createModel(await Weights.open(modelDir), config);
2746
- const tok = await loadTokenizer(modelDir);
2747
- sLoad.done(`model loaded ${style.dim(`· ${modelDir}`)}`);
2748
-
2749
- const { parseSamples, packRows, evalPpl } = await import("./eval/perplexity");
2750
- const sData = step(`tokenizing ${dataPath}`);
2751
- const samples = parseSamples(await Bun.file(dataPath).text(), dataPath);
2752
- const rows = packRows(
2753
- samples.map((t) => tok.encode(t)),
2754
- { sequenceLength: seqLen, numSamples, seed },
2755
- );
2756
- if (rows.length === 0) {
2757
- sData.fail(`dataset too small: fewer than ${seqLen} tokens (need at least one full row)`);
2758
- process.exit(1);
2759
- }
2760
- sData.done(`${rows.length} row(s) × ${seqLen} tokens ${style.dim(`· ${samples.length} sample(s), seed ${seed}`)}`);
2761
-
2762
- const sEval = step(`evaluating (batch ${batchSize})`);
2763
- resetPeakMemory();
2764
- const t0 = performance.now();
2765
- const r = evalPpl(model, rows, batchSize, (done, total) =>
2766
- sEval.update(`batch ${done}/${total}`));
2767
- const seconds = (performance.now() - t0) / 1000;
2768
- sEval.done(`evaluated ${r.tokens.toLocaleString()} tokens ${style.dim(`in ${seconds.toFixed(1)} s`)}`);
2769
- console.log();
2770
- box([
2771
- `${style.green("●")} ${style.bold("perplexity")} ${style.dim(`· ${modelDir.split("/snapshots/")[0]?.split("/").at(-1) ?? modelDir}`)}`,
2772
- "",
2773
- `ppl ${style.green(style.bold(r.ppl.toFixed(3)))} ${style.dim(`± ${r.standardError.toFixed(3)}`)}`,
2774
- `mean CE ${style.bold(r.meanLoss.toFixed(4))} ${style.dim("nats/token")}`,
2775
- `tokens ${style.bold(r.tokens.toLocaleString())} ${style.dim(`· ${r.rows} row(s) × ${seqLen}`)}`,
2776
- `speed ${style.dim(`${(r.tokens / seconds).toFixed(0)} tok/s · peak ${gb(peakMemory())}`)}`,
2777
- ]);
2778
- break;
2779
- }
2780
-
2781
- // `setup` is a true alias for `memory` — same handler, same subcommands
2782
- // (`mlx-bun setup init` == `mlx-bun memory init`; the handler reads the
2783
- // subcommand from argv[1] onward, identical for both spellings).
2784
- case "setup":
2785
- case "memory": {
2786
- const { style, box, step, banner } = await import("./tui");
2787
- const {
2788
- vaultRoot, vaultStatus, setupVault, importArticlesFrom, commitVault,
2789
- listMemoryDocuments, readArticle, resolveArticlePath, searchArticles,
2790
- parseToc, extractSection, getArticleLinks,
2791
- } = await import("./memory/vault");
2792
- const { installSchedule, removeSchedule, scheduleStatus, parseAt } = await import("./memory/schedule");
2793
- const root = vaultRoot();
2794
- const fmtAt = (a: { hour: number; minute: number }) => `${String(a.hour).padStart(2, "0")}:${String(a.minute).padStart(2, "0")}`;
2795
- const openMemoryTarget = async (targetPath = root): Promise<"obsidian" | "fallback"> => {
2796
- // For articles, Obsidian's URL handler opens the exact file inside the
2797
- // vault. For the vault root, `open -a Obsidian <folder>` opens the folder
2798
- // as a vault. Both fall back to macOS `open` if Obsidian is unavailable.
2799
- if (targetPath !== root) {
2800
- const uri = `obsidian://open?path=${encodeURIComponent(targetPath)}`;
2801
- const byUri = Bun.spawn(["open", uri], { stdout: "ignore", stderr: "ignore" });
2802
- if ((await byUri.exited) === 0) return "obsidian";
2803
- } else {
2804
- const byApp = Bun.spawn(["open", "-a", "Obsidian", root], { stdout: "ignore", stderr: "ignore" });
2805
- if ((await byApp.exited) === 0) return "obsidian";
2806
- }
2807
- const fallback = Bun.spawn(["open", targetPath], { stdout: "ignore", stderr: "ignore" });
2808
- await fallback.exited;
2809
- return "fallback";
2810
- };
2811
-
2812
- // Words after `memory`, minus flags: [subcommand, ...query].
2813
- const words = argv.slice(1).filter((a) => !a.startsWith("-"));
2814
- const sub = (words[0] ?? "status").toLowerCase();
2815
- const rest = words.slice(1).join(" ").trim();
2816
-
2817
- // TTY-only line prompt; in a non-interactive shell returns the default so
2818
- // the wizard degrades to a non-destructive no-op rather than hanging.
2819
- const ask = async (q: string, dflt = ""): Promise<string> => {
2820
- if (!process.stdin.isTTY) return dflt;
2821
- const { createInterface } = await import("node:readline/promises");
2822
- const rl = createInterface({ input: process.stdin, output: process.stdout });
2823
- try {
2824
- return ((await rl.question(q)).trim() || dflt);
2825
- } finally {
2826
- rl.close();
2827
- }
2828
- };
2829
- const confirmYN = async (q: string, defaultYes: boolean): Promise<boolean> => {
2830
- const a = (await ask(`${q} ${defaultYes ? "[Y/n]" : "[y/N]"} `)).toLowerCase();
2831
- if (!a) return defaultYes;
2832
- return a === "y" || a === "yes";
2833
- };
2834
-
2835
- if (sub === "init" || sub === "setup") {
2836
- banner(pkg.version);
2837
- box([
2838
- `${style.accent("●")} ${style.bold("Personal memory for your local AI")}`,
2839
- "",
2840
- "A wiki of Markdown articles your local assistant reads to remember",
2841
- "your projects, people, and history across sessions. It's yours —",
2842
- `git-tracked at ${style.dim(root)}, editable in any tool, and it`,
2843
- "never leaves this machine.",
2844
- ]);
2845
- const sSetup = step("creating your wiki");
2846
- const res = await setupVault(root);
2847
- sSetup.done(res.alreadySetUp ? `already set up ${style.dim(`· ${root}`)}` : `wiki ready ${style.dim(`· ${root}`)}`);
2848
-
2849
- // Offer to seed from an existing vault (e.g. a lucien ~/Dreaming) so the
2850
- // assistant has real content to read on day one, before synthesis exists.
2851
- const { homedir } = await import("node:os");
2852
- const { join } = await import("node:path");
2853
- const { readdir } = await import("node:fs/promises");
2854
- const seedRoot = await ask(
2855
- `\n Seed from an existing wiki? Enter its path, or blank to skip ${style.dim(`[${join(homedir(), "Dreaming")}]`)}: `,
2856
- "",
2857
- );
2858
- if (seedRoot) {
2859
- const srcArticles = join(seedRoot.replace(/^~(?=$|[/\\])/, homedir()), "articles");
2860
- let count = 0;
2861
- try { count = (await readdir(srcArticles)).filter((n) => n.endsWith(".md")).length; } catch { count = 0; }
2862
- if (count === 0) {
2863
- console.log(style.dim(` no articles found under ${srcArticles} — skipping import`));
2864
- } else if (await confirmYN(` Import ${count} article(s) from ${srcArticles}?`, true)) {
2865
- const sImp = step(`importing ${count} article(s)`);
2866
- const imported = await importArticlesFrom(srcArticles, root);
2867
- await commitVault(root, `Import ${imported.length} article(s) from ${seedRoot}`);
2868
- sImp.done(`imported ${imported.length} article(s)`);
2869
- }
2870
- }
2871
-
2872
- // Offer to install the nightly synthesis job (a launchd agent). The job
2873
- // runs the real synthesis DAG (`mlx-bun memory synthesize`) — create new
2874
- // entity articles, then the editorial wikify sweep — so installing the
2875
- // schedule keeps your wiki current automatically. Persistent action →
2876
- // TTY-only, defaults to no.
2877
- let scheduled: string | null = null;
2878
- console.log(style.dim(
2879
- "\n Synthesis (turning your conversations into articles) runs as a nightly job;\n" +
2880
- " install it now so your wiki stays current automatically.",
2881
- ));
2882
- if (await confirmYN(" Install the nightly synthesis job (runs in the background)?", false)) {
2883
- const atRaw = await ask(` At what time? 24h HH:MM ${style.dim("[03:00]")}: `, "03:00");
2884
- const sSched = step(`installing nightly job at ${fmtAt(parseAt(atRaw))}`);
2885
- const r = await installSchedule({ at: atRaw });
2886
- scheduled = fmtAt(r.at);
2887
- sSched.done(
2888
- r.loaded
2889
- ? `nightly job installed ${style.dim(`· ${scheduled} · ${r.plistPath}`)}`
2890
- : `plist written ${style.dim(`· ${r.plistPath} · launchctl load failed (load it manually)`)}`,
2891
- );
2892
- }
2893
-
2894
- const st = await vaultStatus(root);
2895
- console.log();
2896
- box([
2897
- `${style.green("●")} ${style.bold("memory is set up")} ${style.dim(`· ${st.articleCount} article(s) · ${st.referenceCount} reference doc(s)`)}`,
2898
- "",
2899
- `chat now ${style.accent("mlx-bun pi")} ${style.dim("— the assistant now reads your memory automatically")}`,
2900
- `inspect ${style.accent("mlx-bun memory status")} ${style.dim("·")} ${style.accent("mlx-bun memory search <q>")}`,
2901
- `browse ${style.dim(`open ${root} in Obsidian, or any editor`)}`,
2902
- scheduled
2903
- ? `nightly ${style.green("scheduled")} ${style.dim(`· ${scheduled} daily · mlx-bun memory unschedule to undo`)}`
2904
- : `nightly ${style.dim("not scheduled · mlx-bun memory schedule to set it up")}`,
2905
- "",
2906
- style.dim("Synthesis (conversations → articles) runs the full DAG: create new entity"),
2907
- style.dim("articles, then an editorial wikify sweep. Run it now: mlx-bun memory synthesize."),
2908
- ]);
2909
- break;
2910
- }
2911
-
2912
- if (sub === "status") {
2913
- const st = await vaultStatus(root);
2914
- if (!st.exists) {
2915
- console.log(`no memory wiki yet at ${style.dim(root)}`);
2916
- console.log(`set one up: ${style.accent("mlx-bun memory init")}`);
2917
- break;
2918
- }
2919
- const sched = await scheduleStatus();
2920
- console.log();
2921
- box([
2922
- `${style.bold("memory")} ${style.dim(`· ${root}`)}`,
2923
- "",
2924
- `articles ${style.bold(String(st.articleCount))}`,
2925
- `reference ${style.bold(String(st.referenceCount))} ${style.dim("read-only docs")}`,
2926
- `git ${st.isGitRepo ? style.green("tracked") : style.dim("not a git repo")}`,
2927
- `synthesis ${style.green("available")} ${style.dim("· mlx-bun memory synthesize")}`,
2928
- `nightly ${
2929
- sched.installed
2930
- ? `${sched.loaded ? style.green("scheduled") : style.accent("installed (not loaded)")} ${style.dim(`· ${sched.plistPath}`)}`
2931
- : style.dim("not scheduled · mlx-bun memory schedule")
2932
- }`,
2933
- st.recentArticles.length
2934
- ? `recent ${st.recentArticles.slice(0, 5).map((r) => r.article).join(", ")}`
2935
- : `recent ${style.dim("none")}`,
2936
- ]);
2937
- break;
2938
- }
2939
-
2940
- if (sub === "open" || sub === "browse") {
2941
- const st = await vaultStatus(root);
2942
- if (!st.exists) {
2943
- console.log(`no memory wiki yet at ${style.dim(root)}`);
2944
- console.log(`set one up: ${style.accent("mlx-bun memory init")}`);
2945
- break;
2946
- }
2947
- let targetPath = root;
2948
- let label = "memory";
2949
- if (rest) {
2950
- try {
2951
- targetPath = await resolveArticlePath(root, rest);
2952
- label = rest.endsWith(".md") ? rest.slice(0, -3) : rest;
2953
- } catch (err) {
2954
- console.error(err instanceof Error ? err.message : String(err));
2955
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
2956
- process.exit(1);
2957
- }
2958
- }
2959
- const where = await openMemoryTarget(targetPath);
2960
- console.log(
2961
- where === "obsidian"
2962
- ? `opened ${label} in Obsidian: ${targetPath}`
2963
- : `opened ${label}: ${targetPath}`,
2964
- );
2965
- break;
2966
- }
2967
-
2968
- // Independent, chronological, resumable STAGE WORKERS. Each pulls its own
2969
- // eligible work from the DB by state, processes a bounded batch, persists,
2970
- // and exits — so a user can run the four as four separate processes on
2971
- // different slices concurrently (GPU/memory allowing). `synthesize` (below)
2972
- // remains the FULL DAG; these are its decomposed pieces.
2973
- if (sub === "segment" || sub === "extract" || sub === "route" || sub === "synthesize-stage" || sub === "stage-synthesize") {
2974
- const stages = await import("./memory/stages");
2975
- const { MemoryStore } = await import("./memory/db");
2976
- const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
2977
- const convsRaw = opt("convs");
2978
- const convIds = convsRaw ? convsRaw.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
2979
- const onEvent = (e: import("./memory/pipeline").SynthesisEvent) => {
2980
- if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
2981
- else console.log(` ${e.message}`);
2982
- };
2983
- const store = new MemoryStore();
2984
- console.log();
2985
- try {
2986
- if (sub === "segment") {
2987
- const r = await stages.runSegmentStage(store, { convIds, limit, onEvent });
2988
- console.log(style.dim(`\n segment: ${r.valid} segmented, ${r.chunks} chunks, ${r.skipped} skipped, ${r.errored} errored`));
2989
- } else if (sub === "extract") {
2990
- const r = await stages.runExtractStage(store, { convIds, limit, onEvent });
2991
- console.log(style.dim(`\n extract: ${r.extracted} chunk(s) extracted, ${r.remaining} still pending`));
2992
- } else if (sub === "route") {
2993
- const r = await stages.runRouteStage(store, { convIds, onEvent });
2994
- console.log(style.dim(`\n route: ${r.decisions.length} entities — ${r.createEligible} create-eligible, ${r.captured.length} captured`));
2995
- } else {
2996
- const r = await stages.runSynthesizeStage(store, { root, convIds, limit, onEvent });
2997
- console.log(style.dim(`\n synthesize: ${r.created.length} created, ${r.patched.length} patched, ${r.skippedByGate.length} gated`));
2998
- }
2999
- } finally {
3000
- store.close();
3001
- }
3002
- break;
3003
- }
3004
-
3005
- // CROSS-LINK — the dedicated edge-building stage. Deterministic + idempotent:
3006
- // inline-link first mentions of other articles + rebuild each ## See also from
3007
- // mentions + co-occurrence. No model required.
3008
- if (sub === "link") {
3009
- const { runLinkStage } = await import("./memory/crosslink");
3010
- const { MemoryStore } = await import("./memory/db");
3011
- const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
3012
- const store = new MemoryStore();
3013
- console.log();
3014
- try {
3015
- const r = await runLinkStage(store, {
3016
- root,
3017
- limit,
3018
- onEvent: (e) => console.log(e.type === "stage" ? ` ${style.dim("·")} ${e.message}` : ` ${e.message}`),
3019
- });
3020
- console.log(
3021
- style.dim(
3022
- `\n link: ${r.linked.length} article(s) linked · ${r.mentionEdges} mention edge(s) · ${r.skippedByGate.length} gated`,
3023
- ),
3024
- );
3025
- } finally {
3026
- store.close();
3027
- }
3028
- break;
3029
- }
3030
-
3031
- if (sub === "synthesize" || sub === "pipeline" || sub === "all") {
3032
- const { runSynthesis } = await import("./memory/pipeline");
3033
- const dryRun = flag("dry-run");
3034
- console.log();
3035
- const summary = await runSynthesis(
3036
- { since: opt("since") ?? undefined, model: opt("model") ?? undefined, dryRun },
3037
- (e) => {
3038
- if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
3039
- else console.log(` ${e.message}`);
3040
- },
3041
- );
3042
- console.log(style.dim(`\n ${summary.note}`));
3043
- break;
3044
- }
3045
-
3046
- if (sub === "schedule") {
3047
- const atRaw = opt("at") ?? "03:00";
3048
- const r = await installSchedule({ at: atRaw });
3049
- console.log();
3050
- box([
3051
- r.loaded
3052
- ? `${style.green("●")} ${style.bold("nightly synthesis scheduled")} ${style.dim(`· ${fmtAt(r.at)} daily`)}`
3053
- : `${style.accent("●")} ${style.bold("plist written, but launchctl load failed")}`,
3054
- "",
3055
- `plist ${style.dim(r.plistPath)}`,
3056
- `runs ${style.dim("mlx-bun memory synthesize (full DAG: create + wikify sweep)")}`,
3057
- `undo ${style.accent("mlx-bun memory unschedule")}`,
3058
- ]);
3059
- break;
3060
- }
3061
-
3062
- if (sub === "unschedule") {
3063
- const removed = await removeSchedule();
3064
- console.log(removed ? "nightly synthesis job removed" : "no nightly job was installed");
3065
- break;
3066
- }
3067
-
3068
- if (sub === "list") {
3069
- const docs = await listMemoryDocuments(root);
3070
- if (docs.length === 0) { console.log("no articles or reference docs yet"); break; }
3071
- for (const s of docs) console.log(` ${s}`);
3072
- const refs = docs.filter((s) => s.startsWith("Reference/")).length;
3073
- console.log(style.dim(`\n ${docs.length - refs} article(s), ${refs} read-only reference doc(s)`));
3074
- break;
3075
- }
3076
-
3077
- if (sub === "search") {
3078
- if (!rest) { console.error("usage: mlx-bun memory search <query>"); process.exit(1); }
3079
- const { summaries, hits } = await searchArticles(root, rest, { limit: 12 });
3080
- if (summaries.length === 0) { console.log(`no articles match "${rest}"`); break; }
3081
- console.log();
3082
- for (const s of summaries.slice(0, 15)) {
3083
- const terms = s.matched_terms?.length ? style.dim(` [${s.matched_terms.join(", ")}]`) : "";
3084
- console.log(` ${style.bold(s.article)} ${style.dim(`· ${s.occurrences} hit(s)`)}${terms}`);
3085
- }
3086
- if (hits.length) {
3087
- console.log(style.dim("\n sample lines:"));
3088
- for (const h of hits.slice(0, 6)) {
3089
- const where = h.anchor ? `${h.article}#${h.anchor}` : h.article;
3090
- console.log(` ${style.dim(`${where}:${h.line}`)} ${h.excerpt}`);
3091
- }
3092
- }
3093
- console.log(style.dim(`\n read one: mlx-bun memory read ${summaries[0]!.article}`));
3094
- break;
3095
- }
3096
-
3097
- if (sub === "toc") {
3098
- if (!rest) { console.error("usage: mlx-bun memory toc <article>"); process.exit(1); }
3099
- try {
3100
- const { content } = await readArticle(root, rest);
3101
- const toc = parseToc(content);
3102
- if (toc.length === 0) { console.log("no headings"); break; }
3103
- for (const h of toc) console.log(`${" ".repeat(Math.max(0, h.depth - 1))}- ${h.title} ${style.dim(`#${h.anchor}`)}`);
3104
- } catch (err) {
3105
- console.error(err instanceof Error ? err.message : String(err));
3106
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3107
- process.exit(1);
3108
- }
3109
- break;
3110
- }
3111
-
3112
- if (sub === "section") {
3113
- const [article, anchorRaw] = words.slice(1);
3114
- const anchor = anchorRaw?.replace(/^#/, "");
3115
- if (!article || !anchor) { console.error("usage: mlx-bun memory section <article> <anchor>"); process.exit(1); }
3116
- try {
3117
- const { content } = await readArticle(root, article);
3118
- const section = extractSection(content, anchor);
3119
- if (!section) { console.error(`no section #${anchor} in ${article}`); console.error(`try: ${style.accent(`mlx-bun memory toc ${article}`)}`); process.exit(1); }
3120
- console.log(section);
3121
- } catch (err) {
3122
- console.error(err instanceof Error ? err.message : String(err));
3123
- console.error(`try: ${style.accent(`mlx-bun memory search ${article}`)}`);
3124
- process.exit(1);
3125
- }
3126
- break;
3127
- }
3128
-
3129
- if (sub === "links") {
3130
- if (!rest) { console.error("usage: mlx-bun memory links <article>"); process.exit(1); }
3131
- try {
3132
- const { outbound, inbound } = await getArticleLinks(root, rest);
3133
- console.log(`${style.bold("outbound")}:`);
3134
- console.log(outbound.length ? outbound.map((s) => ` ${s}`).join("\n") : " (none)");
3135
- console.log(`\n${style.bold("inbound")}:`);
3136
- console.log(inbound.length ? inbound.map((s) => ` ${s}`).join("\n") : " (none)");
3137
- } catch (err) {
3138
- console.error(err instanceof Error ? err.message : String(err));
3139
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3140
- process.exit(1);
3141
- }
3142
- break;
3143
- }
3144
-
3145
- if (sub === "read") {
3146
- if (!rest) { console.error("usage: mlx-bun memory read <article>"); process.exit(1); }
3147
- try {
3148
- const { content } = await readArticle(root, rest);
3149
- console.log(content);
3150
- } catch (err) {
3151
- console.error(err instanceof Error ? err.message : String(err));
3152
- console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
3153
- process.exit(1);
3154
- }
3155
- break;
3156
- }
3157
-
3158
-
3159
- // Unknown subcommand: help on stdout, but a FAILURE exit — scripts and
3160
- // launchd jobs must be able to detect typos (printHelp would exit 0).
3161
- console.error(`unknown: mlx-bun memory ${sub}`);
3162
- console.log(renderHelp(HELP.memory!));
3163
- process.exit(1);
3164
- }
3165
-
3166
- default:
3167
- printHelp(cmd);
3168
- }