mlx-bun 0.5.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1449 -109
- package/THIRD_PARTY_NOTICES.md +882 -0
- package/bin/mlx-bun.convert.mjs +4 -0
- package/bin/mlx-bun.fuse.mjs +4 -0
- package/bin/mlx-bun.generate.mjs +4 -0
- package/bin/mlx-bun.lora.mjs +4 -0
- package/bin/mlx-bun.mjs +13 -21
- package/bin/mlx-bun.server.mjs +4 -0
- package/bin/mlx-bun.upload.mjs +4 -0
- package/dist/web/app.js +9584 -0
- package/package.json +50 -53
- package/src/cli/ane-install.ts +76 -0
- package/src/cli/args.ts +179 -0
- package/src/cli/engine-entry.ts +68 -0
- package/src/cli/inference.ts +214 -0
- package/src/cli/job-entry.ts +97 -0
- package/src/cli/library-host.ts +117 -0
- package/src/cli/main.ts +57 -0
- package/src/cli/memory-engine.ts +163 -0
- package/src/cli/memory.ts +36 -0
- package/src/cli/mlx-lm-aliases.ts +309 -0
- package/src/cli/model-choice.ts +33 -0
- package/src/cli/model-selection.ts +84 -0
- package/src/cli/module-host.ts +11 -0
- package/src/cli/module-verbs.ts +50 -0
- package/src/cli/numerical-policy.ts +12 -0
- package/src/cli/perplexity.ts +89 -0
- package/src/cli/serve-host.ts +423 -0
- package/src/cli/serve-isolated.ts +217 -0
- package/src/cli/serve-options.ts +106 -0
- package/src/cli/serve-state.ts +205 -0
- package/src/cli/serve.ts +500 -0
- package/src/cli/served-model-host.ts +102 -0
- package/src/cli/server-entry.ts +148 -0
- package/src/cli/serving-unit.ts +177 -0
- package/src/{tui.ts → cli/terminal.ts} +13 -8
- package/src/cli/worker-adapters.ts +34 -0
- package/src/cli/worker-entry.ts +272 -0
- package/src/cli/worker-unit.ts +179 -0
- package/src/engine/adapter-operation.ts +26 -0
- package/src/engine/cache-services.ts +114 -0
- package/src/engine/completion.ts +39 -0
- package/src/engine/generation-gateway.ts +358 -0
- package/src/engine/index.ts +59 -0
- package/src/engine/kv-budget.ts +120 -0
- package/src/engine/media-preparation.ts +232 -0
- package/src/engine/model-binding.ts +24 -0
- package/src/engine/model-host.ts +513 -0
- package/src/engine/model-serving.ts +89 -0
- package/src/{serve → engine}/preparation.ts +7 -5
- package/src/{serve → engine}/session-completion-engine.ts +6 -6
- package/src/engine/telemetry.ts +109 -0
- package/src/hub/downloads.ts +116 -0
- package/src/jobs/db.ts +16 -7
- package/src/jobs/events.ts +29 -0
- package/src/jobs/executable.ts +5 -0
- package/src/jobs/host.ts +51 -0
- package/src/jobs/in-process.ts +55 -0
- package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
- package/src/jobs/runner.ts +150 -231
- package/src/jobs/service.ts +75 -0
- package/src/jobs/sse.ts +59 -23
- package/src/jobs/worker-memory.ts +46 -0
- package/src/jobs/worker-process.ts +163 -0
- package/src/jobs/worker-supervisor.ts +258 -0
- package/src/modules.ts +58 -0
- package/src/publishing/catalog-hub.ts +18 -0
- package/src/publishing/credentials.ts +31 -0
- package/src/publishing/upload.ts +37 -0
- package/src/residency/leased-adapters.ts +21 -0
- package/src/residency/local-records.ts +16 -0
- package/src/residency/model-residency.ts +405 -0
- package/src/residency/record-index.ts +28 -0
- package/src/residency/resident-estimate.ts +50 -0
- package/src/{anthropic.ts → server/anthropic.ts} +3 -81
- package/src/server/cache-routes.ts +36 -0
- package/src/{serve → server}/chat-request.ts +10 -10
- package/src/{serve → server}/chat-stage.ts +14 -14
- package/src/{client.ts → server/client.ts} +28 -2
- package/src/{serve → server}/completion-executor.ts +33 -41
- package/src/server/decision-body.ts +55 -0
- package/src/server/decision-routes.ts +134 -0
- package/src/server/decision-wire.ts +228 -0
- package/src/{serve → server}/discovery-routes.ts +25 -82
- package/src/{serve → server}/http.ts +25 -13
- package/src/{serve → server}/inference-request.ts +5 -5
- package/src/server/job-routes.ts +20 -0
- package/src/server/media-prompt.ts +63 -0
- package/src/server/memory-completion-client.ts +46 -0
- package/src/server/model-routes.ts +192 -0
- package/src/{serve → server}/openai-wire.ts +3 -2
- package/src/{serve → server}/pipeline.ts +1 -1
- package/src/server/prompt-builder.ts +13 -0
- package/src/server/prompt-contracts.ts +17 -0
- package/src/server/proxy-routes.ts +187 -0
- package/src/server/publishing-routes.ts +41 -0
- package/src/{serve → server}/request-plan.ts +6 -8
- package/src/{serve → server}/request-prep.ts +47 -50
- package/src/{serve → server}/responses-client.ts +30 -13
- package/src/{responses.ts → server/responses.ts} +12 -93
- package/src/server/routes.ts +205 -0
- package/src/server/start.ts +72 -0
- package/src/server/status-routes.ts +108 -0
- package/src/{serve → server}/text-completion-stage.ts +25 -4
- package/src/server/text-prompt.ts +12 -0
- package/src/{serve → server}/token-streams.ts +36 -38
- package/src/server/worker-routes.ts +311 -0
- package/src/storage/paths.ts +61 -0
- package/src/web/assets.ts +37 -0
- package/src/web/{src → browser}/api.ts +5 -23
- package/src/web/browser/chat-host.ts +17 -0
- package/src/web/browser/installed-panels.ts +6 -0
- package/src/web/browser/main.ts +55 -0
- package/src/web/browser/markdown.ts +279 -0
- package/src/web/browser/memory-panel.ts +9 -0
- package/src/web/browser/palette.ts +64 -0
- package/src/web/browser/protocol.ts +13 -0
- package/src/web/browser/shell.ts +467 -0
- package/src/web/{src → browser}/status.ts +32 -36
- package/src/web/build.ts +51 -0
- package/src/web/public/app.html +1054 -0
- package/src/web/{sw.js → public/sw.js} +4 -3
- package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
- package/src/web/{vendor → public/vendor}/hljs.js +2 -2
- package/THIRD_PARTY_LICENSES.md +0 -51
- package/docs/README.md +0 -107
- package/docs/archive/hlg-sampling.md +0 -350
- package/docs/archive/investigations/audio-input-plan.md +0 -356
- package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +0 -3617
- package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
- package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
- package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
- package/docs/archive/investigations/curve-bisector-routing.md +0 -123
- package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
- package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
- package/docs/archive/investigations/dspark-handoff.md +0 -224
- package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
- package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
- package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
- package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
- package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
- package/docs/archive/investigations/lab-build-journal.md +0 -157
- package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
- package/docs/archive/investigations/openjarvis-fit.md +0 -198
- package/docs/archive/investigations/openwiki-evaluation.md +0 -249
- package/docs/archive/investigations/optimization_plan.md +0 -346
- package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
- package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
- package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
- package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
- package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
- package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
- package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
- package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
- package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
- package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
- package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
- package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
- package/docs/archive/planning/chat-ui-vision.md +0 -50
- package/docs/archive/planning/chunk-task-roadmap.md +0 -77
- package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
- package/docs/archive/planning/journal.md +0 -316
- package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
- package/docs/archive/planning/pi-integration-review.md +0 -178
- package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
- package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
- package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
- package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
- package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
- package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
- package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
- package/docs/archive/planning/release-notes-v0.4.0.md +0 -45
- package/docs/archive/planning/release-notes-v0.5.0.md +0 -37
- package/docs/archive/planning/web-ui-pass-plan.md +0 -213
- package/docs/archive/planning/website-readme-pass-plan.md +0 -224
- package/docs/design/batching.md +0 -2054
- package/docs/design/decode-speed-program.md +0 -2404
- package/docs/design/dreaming-nightly-pipeline.md +0 -684
- package/docs/design/generic-model-support.md +0 -821
- package/docs/design/kv-cache.md +0 -993
- package/docs/design/orpo-training.md +0 -680
- package/docs/design/speculative-decoding.md +0 -2129
- package/docs/design/turboquant.md +0 -2225
- package/docs/design/unified-engine-frontier-plan.md +0 -1987
- package/docs/design/web-chat-redesign.md +0 -695
- package/docs/planning/PRODUCT_ROADMAP.md +0 -593
- package/docs/planning/ResearchTopics.md +0 -119
- package/docs/planning/public-readiness-handoff.md +0 -244
- package/docs/planning/public-readiness.md +0 -149
- package/docs/planning/video-series.md +0 -421
- package/docs/reference/benchmarks.md +0 -3587
- package/docs/reference/cli.md +0 -646
- package/docs/reference/distribution.md +0 -380
- package/docs/reference/environment.md +0 -494
- package/docs/reference/glossary.md +0 -39
- package/docs/reference/library-api.md +0 -688
- package/docs/reference/memory.md +0 -260
- package/docs/reference/models.md +0 -346
- package/docs/reference/server-api.md +0 -1945
- package/docs/reference/server-config.md +0 -973
- package/docs/reference/training.md +0 -462
- package/docs/reference/troubleshooting.md +0 -158
- package/src/admission-plan.ts +0 -18
- package/src/audio/audiotoolbox.ts +0 -112
- package/src/audio/conformer.ts +0 -655
- package/src/audio/decode.ts +0 -133
- package/src/audio/features.ts +0 -202
- package/src/audio/mic-capture.ts +0 -98
- package/src/audio/silero-vad.ts +0 -379
- package/src/audio/transcode.ts +0 -67
- package/src/audio/whisper-decode.ts +0 -774
- package/src/audio/whisper-fast.ts +0 -732
- package/src/audio/whisper-format.ts +0 -73
- package/src/audio/whisper-mel.ts +0 -185
- package/src/audio/whisper-timing.ts +0 -274
- package/src/audio/whisper-tokenizer.ts +0 -239
- package/src/audio/whisper-transcribe.ts +0 -410
- package/src/backends/mlx/assistant-target.ts +0 -46
- package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
- package/src/backends/mlx/autoregressive.ts +0 -129
- package/src/backends/mlx/batch-group.ts +0 -1874
- package/src/backends/mlx/cache-attachment.ts +0 -67
- package/src/backends/mlx/cache-layout.ts +0 -31
- package/src/backends/mlx/checkpoint-state.ts +0 -53
- package/src/backends/mlx/continuation-persistence.ts +0 -107
- package/src/backends/mlx/continuation-request.ts +0 -43
- package/src/backends/mlx/continuation.ts +0 -77
- package/src/backends/mlx/diffusion.ts +0 -39
- package/src/backends/mlx/draft-target.ts +0 -32
- package/src/backends/mlx/encoder-cache.ts +0 -25
- package/src/backends/mlx/evaluation.ts +0 -105
- package/src/backends/mlx/fill-append.ts +0 -58
- package/src/backends/mlx/fill-group.ts +0 -497
- package/src/backends/mlx/full-kv-row-append.ts +0 -32
- package/src/backends/mlx/full-kv-row-donor.ts +0 -54
- package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
- package/src/backends/mlx/gateway-binding.ts +0 -174
- package/src/backends/mlx/graph.ts +0 -38
- package/src/backends/mlx/kv-codec.ts +0 -9
- package/src/backends/mlx/kv-maintenance.ts +0 -161
- package/src/backends/mlx/kv-tensor-rows.ts +0 -212
- package/src/backends/mlx/memory-client.ts +0 -189
- package/src/backends/mlx/methods.ts +0 -90
- package/src/backends/mlx/mixed-iteration.ts +0 -62
- package/src/backends/mlx/model-prompt.ts +0 -178
- package/src/backends/mlx/model-serving.ts +0 -80
- package/src/backends/mlx/paged-cache-codec.ts +0 -27
- package/src/backends/mlx/prefill-cohort.ts +0 -100
- package/src/backends/mlx/prefill-policy.ts +0 -26
- package/src/backends/mlx/prefill-rows.ts +0 -244
- package/src/backends/mlx/prefill.ts +0 -33
- package/src/backends/mlx/projected-context-rows.ts +0 -72
- package/src/backends/mlx/prompt-input.ts +0 -29
- package/src/backends/mlx/qwen-prompt-input.ts +0 -40
- package/src/backends/mlx/request-policy.ts +0 -20
- package/src/backends/mlx/request-state-policy.ts +0 -54
- package/src/backends/mlx/rollback.ts +0 -92
- package/src/backends/mlx/row-sampling.ts +0 -49
- package/src/backends/mlx/serial-executor.ts +0 -251
- package/src/backends/mlx/speculative-group.ts +0 -438
- package/src/backends/mlx/speculative-round.ts +0 -161
- package/src/backends/mlx/speculative.ts +0 -131
- package/src/backends/mlx/state-rows.ts +0 -104
- package/src/backends/mlx/state-views.ts +0 -46
- package/src/backends/mlx/target-layout-capability.ts +0 -35
- package/src/backends/mlx/text-engine.ts +0 -73
- package/src/backends/mlx/transitioning-kv-rows.ts +0 -173
- package/src/chat-template.ts +0 -382
- package/src/cli.ts +0 -3490
- package/src/config.ts +0 -399
- package/src/contracts/completion.ts +0 -16
- package/src/contracts/execution.ts +0 -70
- package/src/contracts/generation.ts +0 -103
- package/src/contracts/host.ts +0 -7
- package/src/contracts/object-cache.ts +0 -8
- package/src/contracts/pi-provider.ts +0 -30
- package/src/contracts/pi.ts +0 -261
- package/src/contracts/prefix-cache.ts +0 -33
- package/src/contracts/resources.ts +0 -25
- package/src/contracts/sampling.ts +0 -23
- package/src/contracts/scheduling.ts +0 -54
- package/src/contracts/tsconfig.json +0 -14
- package/src/dataset/generators.ts +0 -1087
- package/src/dataset/index.ts +0 -12
- package/src/dataset/job.ts +0 -81
- package/src/dataset/llm.ts +0 -71
- package/src/dataset/registry.ts +0 -528
- package/src/diffusion/diffusion-generate.ts +0 -507
- package/src/download.ts +0 -505
- package/src/embed.ts +0 -77
- package/src/engine/admission.ts +0 -59
- package/src/engine/cancellation.ts +0 -38
- package/src/engine/engine.ts +0 -56
- package/src/engine/execution-coordinator.ts +0 -70
- package/src/engine/execution-plan.ts +0 -64
- package/src/engine/execution-tasks.ts +0 -44
- package/src/engine/namespaced-cache.ts +0 -13
- package/src/engine/resources.ts +0 -53
- package/src/engine/scheduler.ts +0 -92
- package/src/engine/session.ts +0 -247
- package/src/eval/README.md +0 -130
- package/src/eval/ifeval.ts +0 -81
- package/src/eval/kl-prompts.ts +0 -29
- package/src/eval/kl.ts +0 -590
- package/src/eval/perplexity.ts +0 -164
- package/src/eval/quality-db.ts +0 -92
- package/src/eval/reference-logits.ts +0 -207
- package/src/eval/runner.ts +0 -124
- package/src/eval/score.ts +0 -49
- package/src/eval/tasks/bfcl.ts +0 -535
- package/src/eval/tasks/gsm8k.ts +0 -98
- package/src/eval/tasks/hashhop.ts +0 -139
- package/src/eval/tasks/humaneval.ts +0 -275
- package/src/eval/tasks/ifeval.ts +0 -525
- package/src/eval/tasks/mmlu.ts +0 -157
- package/src/eval/ultrafeedback.ts +0 -60
- package/src/evaldb.ts +0 -199
- package/src/expert-io.ts +0 -393
- package/src/expert-offload-build.ts +0 -77
- package/src/expert-offload.ts +0 -76
- package/src/expert-residency.ts +0 -1134
- package/src/expert-usage.ts +0 -415
- package/src/fill/echo-index.ts +0 -188
- package/src/fill/fill-session.ts +0 -463
- package/src/fill/proposal.ts +0 -68
- package/src/fill/schema-rows.ts +0 -506
- package/src/fill/tool-boundary.ts +0 -103
- package/src/fit.ts +0 -249
- package/src/generate.ts +0 -1352
- package/src/grammar.ts +0 -604
- package/src/harness-pi.ts +0 -184
- package/src/hf-push.ts +0 -628
- package/src/hub-rest.ts +0 -242
- package/src/index.ts +0 -63
- package/src/inference/denoising.ts +0 -17
- package/src/inference/draft-acceptance.ts +0 -41
- package/src/inference/draft-output.ts +0 -49
- package/src/inference/graph.ts +0 -33
- package/src/inference/prefill.ts +0 -45
- package/src/inference/rollback.ts +0 -7
- package/src/jobs/index.ts +0 -31
- package/src/jobs/job-entry.ts +0 -107
- package/src/jobs/task-client.ts +0 -19
- package/src/jobs/types.ts +0 -2
- package/src/kv-scheme.ts +0 -279
- package/src/kv-store.ts +0 -1135
- package/src/lab/README.md +0 -22
- package/src/lab/curve/curve-designer.html +0 -303
- package/src/lab/curve/curve-sampler.ts +0 -122
- package/src/lab/expert-trace/expert-trace.ts +0 -69
- package/src/lab/paged-kv/paged-attention.ts +0 -117
- package/src/lab/paged-kv/paged-kv-rows.ts +0 -111
- package/src/lab/paged-kv/paged-kv.ts +0 -336
- package/src/library.ts +0 -40
- package/src/lora.ts +0 -408
- package/src/md-modules.d.ts +0 -54
- package/src/media-fetch.ts +0 -442
- package/src/memory/article.ts +0 -558
- package/src/memory/chunk-validation.ts +0 -150
- package/src/memory/chunk.ts +0 -318
- package/src/memory/cluster.ts +0 -351
- package/src/memory/coverage.ts +0 -134
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +0 -253
- package/src/memory/entity.ts +0 -203
- package/src/memory/gate.ts +0 -187
- package/src/memory/history.ts +0 -152
- package/src/memory/ingest.ts +0 -335
- package/src/memory/model.ts +0 -183
- package/src/memory/normalize.ts +0 -671
- package/src/memory/parse.ts +0 -22
- package/src/memory/pipeline.ts +0 -275
- package/src/memory/prompts.ts +0 -65
- package/src/memory/query.ts +0 -334
- package/src/memory/reconcile.ts +0 -734
- package/src/memory/redirect.ts +0 -108
- package/src/memory/reindex.ts +0 -325
- package/src/memory/resolve.ts +0 -307
- package/src/memory/rest.ts +0 -245
- package/src/memory/route.ts +0 -272
- package/src/memory/schedule.ts +0 -165
- package/src/memory/stages.ts +0 -777
- package/src/memory/synthesize.ts +0 -1638
- package/src/memory/tools.ts +0 -749
- package/src/memory/vault.ts +0 -1194
- package/src/memory/wikify.ts +0 -1064
- package/src/mlx/array.ts +0 -425
- package/src/mlx/autograd.ts +0 -304
- package/src/mlx/checkpoint.ts +0 -149
- package/src/mlx/compile.ts +0 -168
- package/src/mlx/custom-vjp.ts +0 -188
- package/src/mlx/ffi.ts +0 -531
- package/src/mlx/materialize.ts +0 -23
- package/src/mlx/metal-kernel.ts +0 -159
- package/src/mlx/normalized-argmax.ts +0 -116
- package/src/mlx/ops.ts +0 -1108
- package/src/mlx/quantized-rows.ts +0 -36
- package/src/mlx/token-bitmask.ts +0 -52
- package/src/mlx/turboquant-kv-decode.ts +0 -82
- package/src/mlx/turboquant-kv-inverse.ts +0 -122
- package/src/mlx/turboquant-ops.ts +0 -861
- package/src/mlx/turboquant-tables.ts +0 -237
- package/src/mmap.ts +0 -90
- package/src/model/artifact-identity.ts +0 -73
- package/src/model/batched-kv.ts +0 -235
- package/src/model/batched-mask.ts +0 -279
- package/src/model/batched-quant.ts +0 -212
- package/src/model/batched-quantized-kv.ts +0 -144
- package/src/model/batched-rotating-quant.ts +0 -353
- package/src/model/batched-rotating-state.ts +0 -149
- package/src/model/batched-rotating.ts +0 -338
- package/src/model/batched-row-storage.ts +0 -179
- package/src/model/batched-ssm.ts +0 -52
- package/src/model/batched-turboquant-kv.ts +0 -94
- package/src/model/compiled-decode.ts +0 -712
- package/src/model/decoded-kv-donor.ts +0 -12
- package/src/model/delayed-quantized-kv.ts +0 -72
- package/src/model/delayed-rotating-quantized-kv.ts +0 -93
- package/src/model/delayed-turboquant-kv.ts +0 -57
- package/src/model/diffusion-gemma.ts +0 -1026
- package/src/model/factory.ts +0 -215
- package/src/model/fingerprint.ts +0 -45
- package/src/model/flash-attention.ts +0 -694
- package/src/model/full-prefill-padding.ts +0 -56
- package/src/model/full-prefill-row.ts +0 -105
- package/src/model/gemma4-base.ts +0 -2370
- package/src/model/gemma4.ts +0 -1395
- package/src/model/generated/gemma4-12b.ts +0 -429
- package/src/model/generated/gemma4-26b.ts +0 -448
- package/src/model/generated/gemma4-e4b.ts +0 -717
- package/src/model/generated/index.ts +0 -18
- package/src/model/glm52-atlas.ts +0 -511
- package/src/model/glm52-cache.ts +0 -792
- package/src/model/glm52-config.ts +0 -262
- package/src/model/glm52-container.ts +0 -305
- package/src/model/glm52-coupling.ts +0 -388
- package/src/model/glm52-dsa.ts +0 -606
- package/src/model/glm52-expert-layout.ts +0 -253
- package/src/model/glm52-layout.ts +0 -152
- package/src/model/glm52-memory.ts +0 -500
- package/src/model/glm52-mla.ts +0 -792
- package/src/model/glm52-moe.ts +0 -223
- package/src/model/glm52-pilot.ts +0 -537
- package/src/model/glm52-reference.ts +0 -445
- package/src/model/glm52-residency.ts +0 -557
- package/src/model/glm52-resident-weights.ts +0 -135
- package/src/model/glm52-streamed-experts.ts +0 -453
- package/src/model/glm52-streamed-metal.ts +0 -610
- package/src/model/glm52-weights.ts +0 -503
- package/src/model/glm52.ts +0 -1132
- package/src/model/implementation.ts +0 -69
- package/src/model/kv-attention-view.ts +0 -70
- package/src/model/minicpm5.ts +0 -376
- package/src/model/profile.ts +0 -468
- package/src/model/quantized-append-attention.ts +0 -31
- package/src/model/qwen-append.ts +0 -12
- package/src/model/qwen-conv.ts +0 -52
- package/src/model/qwen3-delta-state.ts +0 -74
- package/src/model/qwen3-delta.ts +0 -451
- package/src/model/qwen3-moe.ts +0 -404
- package/src/model/qwen3-mrope.ts +0 -250
- package/src/model/qwen3.ts +0 -277
- package/src/model/qwen3_5-checkpoint.ts +0 -164
- package/src/model/qwen3_5.ts +0 -881
- package/src/model/rotating-kv-donor.ts +0 -35
- package/src/model/rotating-kv-layout.ts +0 -262
- package/src/model/rotating-row-transaction.ts +0 -35
- package/src/model/speculative-rotating-kv.ts +0 -101
- package/src/model/ssm-prefill-padding.ts +0 -39
- package/src/model/support.ts +0 -110
- package/src/model/token-groups.ts +0 -66
- package/src/model/trellis-balanced-scatter.ts +0 -51
- package/src/model/trellis-linear.ts +0 -672
- package/src/model/trellis-shared-m.ts +0 -151
- package/src/model/trellis-shared-scatter.ts +0 -144
- package/src/model/trellis-splitk-prefill.ts +0 -158
- package/src/model/trellis-tiled-prefill.ts +0 -136
- package/src/model/trellis-vector-expand.ts +0 -62
- package/src/model/trellis-wide-prefill.ts +0 -94
- package/src/model/turboquant-codec.ts +0 -89
- package/src/model/universal/archs.ts +0 -332
- package/src/model/universal/dense.ts +0 -612
- package/src/model/universal/modules.ts +0 -170
- package/src/model/universal/rope.ts +0 -381
- package/src/model/whisper.ts +0 -435
- package/src/native/expert_io.c +0 -523
- package/src/native/frame_extract.swift +0 -140
- package/src/native/mic_capture.swift +0 -92
- package/src/native-pack.ts +0 -163
- package/src/pi-provider.ts +0 -53
- package/src/pi-session.ts +0 -81
- package/src/pi-terminal.ts +0 -274
- package/src/pi-web.ts +0 -1940
- package/src/preflight.ts +0 -158
- package/src/prompt-cache.ts +0 -682
- package/src/quantize/allocator.ts +0 -627
- package/src/quantize/atomic-output.ts +0 -38
- package/src/quantize/calibration.ts +0 -181
- package/src/quantize/config-writer.ts +0 -145
- package/src/quantize/index.ts +0 -84
- package/src/quantize/job.ts +0 -162
- package/src/quantize/quantizer.ts +0 -666
- package/src/quantize/rotate.ts +0 -603
- package/src/quantize/safetensors-writer.ts +0 -249
- package/src/quantize/sensitivity.ts +0 -216
- package/src/quantize/trellis.ts +0 -586
- package/src/quantize/weight-transform.ts +0 -292
- package/src/registry.ts +0 -602
- package/src/runtime-config.ts +0 -77
- package/src/safetensors.ts +0 -165
- package/src/sampler.ts +0 -1012
- package/src/serve/admin-routes.ts +0 -184
- package/src/serve/assets/curve-terrain.html +0 -105
- package/src/serve/assets/training-inference-map.html +0 -575
- package/src/serve/audio-routes.ts +0 -323
- package/src/serve/aux-routes.ts +0 -132
- package/src/serve/batch-scheduler.ts +0 -8
- package/src/serve/checkpoint-identity.ts +0 -64
- package/src/serve/generation-gateway.ts +0 -537
- package/src/serve/isolate.ts +0 -609
- package/src/serve/kv-budget.ts +0 -14
- package/src/serve/lab-routes.ts +0 -341
- package/src/serve/lane-registry.ts +0 -45
- package/src/serve/model-admin-routes.ts +0 -155
- package/src/serve/model-binding.ts +0 -49
- package/src/serve/model-host.ts +0 -511
- package/src/serve/parent-application.ts +0 -56
- package/src/serve/prompt-response-trace.ts +0 -161
- package/src/serve/response-lease.ts +0 -24
- package/src/serve/session-search.ts +0 -326
- package/src/serve/static-routes.ts +0 -101
- package/src/serve/transcription-server.ts +0 -64
- package/src/serve/transcription-service.ts +0 -502
- package/src/server.ts +0 -1282
- package/src/spec/assistant-rows.ts +0 -93
- package/src/spec/assistant-source.ts +0 -126
- package/src/spec/cache-identity.ts +0 -22
- package/src/spec/deepspec-rows.ts +0 -29
- package/src/spec/deepspec-source.ts +0 -145
- package/src/spec/dflash-source.ts +0 -149
- package/src/spec/drafter.ts +0 -304
- package/src/spec/dspark/ab-stats.ts +0 -136
- package/src/spec/dspark/calibration.ts +0 -112
- package/src/spec/dspark/data-dflash.ts +0 -156
- package/src/spec/dspark/data.ts +0 -201
- package/src/spec/dspark/deepspec-module.ts +0 -776
- package/src/spec/dspark/generate-dflash.ts +0 -245
- package/src/spec/dspark/generate.ts +0 -290
- package/src/spec/dspark/loader.ts +0 -23
- package/src/spec/dspark/loss.ts +0 -213
- package/src/spec/dspark/module-dflash.ts +0 -560
- package/src/spec/dspark/module.ts +0 -540
- package/src/spec/dspark/quantize-drafter.ts +0 -91
- package/src/spec/dspark/sample.ts +0 -112
- package/src/spec/generate.ts +0 -276
- package/src/spec/glm52-mtp-graph.ts +0 -29
- package/src/spec/glm52-mtp-rows.ts +0 -132
- package/src/spec/glm52-mtp-source.ts +0 -318
- package/src/spec/glm52-mtp-state.ts +0 -33
- package/src/spec/ngram-source.ts +0 -226
- package/src/spec/projected-draft-rows.ts +0 -73
- package/src/spec/qwen-mtp-module.ts +0 -171
- package/src/spec/qwen-mtp-rows.ts +0 -230
- package/src/spec/qwen-mtp-source.ts +0 -386
- package/src/spec/qwen-mtp-state.ts +0 -30
- package/src/spec/serve-loop.ts +0 -587
- package/src/spec/source.ts +0 -228
- package/src/spec/two-model-rows.ts +0 -62
- package/src/spec/two-model.ts +0 -113
- package/src/ssd-cache.ts +0 -431
- package/src/ssd-durability.ts +0 -209
- package/src/storage/host-buffer.ts +0 -24
- package/src/storage/kv-writer.ts +0 -83
- package/src/storage/kv-writer.worker.js +0 -264
- package/src/storage/retention-policy.ts +0 -37
- package/src/tiered-prompt-cache.ts +0 -45
- package/src/tokenizer.ts +0 -335
- package/src/tool-approvals.ts +0 -131
- package/src/tool-call.ts +0 -695
- package/src/train/dataset.ts +0 -484
- package/src/train/diffusion-lora.ts +0 -216
- package/src/train/export.ts +0 -32
- package/src/train/flash-cce.ts +0 -1234
- package/src/train/forward.ts +0 -266
- package/src/train/fuse.ts +0 -243
- package/src/train/index.ts +0 -46
- package/src/train/job.ts +0 -205
- package/src/train/lora-params.ts +0 -254
- package/src/train/loss.ts +0 -1591
- package/src/train/merge.ts +0 -195
- package/src/train/optimizer.ts +0 -183
- package/src/train/prefix-shared.ts +0 -490
- package/src/train/rank.ts +0 -137
- package/src/train/segmented.ts +0 -1828
- package/src/train/steel-qmm-header.ts +0 -1461
- package/src/train/trainer.ts +0 -1221
- package/src/train/watch.ts +0 -542
- package/src/vision/diffusion-vision.ts +0 -620
- package/src/vision/embedder.ts +0 -176
- package/src/vision/preprocess.ts +0 -247
- package/src/vision/prompt.ts +0 -409
- package/src/vision/qwen3vl-preprocess.ts +0 -409
- package/src/vision/qwen3vl-prompt.ts +0 -233
- package/src/vision/qwen3vl-tower.ts +0 -566
- package/src/vision/siglip.ts +0 -607
- package/src/vision/video-frames.ts +0 -148
- package/src/web/app.html +0 -2155
- package/src/web/app.js +0 -5863
- package/src/web/skills/memory/SKILL.md +0 -120
- package/src/web/skills/web-research/SKILL.md +0 -46
- package/src/web/skills.ts +0 -97
- package/src/web/src/adapters-panel.ts +0 -288
- package/src/web/src/assistant.ts +0 -451
- package/src/web/src/chat.ts +0 -1060
- package/src/web/src/composer.ts +0 -962
- package/src/web/src/dataset.ts +0 -132
- package/src/web/src/finetune.ts +0 -267
- package/src/web/src/hub.ts +0 -428
- package/src/web/src/main.ts +0 -98
- package/src/web/src/markdown.ts +0 -0
- package/src/web/src/memory-panel.ts +0 -572
- package/src/web/src/model-picker.ts +0 -159
- package/src/web/src/palette.ts +0 -346
- package/src/web/src/protocol.ts +0 -26
- package/src/web/src/quantize.ts +0 -167
- package/src/web/src/rag.ts +0 -274
- package/src/web/src/sessions.ts +0 -469
- package/src/web/src/shell.ts +0 -804
- package/src/web/src/ui-catalog.ts +0 -78
- package/src/web/src/voice.ts +0 -141
- package/src/web/vendor/README.md +0 -61
- package/src/web-assets.ts +0 -47
- package/src/web-tools.ts +0 -478
- package/src/weights.ts +0 -255
- /package/src/{serve → server}/completion-sink.ts +0 -0
- /package/src/{serve → server}/generated-token-history.ts +0 -0
- /package/src/web/{icon.svg → public/icon.svg} +0 -0
- /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
- /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
package/src/cli.ts
DELETED
|
@@ -1,3490 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bun
|
|
2
|
-
// mlx-bun CLI: the appliance interface.
|
|
3
|
-
//
|
|
4
|
-
// mlx-bun get <org/repo> [--revision main] resumable verified download
|
|
5
|
-
// mlx-bun scan index the HF cache
|
|
6
|
-
// mlx-bun ls [--vision] [--max-size 10GB] [query]
|
|
7
|
-
// mlx-bun fit <query> [--ctx 8192] [--skus]
|
|
8
|
-
// mlx-bun serve [query] [--port 8080] [--memory-budget GB]
|
|
9
|
-
// mlx-bun evals recent benchmark runs
|
|
10
|
-
// mlx-bun harness pi [--base-url <url>] [--remove] connect your own pi to the local server
|
|
11
|
-
|
|
12
|
-
import { existsSync } from "node:fs";
|
|
13
|
-
import { basename, join, resolve } from "node:path";
|
|
14
|
-
import { Registry } from "./registry";
|
|
15
|
-
import { loadModelConfig, parseTurboQuantScheme, type TurboQuantScheme } from "./config";
|
|
16
|
-
import { fit, skuMatrix, thisMachine, type FitKvScheme } from "./fit";
|
|
17
|
-
import { resolveKvScheme } from "./kv-scheme";
|
|
18
|
-
import { EvalDB } from "./evaldb";
|
|
19
|
-
import pkg from "../package.json" with { type: "json" };
|
|
20
|
-
import { renderHelp } from "./tui";
|
|
21
|
-
import { isGlm52Config, isSupportedModelRecord } from "./model/support";
|
|
22
|
-
import {
|
|
23
|
-
GLM52_G5_ASPIRATIONAL_DECODE_TPS,
|
|
24
|
-
GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS,
|
|
25
|
-
GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES,
|
|
26
|
-
GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS,
|
|
27
|
-
GLM52_G5_MEASURED_WARM_DECODE_TPS,
|
|
28
|
-
planGlm52MemoryForArtifact,
|
|
29
|
-
} from "./model/glm52-memory";
|
|
30
|
-
import type { JobEvent } from "./jobs/types";
|
|
31
|
-
import {
|
|
32
|
-
configureRuntime,
|
|
33
|
-
runtimeValue,
|
|
34
|
-
type RuntimeKey,
|
|
35
|
-
} from "./runtime-config";
|
|
36
|
-
|
|
37
|
-
const argv = process.argv.slice(2);
|
|
38
|
-
// Private self-exec protocol for managed jobs in the single-file binary.
|
|
39
|
-
// Literal imports in job-entry keep the numerical runners inside the bundle.
|
|
40
|
-
if (argv[0] === "__job") {
|
|
41
|
-
const { runJobEntry } = await import("./jobs/job-entry");
|
|
42
|
-
await runJobEntry(argv[1]);
|
|
43
|
-
process.exit(0);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
// The appliance path: naked `mlx-bun` (or only options, e.g.
|
|
47
|
-
// `mlx-bun --port 9000`) runs `serve` — first run downloads a model and
|
|
48
|
-
// opens the chat UI. `--help`/`--version` and explicit subcommands still win.
|
|
49
|
-
const HELP_VERSION_FLAGS = ["--help", "-h", "--version", "-v"];
|
|
50
|
-
const noSubcommand =
|
|
51
|
-
!argv[0] || (argv[0].startsWith("-") && !HELP_VERSION_FLAGS.includes(argv[0]));
|
|
52
|
-
const cmd = noSubcommand ? "serve" : argv[0];
|
|
53
|
-
|
|
54
|
-
const OVERVIEW = `mlx-bun ${pkg.version} — local AI on Apple silicon. One binary, no Python.
|
|
55
|
-
|
|
56
|
-
Usage: mlx-bun [command] [options] (no command = serve)
|
|
57
|
-
|
|
58
|
-
Commands:
|
|
59
|
-
pi Launch a pi coding-agent session on a local model (the appliance path)
|
|
60
|
-
serve Start the OpenAI/Anthropic-compatible server + web app
|
|
61
|
-
get Download a model from Hugging Face (resumable, verified)
|
|
62
|
-
ls List downloaded models
|
|
63
|
-
gc Reclaim superseded model snapshots + dead blobs from the HF cache
|
|
64
|
-
fit Will a model fit this machine? Memory + speed assessment
|
|
65
|
-
scan Re-index the Hugging Face cache
|
|
66
|
-
harness Connect your own pi install to the local mlx-bun server
|
|
67
|
-
train Fine-tune a LoRA adapter (SFT / DPO / ORPO) on your data
|
|
68
|
-
train-watch Live dashboard for a training run (tails metrics.jsonl)
|
|
69
|
-
fuse Merge a LoRA adapter into the base weights (standalone model)
|
|
70
|
-
convert Quantize an HF model into a local MLX snapshot
|
|
71
|
-
upload Push a local model directory to the Hugging Face Hub
|
|
72
|
-
perplexity Perplexity of a model over a local text/JSONL dataset
|
|
73
|
-
memory Set up + manage your local AI's personal wiki memory
|
|
74
|
-
generate One-shot raw text generation from a local model (no server)
|
|
75
|
-
benchmark Measure decode/prefill speed of OUR stack on this machine
|
|
76
|
-
embed Text embeddings from a local embedding model (Qwen3-Embedding)
|
|
77
|
-
transcribe Speech-to-text from an audio file with a local Whisper model
|
|
78
|
-
dictate Push-to-talk: stream your microphone into Whisper, print/copy/type the text
|
|
79
|
-
evals Show recent benchmark runs (all stacks)
|
|
80
|
-
help Show help for a command (also: mlx-bun <command> --help)
|
|
81
|
-
|
|
82
|
-
Options:
|
|
83
|
-
-h, --help Show help
|
|
84
|
-
-v, --version Show version
|
|
85
|
-
|
|
86
|
-
Examples:
|
|
87
|
-
mlx-bun # download (first run) + serve + open the chat UI
|
|
88
|
-
mlx-bun pi # start (download if needed) and chat with an agent
|
|
89
|
-
mlx-bun serve 12B # serve the 12B; web app at http://localhost:8080/
|
|
90
|
-
mlx-bun train e4b --data ./prefs # ORPO LoRA fine-tune on {prompt,chosen,rejected} data
|
|
91
|
-
mlx-bun get mlx-community/gemma-4-12B-it-OptiQ-4bit`;
|
|
92
|
-
|
|
93
|
-
const SERVER_FLAGS = `Server options:
|
|
94
|
-
--model <path|query> Explicit model override (mlx_lm.server spelling):
|
|
95
|
-
a directory containing config.json loads from that
|
|
96
|
-
path; anything else resolves like the positional
|
|
97
|
-
query. Wins over the positional and auto-pick.
|
|
98
|
-
--host <addr> Interface to bind [default: 127.0.0.1, loopback
|
|
99
|
-
only; use 0.0.0.0 to expose on your network]
|
|
100
|
-
--port <n> Listen port [default: 8080]
|
|
101
|
-
--memory-budget <GB> Admission-control memory budget; requests that
|
|
102
|
-
cannot fit are rejected instead of crashing the
|
|
103
|
-
GPU [default: machine RAM × 0.75, check-only]
|
|
104
|
-
--kv-budget <GB> Aggregate KV budget across concurrent batch rows
|
|
105
|
-
(decimal GB): joiners whose projected KV
|
|
106
|
-
(prompt + max_tokens) would exceed it queue until
|
|
107
|
-
rows finish; a request over the budget alone is
|
|
108
|
-
rejected. Off unless set.
|
|
109
|
-
--prompt-cache <GB> Prompt (KV) cache byte cap [default: 8 GB;
|
|
110
|
-
--prompt-cache 0 disables the cache]
|
|
111
|
-
--ssd-cache <dir> SSD cold tier for the prompt cache: prefix KV
|
|
112
|
-
spills to disk on eviction and persists across
|
|
113
|
-
restarts (long-context agent TTFT drops from a
|
|
114
|
-
full re-prefill to a zero-copy mmap restore).
|
|
115
|
-
Off unless set.
|
|
116
|
-
--ssd-cache-max <GB> Optional SSD tier byte cap; 0 = unlimited
|
|
117
|
-
[default: unlimited]
|
|
118
|
-
--ssd-demote-idle <sec> Demote prompt-cache entries idle this long to the
|
|
119
|
-
SSD tier, freeing their GPU memory (prefixes stay
|
|
120
|
-
reachable via zero-copy restore). 0 disables.
|
|
121
|
-
[default: 300 when --ssd-cache is on]
|
|
122
|
-
--ssd-cache-verify Verify tensor hashes on every restore (reads all
|
|
123
|
-
bytes eagerly — integrity paranoia only)
|
|
124
|
-
--generation-checkpoint <tokens>
|
|
125
|
-
Queue an in-flight generation snapshot to the SSD
|
|
126
|
-
cache every N emitted tokens. Repeating the same
|
|
127
|
-
request after restart replays and resumes it.
|
|
128
|
-
Requires --ssd-cache; supports ordinary requests
|
|
129
|
-
with shared cache layouts.
|
|
130
|
-
--isolate Run the inference engine as a CHILD process and
|
|
131
|
-
keep this process a pure UI/API proxy — the UI
|
|
132
|
-
stays instant under GPU load and survives engine
|
|
133
|
-
crashes (respawned). /ws/chat not proxied yet.
|
|
134
|
-
Requests naming another model (exact /v1/models
|
|
135
|
-
id) spawn/route to THAT model's engine child.
|
|
136
|
-
--model-pool <n> With --isolate: max RESIDENT model engines
|
|
137
|
-
[default: 1]. Over the cap, the least-recently-
|
|
138
|
-
used engine drains, demotes its caches to the
|
|
139
|
-
SSD tier, and exits — switching back respawns
|
|
140
|
-
it with all state restored from disk.
|
|
141
|
-
--unix <path> (internal) listen on a unix socket instead of a
|
|
142
|
-
TCP port — the engine half of --isolate.
|
|
143
|
-
--batch <n> Max concurrent requests decoded together
|
|
144
|
-
[default: 8]. A lone scheduler request uses the
|
|
145
|
-
parity- and latency-gated B=1 fast path; concurrent
|
|
146
|
-
requests can form B=N (agent fan-out). 1 pins the
|
|
147
|
-
preserved strict serial executor.
|
|
148
|
-
--decode-concurrency is accepted as the
|
|
149
|
-
mlx_lm.server alias (semantics differ; see
|
|
150
|
-
docs/reference/server-config.md)
|
|
151
|
-
--no-open Don't open the chat UI in your browser on start.
|
|
152
|
-
By default an interactive terminal opens
|
|
153
|
-
http://<host>:<port>/#/chat once the server is up.
|
|
154
|
-
--allow-private-media Let image_url/audio_url content parts fetch from
|
|
155
|
-
private/loopback/link-local hosts (NAS, another
|
|
156
|
-
LAN box). Blocked by default: a request's URL is
|
|
157
|
-
attacker-controlled input, and the default keeps
|
|
158
|
-
the server from being steered at cloud metadata
|
|
159
|
-
or LAN-internal services (SSRF). data: URLs are
|
|
160
|
-
always fine; the 10 s timeout and 64 MB response
|
|
161
|
-
cap apply either way.
|
|
162
|
-
|
|
163
|
-
Model & quality:
|
|
164
|
-
--adapter <dir> Mount a LoRA adapter at startup and use it as the
|
|
165
|
-
default for requests that don't select one (a
|
|
166
|
-
request's \`adapter\` field, incl. "none", wins).
|
|
167
|
-
--adapter-path is accepted as the mlx_lm.server
|
|
168
|
-
alias. Hot-swap via POST /v1/adapters unchanged.
|
|
169
|
-
--whisper-model <p|query> Whisper checkpoint for /v1/audio/transcriptions
|
|
170
|
-
[default: the first downloaded whisper model,
|
|
171
|
-
resolved on the first request]. Serving a whisper
|
|
172
|
-
model ALONE starts a transcription-only server.
|
|
173
|
-
--preload Transcription-only server: load the Whisper
|
|
174
|
-
weights before listening instead of on the first
|
|
175
|
-
request.
|
|
176
|
-
--whisper-idle-unload <s> Keep the Whisper weights for this many idle seconds
|
|
177
|
-
after a take before disposing them. [default: 0 —
|
|
178
|
-
released right after every take, so the chat model
|
|
179
|
-
has the memory back before it prefills; a cold
|
|
180
|
-
take costs ~150 ms more than a warm one]
|
|
181
|
-
--whisper-resident Never dispose the Whisper weights (sotto-style).
|
|
182
|
-
--draft-model <query> Speculative decoding: a drafter proposes tokens
|
|
183
|
-
the main model verifies (exact results, faster
|
|
184
|
-
decode when drafts land). The artifact's kind is
|
|
185
|
-
auto-detected — a full same-tokenizer model
|
|
186
|
-
(mlx_lm.server parity), a Gemma "-assistant"
|
|
187
|
-
KV-borrowing drafter, or a DSpark checkpoint.
|
|
188
|
-
Supported draft providers use shared batched
|
|
189
|
-
execution; ordinary requests can batch too.
|
|
190
|
-
--draft-kind <kind> Override draft-artifact detection:
|
|
191
|
-
two-model | assistant | dspark | deepspec | mtp | ngram
|
|
192
|
-
(deepspec = DeepSeek's released Gemma4DSparkModel
|
|
193
|
-
drafters; mtp = a NATIVE multi-token-prediction
|
|
194
|
-
head split from the target's own release, e.g.
|
|
195
|
-
mlx-community/Qwen3.8-27B-MTP-bf16 — auto-detected
|
|
196
|
-
from model_type *_mtp;
|
|
197
|
-
ngram = MODEL-FREE prompt lookup — no
|
|
198
|
-
--draft-model, drafts copied from the request's
|
|
199
|
-
own context, lossless) [default: auto]
|
|
200
|
-
--num-draft-tokens <n> Drafts per verify round [default: 3; ngram: 10;
|
|
201
|
-
DSpark pins to its trained block width]
|
|
202
|
-
--ngram-max <k> Prompt-lookup longest match window [default: 3]
|
|
203
|
-
--ngram-min <k> Prompt-lookup shortest match window [default: 1]
|
|
204
|
-
--mtp <on|off> GLM-5.2 checkpoint-native MTP. On by default;
|
|
205
|
-
routes requests through the exact serial verify
|
|
206
|
-
lane. off enables ordinary continuous batching.
|
|
207
|
-
--context-length <n> GLM-5.2 resource-plan context reservation
|
|
208
|
-
[default: 4096]
|
|
209
|
-
--kv-quant <mode> KV cache quantization: config (per-layer
|
|
210
|
-
kv_config.json when the model ships one), off
|
|
211
|
-
(bf16), 4 / 8 (uniform bits), or turbo[:k<bits>v
|
|
212
|
-
<bits>] (rotation-based TurboQuant — default
|
|
213
|
-
k8v3; kBits in {2,4,5,8}, vBits in {2,3,4,5,8};
|
|
214
|
-
docs/design/turboquant.md)
|
|
215
|
-
[default: off — quantized KV measured 5-20% slower
|
|
216
|
-
decode than bf16 at ≤16k (on mlx-lm too); it buys
|
|
217
|
-
memory headroom, so opt in when context is tight.
|
|
218
|
-
The batched engine (--batch N) is bf16-only: an
|
|
219
|
-
explicit --kv-quant routes those requests serial.
|
|
220
|
-
turbo is solo-only in v1 (full-attention layers
|
|
221
|
-
only; sliding-window layers stay bf16)]
|
|
222
|
-
--paged-kv OPTIONAL vLLM-style paged KV storage (block pool +
|
|
223
|
-
gather before the stock SDPA; env MLX_BUN_PAGED_KV=1).
|
|
224
|
-
Gemma4-family, bf16; supports --batch N.
|
|
225
|
-
Refuses --kv-quant and --draft-model; bypasses the prompt
|
|
226
|
-
cache; runs uncompiled decode. Bit-exact with the
|
|
227
|
-
plain path; expect a small decode cost at batch=1
|
|
228
|
-
(the gather copy). docs/design/kv-cache.md
|
|
229
|
-
[default: off]
|
|
230
|
-
--paged-kv-block-size <n> Tokens per KV block [default: 256 = the plain
|
|
231
|
-
cache's growth step]
|
|
232
|
-
--thinking <true|false> Default for the chat template's enable_thinking
|
|
233
|
-
variable (CPM and other hybrid-reasoning models);
|
|
234
|
-
a request's chat_template_kwargs overrides it
|
|
235
|
-
[default: the model's own, false for CPM]
|
|
236
|
-
--temperature <n> Server-wide sampling defaults; a per-request field
|
|
237
|
-
--top-p <n> still overrides them, and the browser chat (which
|
|
238
|
-
--top-k <n> sends none) inherits them. --temp is accepted as
|
|
239
|
-
an alias for --temperature (mlx_lm.server compat;
|
|
240
|
-
NOTE mlx_lm.server defaults temperature to 0.0 —
|
|
241
|
-
pass --temp 0 for its unset-request behavior)
|
|
242
|
-
[default: the model's generation_config.json]
|
|
243
|
-
--max-tokens <n> Completion cap when a request omits max_tokens
|
|
244
|
-
(mlx_lm.server flag; --max-tokens 512 reproduces
|
|
245
|
-
its default) [default: 128 GLM-5.2 (memory-plan
|
|
246
|
-
reservation); otherwise none — a defaulted
|
|
247
|
-
request runs to EOS or the admitted context]
|
|
248
|
-
--hlg-sampling on|off Piecewise tone-curve sampling (HLG): rolls off the
|
|
249
|
-
top, boosts the mids, gentles the tail. Gain folds
|
|
250
|
-
from --temperature. [default: off]
|
|
251
|
-
See docs/archive/hlg-sampling.md
|
|
252
|
-
--hlg-width <nats> HLG mid-region half-width [default: 4]
|
|
253
|
-
--hlg-shoulder <nats> HLG highlight rolloff scale [default: 4]
|
|
254
|
-
--hlg-toe <nats> HLG shadow rolloff scale [default: 6]
|
|
255
|
-
--hlg-pivot-offset <nats> HLG pivot: nats below the top token [default: 6]
|
|
256
|
-
|
|
257
|
-
Parity tier (an ALIAS for the per-fork flags below; each expands to the fastest
|
|
258
|
-
set of forks that still holds the guarantee — a per-fork flag overrides one):
|
|
259
|
-
--l1 Bit-for-bit IDENTICAL to mlx-lm (bf16 KV, compiled
|
|
260
|
-
activations — the faithful @mx.compile kernels).
|
|
261
|
-
No tier given = --l1 (decided 2026-07-05: the L1
|
|
262
|
-
faithful kernels match mlx-lm 1.00x on every model;
|
|
263
|
-
anything output-changing must beat that baseline in
|
|
264
|
-
a paired A/B to earn a default).
|
|
265
|
-
--l2 Bit-for-bit IDENTICAL to mlx-optiq (quantized KV +
|
|
266
|
-
fused prefill SDPA, stock unfused decode — the
|
|
267
|
-
optiq-golden composition)
|
|
268
|
-
|
|
269
|
-
Kill switches (bit-exact A/B levers — each selects a slower same-parity path):
|
|
270
|
-
--compiled-decode on|off Compiled decode graphs [default: on]
|
|
271
|
-
--compiled-activations on|off
|
|
272
|
-
Compiled geglu/swiglu (mlx-lm's @mx.compile kernel —
|
|
273
|
-
bit-exact AND fewer dispatches). off = the uncompiled
|
|
274
|
-
composition (same parity, slower) [default: on]
|
|
275
|
-
--fused-sdpa on|off Fused SDPA path (quantized KV only) [default:
|
|
276
|
-
follows --kv-quant — on for config (the optiq
|
|
277
|
-
composition), off for uniform 4/8 (the mlx-lm
|
|
278
|
-
composition) and bf16]
|
|
279
|
-
--force-wire Wire weights into memory at load
|
|
280
|
-
--expert-offload MoE only: serve experts from a page-aligned file
|
|
281
|
-
mmap (built on first use) — keeps the model out of
|
|
282
|
-
memory pressure (phys_footprint ≈ active params),
|
|
283
|
-
bit-exact with the resident path`;
|
|
284
|
-
|
|
285
|
-
const HELP: Record<string, string> = {
|
|
286
|
-
dictate: `mlx-bun dictate — push-to-talk dictation from your microphone
|
|
287
|
-
|
|
288
|
-
Usage: mlx-bun dictate [query] [options]
|
|
289
|
-
|
|
290
|
-
Captures the mic through the AVAudioEngine sidecar (16 kHz mono; macOS asks
|
|
291
|
-
for Microphone permission on first use) and streams it into a Whisper
|
|
292
|
-
transcription session while you speak — every finished 30 s window is
|
|
293
|
-
transcribed during capture, so the result lands about one window after you
|
|
294
|
-
stop. Silence never runs Whisper (Silero VAD).
|
|
295
|
-
|
|
296
|
-
Modes:
|
|
297
|
-
(default) Press Enter to start a take, Enter again to stop;
|
|
298
|
-
q + Enter or Ctrl-C quits
|
|
299
|
-
--hotkey [keycode] Hold a key instead (default 61 = Right Option, like
|
|
300
|
-
sotto); needs Input Monitoring for your terminal
|
|
301
|
-
Backend:
|
|
302
|
-
[query] / --model <p> Whisper model for in-process transcription (default:
|
|
303
|
-
the first downloaded whisper checkpoint)
|
|
304
|
-
--server <url> Use a running mlx-bun server's /v1/audio/sessions
|
|
305
|
-
instead of loading the model here
|
|
306
|
-
--idle-unload <s> Release the weights after this many idle seconds
|
|
307
|
-
(0 = right after each take; default 30); --resident
|
|
308
|
-
keeps them loaded
|
|
309
|
-
Decoding:
|
|
310
|
-
--language <code> default en; "auto" detects
|
|
311
|
-
--beam-size <n> beam search (default greedy)
|
|
312
|
-
--prompt "…" initial prompt / --vocabulary "a,b,c" hint terms
|
|
313
|
-
--no-vad skip the Silero gate
|
|
314
|
-
Output (any combination; default prints):
|
|
315
|
-
--copy copy the transcript to the clipboard (pbcopy)
|
|
316
|
-
--type type it into the frontmost app via System Events
|
|
317
|
-
keystrokes (needs Accessibility for your terminal);
|
|
318
|
-
--type-delay <s> waits first so you can Cmd-Tab
|
|
319
|
-
(default 1 in Enter mode, 0 with --hotkey)`,
|
|
320
|
-
transcribe: `mlx-bun transcribe — speech-to-text with a local Whisper model (no server)
|
|
321
|
-
|
|
322
|
-
Usage: mlx-bun transcribe <audio-file> [query] [options]
|
|
323
|
-
|
|
324
|
-
Loads a Whisper checkpoint (mlx-community/whisper-*) and prints the
|
|
325
|
-
transcript. WAV is decoded natively; any other container CoreAudio reads
|
|
326
|
-
(mp3/m4a/flac/ogg/aiff/…) is transcoded with afconvert first.
|
|
327
|
-
|
|
328
|
-
Model:
|
|
329
|
-
[query] / --model <p> Whisper model; omitted = the first downloaded
|
|
330
|
-
\`whisper\` checkpoint (errors with an mlx-bun get
|
|
331
|
-
suggestion when none)
|
|
332
|
-
Decoding:
|
|
333
|
-
--language <code> ISO code or name ("en", "japanese"); "auto" or omitted
|
|
334
|
-
detects the language from the first 30 s
|
|
335
|
-
--task translate Translate to English instead of transcribing
|
|
336
|
-
--beam-size <n> Beam search width (default: greedy)
|
|
337
|
-
--temperature <t> Sampling temperature; the default runs the
|
|
338
|
-
(0, 0.2, …, 1.0) fallback ladder like mlx-whisper
|
|
339
|
-
--no-fallback Temperature 0 only, no retry ladder
|
|
340
|
-
--prompt "…" Initial prompt (vocabulary hints / style)
|
|
341
|
-
--no-timestamps Decode text only (<|notimestamps|>)
|
|
342
|
-
--no-condition Do not condition each window on the previous text
|
|
343
|
-
--vad Silero VAD gate: skip Whisper when no speech is
|
|
344
|
-
detected (weights: ggml-org/whisper-vad in the HF cache
|
|
345
|
-
or --vad-model <path>); --vad-trim crops to speech
|
|
346
|
-
--word-timestamps Word-level timestamps (verbose_json / verbose output)
|
|
347
|
-
--faithful Run the oracle-parity graph instead of the fast path
|
|
348
|
-
--audio-ctx <n> Lab: encode only n of the 1500 encoder positions
|
|
349
|
-
(whisper.cpp -ac); cheaper for short clips, degrades
|
|
350
|
-
below ~1024 — off by default
|
|
351
|
-
Output:
|
|
352
|
-
--format <f> text (default) · json · verbose_json · srt · vtt
|
|
353
|
-
--verbose Print each segment with its time range as it decodes`,
|
|
354
|
-
pi: `mlx-bun pi — drop into a coding-agent session on a local model
|
|
355
|
-
|
|
356
|
-
Usage: mlx-bun pi [options] [message...]
|
|
357
|
-
|
|
358
|
-
Runs the pi coding agent's own terminal UI in-process against the local
|
|
359
|
-
model — nothing to install, pi is built in. Reuses a healthy local
|
|
360
|
-
server when one is running; otherwise serves e4b by default (override
|
|
361
|
-
with --query) and starts a server for the session (the server ends with
|
|
362
|
-
the session). Fresh install: downloads a small starter model first so
|
|
363
|
-
you're chatting in minutes, then streams e4b in the background — it
|
|
364
|
-
becomes the default on the next run.
|
|
365
|
-
|
|
366
|
-
Model selection:
|
|
367
|
-
-q, --query <q> Model to serve when starting a server (registry query)
|
|
368
|
-
|
|
369
|
-
${SERVER_FLAGS}
|
|
370
|
-
|
|
371
|
-
Run modes:
|
|
372
|
-
message... Pre-fill the first turn (interactive)
|
|
373
|
-
-p, --print [msg] One-shot: print the reply and exit (also reads piped
|
|
374
|
-
stdin); add --json for the full event stream
|
|
375
|
-
--mode json|rpc Structured print (json) or JSONL RPC over stdio
|
|
376
|
-
|
|
377
|
-
Already use pi? Connect your own pi to this local model instead:
|
|
378
|
-
mlx-bun harness pi`,
|
|
379
|
-
|
|
380
|
-
serve: `mlx-bun serve — OpenAI/Anthropic-compatible server for a local model
|
|
381
|
-
|
|
382
|
-
Usage: mlx-bun serve [query] [options]
|
|
383
|
-
|
|
384
|
-
[query] Registry query (e.g. "12B", "e4b", a repo substring).
|
|
385
|
-
--query <q> is accepted for the positional. Omitted:
|
|
386
|
-
serves e4b (the default), else the largest downloaded
|
|
387
|
-
model that fits. Fresh install: downloads a small
|
|
388
|
-
starter model first, then e4b in the background
|
|
389
|
-
(default on next run).
|
|
390
|
-
|
|
391
|
-
${SERVER_FLAGS}
|
|
392
|
-
|
|
393
|
-
Endpoints: /v1/chat/completions, /v1/completions, /v1/messages, /v1/responses,
|
|
394
|
-
/v1/models, /v1/adapters, /health, /stats, /fit, /library, /downloads —
|
|
395
|
-
web app at / (lands on chat) — /chat, /status, /quantize, /finetune, /dataset deep-link its views`,
|
|
396
|
-
|
|
397
|
-
get: `mlx-bun get — download a model from Hugging Face
|
|
398
|
-
|
|
399
|
-
Usage: mlx-bun get <org/repo | substring> [options]
|
|
400
|
-
|
|
401
|
-
<org/repo> Full repo id (e.g. mlx-community/gemma-4-12B-it-OptiQ-4bit)
|
|
402
|
-
<substring> No "/" = a registry query over already-downloaded repos
|
|
403
|
-
(e.g. \`mlx-bun get 12B\` re-gets/refreshes the match)
|
|
404
|
-
|
|
405
|
-
Options:
|
|
406
|
-
--revision <rev> Git revision [default: main]
|
|
407
|
-
|
|
408
|
-
Resumable (Range requests against partial blobs) and verified (sha256
|
|
409
|
-
for LFS blobs). Re-running after an interruption continues where it
|
|
410
|
-
stopped. Before payload transfer, complete blobs and partial prefixes are
|
|
411
|
-
credited and the target volume must fit the remainder plus a 1 GiB reserve.
|
|
412
|
-
Uses HF_TOKEN / hf auth login credentials when present.
|
|
413
|
-
When upstream pushed a new revision, the previous snapshot stays on
|
|
414
|
-
disk — \`mlx-bun gc\` reclaims it.`,
|
|
415
|
-
|
|
416
|
-
ls: `mlx-bun ls — list downloaded models
|
|
417
|
-
|
|
418
|
-
Usage: mlx-bun ls [query] [options]
|
|
419
|
-
|
|
420
|
-
One row per repo (the canonical revision: refs/main). The HF cache keeps
|
|
421
|
-
a snapshot dir per downloaded revision — superseded ones are hidden here
|
|
422
|
-
and reclaimed by \`mlx-bun gc\`.
|
|
423
|
-
|
|
424
|
-
Options:
|
|
425
|
-
--vision Only vision-capable models (SigLIP sidecar or the
|
|
426
|
-
unified encoder-free tower)
|
|
427
|
-
--max-size <size> Filter by weight size (e.g. 10GB, 800MB)
|
|
428
|
-
--all-revisions One row per cached snapshot (canonical marked *)`,
|
|
429
|
-
|
|
430
|
-
gc: `mlx-bun gc — reclaim superseded snapshots + dead blobs
|
|
431
|
-
|
|
432
|
-
Usage: mlx-bun gc [options]
|
|
433
|
-
|
|
434
|
-
Every \`get\` that follows an upstream push creates a NEW
|
|
435
|
-
snapshots/<commit> dir; the old one (and any blobs only it references)
|
|
436
|
-
stays on disk forever. gc keeps the snapshots refs/* point at, deletes
|
|
437
|
-
the rest, then deletes blobs no surviving snapshot links to.
|
|
438
|
-
|
|
439
|
-
Without --yes it only prints what would be deleted (per-repo reclaim).
|
|
440
|
-
|
|
441
|
-
Options:
|
|
442
|
-
--yes Actually delete (destructive; default prints only)
|
|
443
|
-
--dry-run Print the plan and never delete, even with --yes
|
|
444
|
-
--force Also prune superseded snapshots that contain files
|
|
445
|
-
the canonical revision lacks (normally skipped with
|
|
446
|
-
a warning — deleting them loses the only copy)`,
|
|
447
|
-
|
|
448
|
-
fit: `mlx-bun fit — will this model fit on this machine?
|
|
449
|
-
|
|
450
|
-
Usage: mlx-bun fit <query> [options]
|
|
451
|
-
|
|
452
|
-
Options:
|
|
453
|
-
--ctx <tokens> Context size to assess [default: 8192]
|
|
454
|
-
--kv-quant <mode> Bill the KV cache quantized: 4 | 8 | config | off
|
|
455
|
-
(mirrors serve --kv-quant; default bf16)
|
|
456
|
-
--skus Also print the Apple silicon SKU matrix
|
|
457
|
-
|
|
458
|
-
Solves weights + KV growth + prefill transient against wired memory and
|
|
459
|
-
predicts decode speed from memory bandwidth.`,
|
|
460
|
-
|
|
461
|
-
scan: `mlx-bun scan — re-index the Hugging Face cache
|
|
462
|
-
|
|
463
|
-
Usage: mlx-bun scan
|
|
464
|
-
|
|
465
|
-
Reads config.json + safetensors headers (never tensor bytes) for every
|
|
466
|
-
snapshot in ~/.cache/huggingface/hub and refreshes the model registry.`,
|
|
467
|
-
|
|
468
|
-
harness: `mlx-bun harness pi — connect your own pi to the local model
|
|
469
|
-
|
|
470
|
-
Usage: mlx-bun harness pi [options]
|
|
471
|
-
|
|
472
|
-
For people who already use pi: this points your existing pi install at
|
|
473
|
-
the local mlx-bun server so you can run pi the way you already do, on a
|
|
474
|
-
local model. (To just chat now with no setup, use \`mlx-bun pi\` — the
|
|
475
|
-
built-in agent.)
|
|
476
|
-
|
|
477
|
-
Options:
|
|
478
|
-
--base-url <url> Server base URL [default: http://127.0.0.1:8080/v1]
|
|
479
|
-
--remove Disconnect (delete the extension)
|
|
480
|
-
|
|
481
|
-
Installs a small discovery extension into ~/.pi/agent/extensions that
|
|
482
|
-
registers mlx-bun as a pi provider (models discovered live from
|
|
483
|
-
/v1/models at pi startup). One file, reversible with --remove; never
|
|
484
|
-
touches your existing pi config. Then: pi --provider mlx-bun`,
|
|
485
|
-
|
|
486
|
-
benchmark: `mlx-bun benchmark — measure OUR stack on this machine
|
|
487
|
-
|
|
488
|
-
Usage: mlx-bun benchmark [query] [options]
|
|
489
|
-
|
|
490
|
-
Runs mlx-bun by itself (no other systems required) and records the
|
|
491
|
-
result to the eval DB. For quotable, cross-stack numbers use
|
|
492
|
-
bun scripts/bench-serve.ts all from the repo — it preflight-gates on an idle machine
|
|
493
|
-
and runs the mlx-lm/optiq comparison legs.
|
|
494
|
-
|
|
495
|
-
[query] Model to benchmark (--query <q> is accepted for the
|
|
496
|
-
positional) [default: auto-pick]
|
|
497
|
-
|
|
498
|
-
Options:
|
|
499
|
-
--model <path|query> Explicit model override: a directory containing
|
|
500
|
-
config.json loads from that path; anything else
|
|
501
|
-
resolves like the positional query. Wins over both.
|
|
502
|
-
--tokens <n> Tokens to decode per run [default: 256]
|
|
503
|
-
--runs <n> Runs (median reported) [default: 3]
|
|
504
|
-
--prompt-tokens <n> Pad the prompt to ~n tokens (long-context decode)
|
|
505
|
-
--kv-quant <mode> config | off | 4 | 8 | turbo[:k<bits>v<bits>]
|
|
506
|
-
[default: off, the historical baseline]
|
|
507
|
-
|
|
508
|
-
Kill switches (--compiled-decode, --compiled-activations, --fused-sdpa)
|
|
509
|
-
apply to the run — A/B by running twice.`,
|
|
510
|
-
|
|
511
|
-
embed: `mlx-bun embed — text embeddings from a local embedding model
|
|
512
|
-
|
|
513
|
-
Usage: mlx-bun embed [query] --text "…" [options]
|
|
514
|
-
mlx-bun embed [query] "…" (text as a positional)
|
|
515
|
-
echo -e "line one\\nline two" | mlx-bun embed [query]
|
|
516
|
-
|
|
517
|
-
Embeds text with a local Qwen3-Embedding model (last-token pooled,
|
|
518
|
-
L2-normalized — bit-exact vs mlx-lm). One text per --text, or one per
|
|
519
|
-
line on stdin. Prints one JSON array (the vector) per input line.
|
|
520
|
-
|
|
521
|
-
[query] Embedding model (--query <q> is accepted for the
|
|
522
|
-
positional) [default: the first downloaded
|
|
523
|
-
embedding model; errors with an mlx-bun get
|
|
524
|
-
suggestion when none is downloaded]
|
|
525
|
-
|
|
526
|
-
Options:
|
|
527
|
-
--text "…" Text to embed (omit to read lines from stdin)
|
|
528
|
-
--instruct "…" Query instruction (Qwen3-Embedding query format);
|
|
529
|
-
steers WHICH similarity axis the geometry reflects.
|
|
530
|
-
Omit for plain document embeddings.
|
|
531
|
-
--json Emit one OpenAI-style {object:"list",data:[…]} object
|
|
532
|
-
instead of bare vector arrays
|
|
533
|
-
|
|
534
|
-
For a server endpoint use \`mlx-bun serve <embedding-model>\` then
|
|
535
|
-
POST /v1/embeddings.`,
|
|
536
|
-
|
|
537
|
-
generate: `mlx-bun generate — one-shot raw text generation (no server)
|
|
538
|
-
|
|
539
|
-
Usage: mlx-bun generate [query] --prompt "…" [options]
|
|
540
|
-
mlx-bun generate [query] "…" (prompt as a positional)
|
|
541
|
-
|
|
542
|
-
A direct one-shot entry point: load the model, generate, print, exit —
|
|
543
|
-
no server, no chat UI. By default the prompt is rendered through the
|
|
544
|
-
model's chat template; --raw feeds it verbatim.
|
|
545
|
-
|
|
546
|
-
[query] Model (registry query; --query is accepted for the
|
|
547
|
-
positional) [default: auto-pick]
|
|
548
|
-
|
|
549
|
-
Options:
|
|
550
|
-
--prompt "…" The prompt (or pass it as the positional)
|
|
551
|
-
--raw Skip the chat template (verbatim prompt)
|
|
552
|
-
--max-tokens <n> Completion cap [default: 256]
|
|
553
|
-
--temperature <n> Sampling temperature (--temp is accepted as an alias)
|
|
554
|
-
--top-p <n> Nucleus sampling
|
|
555
|
-
--top-k <n> Top-k sampling
|
|
556
|
-
--seed <n> Sampler seed (reproducible runs)
|
|
557
|
-
|
|
558
|
-
Decode-path levers mirror serve: --l1/--l2 and the per-fork flags
|
|
559
|
-
(--fused-sdpa/--compiled-decode/--compiled-activations/--kv-quant).
|
|
560
|
-
The tier's KV scheme applies here too (--l2 = the model's kv_config,
|
|
561
|
-
same tokens as serve); with no tier/--kv-quant the KV cache stays bf16
|
|
562
|
-
(the bit-exact mlx-lm greedy path). mlx-lm compat = --l1.
|
|
563
|
-
|
|
564
|
-
Aliases: mlx-bun gen`,
|
|
565
|
-
|
|
566
|
-
evals: `mlx-bun evals — recent benchmark runs
|
|
567
|
-
|
|
568
|
-
Usage: mlx-bun evals [options]
|
|
569
|
-
|
|
570
|
-
Options:
|
|
571
|
-
--limit <n> Rows to show [default: 20]
|
|
572
|
-
--raw Full records as JSON lines (for scripts/jq)
|
|
573
|
-
--clear Archive the eval DB (timestamped backup) and start fresh
|
|
574
|
-
|
|
575
|
-
Table view: when, model, bench kind, KV mode, decode tok/s, TTFT,
|
|
576
|
-
peak memory, commit. Runs are written by bun scripts/bench-serve.ts all.`,
|
|
577
|
-
|
|
578
|
-
train: `mlx-bun train — fine-tune a LoRA adapter on your data
|
|
579
|
-
|
|
580
|
-
Usage: mlx-bun train <model> --data <dir> [options]
|
|
581
|
-
|
|
582
|
-
<model> Model to fine-tune (query or snapshot path; auto-picks
|
|
583
|
-
the default model if omitted)
|
|
584
|
-
|
|
585
|
-
Data: <dir> must contain train.jsonl (+ optional valid.jsonl). Rows are
|
|
586
|
-
{prompt, chosen, rejected} for dpo/orpo, or {messages|text} for sft.
|
|
587
|
-
|
|
588
|
-
Options:
|
|
589
|
-
--data <dir> Dataset dir with train.jsonl (required)
|
|
590
|
-
--method <m> sft | dpo | orpo [default: orpo]
|
|
591
|
-
--adapter <dir> Output adapter dir
|
|
592
|
-
[default: ~/.cache/mlx-bun/mlx-bun-finetunes/<method>-<model>]
|
|
593
|
-
--iters <n> Training iterations [default: 100]
|
|
594
|
-
--lr <f> Learning rate [default: orpo 1e-5 · dpo 5e-5 · sft 2e-4]
|
|
595
|
-
--rank <n> LoRA rank [default: orpo 16 · else 8]
|
|
596
|
-
--scale <f> LoRA scale [default: orpo 2.0 · else 1.0]
|
|
597
|
-
--seq <n> Max sequence length [default: gemma 8192 · else 4096]
|
|
598
|
-
--batch <n> Batch size [default: 1]
|
|
599
|
-
--grad-accum <n> Gradient accumulation steps (effective batch =
|
|
600
|
-
batch × grad-accum at batch-size-1 memory) [default: 1]
|
|
601
|
-
--grad-clip <f> Gradient-norm clip (0 = off) [default: 1.0]
|
|
602
|
-
--seed <n> Data-shuffle / init seed [default: 0]
|
|
603
|
-
--val-size <n> Max validation examples per eval [default: 256]
|
|
604
|
-
--lambda <f> ORPO odds-ratio weight [default: 0.1]
|
|
605
|
-
--sft-scope <s> ORPO chosen-NLL scope: full (paper/TRL-faithful,
|
|
606
|
-
prompt+response) | response (pre-2026-07 runs,
|
|
607
|
-
bit-exact) [default: full]
|
|
608
|
-
--seg <n> Layers per segment (segmented backward; orpo default 2)
|
|
609
|
-
--save-every <n> Crash-safe mountable checkpoint every n steps
|
|
610
|
-
--resume <dir> Warm-start LoRA weights from a checkpoint/adapter dir
|
|
611
|
-
--no-flash Disable the flash-CCE Metal head (use the MLX fused head)
|
|
612
|
-
--no-prefix Disable prefix-sharing (two-forward branches)
|
|
613
|
-
--no-segment Disable the segmented backward (hold all activations)
|
|
614
|
-
--dry-run Inspect the dataset + print the resolved plan, don't train
|
|
615
|
-
|
|
616
|
-
ORPO defaults run the full stack: the [M,vocab]-free flash-CCE head +
|
|
617
|
-
prefix-sharing + segmented backward (each falls back + logs if a row's
|
|
618
|
-
preconditions aren't met). Gemma/e4b sets its required training env flags
|
|
619
|
-
automatically. Adapters are mountable directly (mlx-bun serve --adapter).
|
|
620
|
-
Run long jobs detached from your own shell: nohup mlx-bun train … &
|
|
621
|
-
Watch a run live from another tab: mlx-bun train-watch <adapter-dir>`,
|
|
622
|
-
|
|
623
|
-
"train-watch": `mlx-bun train-watch — live dashboard for a training run
|
|
624
|
-
|
|
625
|
-
Usage: mlx-bun train-watch [adapter-dir]
|
|
626
|
-
|
|
627
|
-
Tails <adapter-dir>/metrics.jsonl (written by mlx-bun train and the web
|
|
628
|
-
fine-tune tile) and renders a live loss/speed/memory dashboard. Point it
|
|
629
|
-
at the same --adapter directory the training run uses.
|
|
630
|
-
|
|
631
|
-
[adapter-dir] Adapter directory to watch; --adapter <dir> is
|
|
632
|
-
accepted for the positional
|
|
633
|
-
[default: ~/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5]`,
|
|
634
|
-
fuse: `mlx-bun fuse — merge a LoRA adapter into the base weights
|
|
635
|
-
|
|
636
|
-
Usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [options]
|
|
637
|
-
|
|
638
|
-
Folds the adapter's low-rank deltas into the base weights and writes a
|
|
639
|
-
standalone, loadable model snapshot (weights + config + tokenizer). A
|
|
640
|
-
quantized base keeps its exact quantization layout: fused modules are
|
|
641
|
-
dequantized, updated, and re-quantized with their own per-module spec.
|
|
642
|
-
|
|
643
|
-
<model> Base model (registry query or a snapshot path);
|
|
644
|
-
--model is accepted as the mlx_lm.fuse spelling
|
|
645
|
-
|
|
646
|
-
Options:
|
|
647
|
-
--adapter <dir> Adapter directory (adapters.safetensors +
|
|
648
|
-
adapter_config.json). --adapter-path is accepted
|
|
649
|
-
as the mlx_lm.fuse alias [default: adapters]
|
|
650
|
-
--save-path <dir> Output model directory [default: fused_model]
|
|
651
|
-
|
|
652
|
-
Not supported (mlx_lm.fuse flags we don't implement — the command exits
|
|
653
|
-
with an error rather than guessing): --de-quantize, --export-gguf,
|
|
654
|
-
--gguf-path, --upload-repo.
|
|
655
|
-
|
|
656
|
-
Serve the result: mlx-bun serve <save-path>`,
|
|
657
|
-
|
|
658
|
-
convert: `mlx-bun convert — quantize an HF model into a local MLX snapshot
|
|
659
|
-
|
|
660
|
-
Usage: mlx-bun convert --hf-path <repo-or-path> -q [options]
|
|
661
|
-
mlx-bun convert <repo-or-path> -q [options] (positional source)
|
|
662
|
-
|
|
663
|
-
Wraps mlx-bun's native quantize pipeline (the same engine as the web
|
|
664
|
-
/api/quantize). The source may be a local model directory, a downloaded
|
|
665
|
-
model (registry query), or a Hugging Face repo id — an un-downloaded
|
|
666
|
-
repo is fetched first (resumable, verified; can be many GB).
|
|
667
|
-
|
|
668
|
-
Options:
|
|
669
|
-
--hf-path <src> Source model: local path, downloaded model, or HF
|
|
670
|
-
repo id (--model and the bare positional are
|
|
671
|
-
accepted as aliases)
|
|
672
|
-
--mlx-path <dir> Output directory [default: mlx_model; must not
|
|
673
|
-
already exist, matching mlx_lm.convert]
|
|
674
|
-
-q, --quantize Quantize the model (uniform affine)
|
|
675
|
-
--q-bits <n> Bits per weight: 4 or 8 [default: 4]
|
|
676
|
-
--q-group-size <n> Quantization group size: 32 or 64 [default: 64]
|
|
677
|
-
--upload-repo <id> Push the converted model to this Hugging Face repo
|
|
678
|
-
after converting (mlx_lm.convert parity; same engine
|
|
679
|
-
as \`mlx-bun upload\`, needs a write token)
|
|
680
|
-
|
|
681
|
-
Mixed precision (the mlx-bun differentiator — OptiQ sensitivity sweep +
|
|
682
|
-
knapsack per-layer bit allocation; implies quantization, no -q needed):
|
|
683
|
-
--target-bpw <f> Target bits-per-weight (e.g. 4.5). Runs
|
|
684
|
-
calibration → per-layer KL sensitivity → greedy
|
|
685
|
-
knapsack; writes per-module bits into config.json
|
|
686
|
-
--candidate-bits <l> Comma list the knapsack may pick from [default: 4,8]
|
|
687
|
-
--calibration-mix <m> "optiq" or a JSONL path [default: optiq]
|
|
688
|
-
--n-calibration <n> Calibration samples [default: 2]
|
|
689
|
-
--rotate-weights Fold the model's offline TurboQuant rotation before
|
|
690
|
-
quantization (auto-detects Llama/Qwen3.5/Qwen MTP)
|
|
691
|
-
--rotation-seed <n> Deterministic rotation seed [default: 42]
|
|
692
|
-
|
|
693
|
-
Not supported (mlx_lm.convert flags we don't implement — the command
|
|
694
|
-
exits with an error rather than guessing): --dtype, -d/--dequantize,
|
|
695
|
-
--q-mode other than affine, and the mlx-lm --quant-predicate recipes
|
|
696
|
-
(use --target-bpw instead). Plain non-quantizing conversion is also not
|
|
697
|
-
supported: pass -q or --target-bpw.`,
|
|
698
|
-
|
|
699
|
-
upload: `mlx-bun upload — push a local model directory to the Hugging Face Hub
|
|
700
|
-
|
|
701
|
-
Usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [options]
|
|
702
|
-
|
|
703
|
-
Native push-to-hub, no Python (src/hf-push.ts — the same engine as the
|
|
704
|
-
web UI's push buttons): creates the repo if needed (idempotent), uploads
|
|
705
|
-
weights via the git-LFS batch protocol, and commits. Flag names match
|
|
706
|
-
mlx_lm.upload.
|
|
707
|
-
|
|
708
|
-
Options:
|
|
709
|
-
--path <dir> Local model directory to upload [default: mlx_model]
|
|
710
|
-
--upload-repo <id> Hub repo id, org/name or bare name (required)
|
|
711
|
-
--private Create the repo as private (mlx-bun extension)
|
|
712
|
-
|
|
713
|
-
Auth: needs a Hugging Face WRITE token, resolved from ~/.mlx-bun/hf.json
|
|
714
|
-
(web UI Settings → Hugging Face), $HF_TOKEN, or ~/.cache/huggingface/token
|
|
715
|
-
(hf auth login). Errors before uploading anything if none is found.
|
|
716
|
-
|
|
717
|
-
Convert + push in one step: mlx-bun convert … --upload-repo <org/repo>`,
|
|
718
|
-
|
|
719
|
-
perplexity: `mlx-bun perplexity — perplexity over a local text/JSONL dataset
|
|
720
|
-
|
|
721
|
-
Usage: mlx-bun perplexity <model-query-or-path> --data-path <file> [options]
|
|
722
|
-
|
|
723
|
-
mlx_lm.perplexity methodology, exactly: samples are visited in a seeded
|
|
724
|
-
random order, tokenized, concatenated, and cut into NON-OVERLAPPING rows
|
|
725
|
-
of --sequence-length tokens; per batch the model scores rows[:, :-1]
|
|
726
|
-
against rows[:, 1:] in f32 (every position counts); reported as
|
|
727
|
-
ppl = exp(mean CE) ± the delta-method standard error. The one deliberate
|
|
728
|
-
difference: the data source is a LOCAL file (never a Hugging Face
|
|
729
|
-
dataset download).
|
|
730
|
-
|
|
731
|
-
<model> Model (registry query or snapshot path);
|
|
732
|
-
--model (mlx_lm spelling) and --query are accepted
|
|
733
|
-
for the positional
|
|
734
|
-
|
|
735
|
-
Options:
|
|
736
|
-
--data-path <file> .jsonl ({"text": …} rows) or plain .txt (required)
|
|
737
|
-
--sequence-length <n> Tokens per row [default: 512]
|
|
738
|
-
--num-samples <n> Rows to score (-1 = all available) [default: 256]
|
|
739
|
-
--batch-size <n> Rows per forward [default: 8]
|
|
740
|
-
--seed <n> Sample-shuffle seed [default: 123]`,
|
|
741
|
-
|
|
742
|
-
memory: `mlx-bun memory — your local AI's personal wiki
|
|
743
|
-
|
|
744
|
-
Usage: mlx-bun memory <subcommand>
|
|
745
|
-
|
|
746
|
-
A local, durable memory for the assistant: a wiki of Markdown articles
|
|
747
|
-
(~/.mlx-bun/wiki) it reads to remember your projects, people, and history
|
|
748
|
-
across sessions. It's yours — git-tracked, editable in any tool (Obsidian
|
|
749
|
-
opens it as a vault), and it never leaves the machine. A read-only Reference/
|
|
750
|
-
folder symlinks mlx-bun's own docs into memory on day one. Once set up, it
|
|
751
|
-
loads automatically into every \`mlx-bun pi\` and \`mlx-bun serve\` session.
|
|
752
|
-
|
|
753
|
-
Subcommands:
|
|
754
|
-
init, setup Create the wiki + walk through setup (idempotent);
|
|
755
|
-
offers to import an existing vault and install the
|
|
756
|
-
nightly synthesis job
|
|
757
|
-
status Path, article count, git + schedule state
|
|
758
|
-
open, browse [article]
|
|
759
|
-
Open the wiki, or a specific article, in Obsidian
|
|
760
|
-
(falls back to Finder / the default Markdown app)
|
|
761
|
-
list List article titles + read-only Reference docs
|
|
762
|
-
search <query> Search articles from the terminal
|
|
763
|
-
toc <article> Print an article's headings + anchors
|
|
764
|
-
section <article> <anchor>
|
|
765
|
-
Print one article section
|
|
766
|
-
links <article> Show resolved outbound + inbound wikilinks
|
|
767
|
-
read <article> Print an article (stem, e.g. Archie_Project)
|
|
768
|
-
synthesize Run the FULL synthesis DAG now (--since, --model,
|
|
769
|
-
--dry-run); also: pipeline, all
|
|
770
|
-
segment | extract | route | synthesize-stage
|
|
771
|
-
Run ONE decomposed stage worker (--limit N —
|
|
772
|
-
segment/extract/synthesize-stage only; --convs a,b).
|
|
773
|
-
Each pulls its eligible work from the DB by state, walks
|
|
774
|
-
oldest-conversation-first, persists, and exits — resumable,
|
|
775
|
-
and runnable as separate concurrent processes on slices.
|
|
776
|
-
link Deterministic cross-linking stage: inline-link first
|
|
777
|
-
mentions + rebuild ## See also (--limit N; no model)
|
|
778
|
-
schedule Install the nightly launchd job (--at HH:MM [03:00])
|
|
779
|
-
unschedule Remove the nightly launchd job
|
|
780
|
-
|
|
781
|
-
The read path is live: the assistant reads a wiki you set up by hand or
|
|
782
|
-
import during \`memory init\`. Ask the assistant to "open my memory" or run
|
|
783
|
-
\`mlx-bun memory open\` to browse it in Obsidian/Finder, or
|
|
784
|
-
\`mlx-bun memory open <article>\` to jump to a specific page. Synthesis
|
|
785
|
-
(conversations → articles) runs the full local pipeline via
|
|
786
|
-
\`mlx-bun memory synthesize\` (or per-stage: segment/extract/route/
|
|
787
|
-
synthesize-stage/link); the nightly job runs it on a schedule.`,
|
|
788
|
-
};
|
|
789
|
-
|
|
790
|
-
HELP.bench = HELP.benchmark!;
|
|
791
|
-
HELP.gen = HELP.generate!;
|
|
792
|
-
HELP.setup = HELP.memory!;
|
|
793
|
-
|
|
794
|
-
function printHelp(topic?: string): never {
|
|
795
|
-
if (topic && HELP[topic]) console.log(renderHelp(HELP[topic]));
|
|
796
|
-
else if (topic) {
|
|
797
|
-
console.error(`unknown command: ${topic}\n`);
|
|
798
|
-
console.log(renderHelp(OVERVIEW));
|
|
799
|
-
process.exit(1);
|
|
800
|
-
} else console.log(renderHelp(OVERVIEW));
|
|
801
|
-
process.exit(0);
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
if (cmd === "--help" || cmd === "-h") printHelp();
|
|
805
|
-
if (cmd === "--version" || cmd === "-v" || cmd === "version") {
|
|
806
|
-
console.log(`mlx-bun ${pkg.version}`);
|
|
807
|
-
process.exit(0);
|
|
808
|
-
}
|
|
809
|
-
if (cmd === "help") printHelp(argv[1]);
|
|
810
|
-
// Per-command --help/-h — except `pi`, whose extra args pass through
|
|
811
|
-
// (use `mlx-bun help pi` for our help, `pi --help` for pi's).
|
|
812
|
-
if (cmd !== "pi" && (argv.includes("--help") || argv.includes("-h"))) printHelp(cmd);
|
|
813
|
-
if (cmd === "pi" && (argv[1] === "--help" || argv[1] === "-h") && argv.length === 2) printHelp("pi");
|
|
814
|
-
|
|
815
|
-
const flag = (name: string): boolean => argv.includes(`--${name}`);
|
|
816
|
-
const opt = (name: string, dflt: string | null = null): string | null => {
|
|
817
|
-
const i = argv.indexOf(`--${name}`);
|
|
818
|
-
return i > -1 && argv[i + 1] ? argv[i + 1]! : dflt;
|
|
819
|
-
};
|
|
820
|
-
/** Single-dash value flag (e.g. -q), the short-alias counterpart to opt(). */
|
|
821
|
-
const optShort = (short: string, dflt: string | null = null): string | null => {
|
|
822
|
-
const i = argv.indexOf(`-${short}`);
|
|
823
|
-
return i > -1 && argv[i + 1] ? argv[i + 1]! : dflt;
|
|
824
|
-
};
|
|
825
|
-
const positional = (n: number): string | undefined =>
|
|
826
|
-
argv.filter((a, i) => !a.startsWith("--") && (i === 0 || !argv[i - 1]!.startsWith("--"))).slice(1)[n];
|
|
827
|
-
|
|
828
|
-
const gb = (bytes: number) => `${(bytes / 2 ** 30).toFixed(2)} GB`;
|
|
829
|
-
|
|
830
|
-
/** First-run: fetch the MLX native runtime pack if no copy is resolvable
|
|
831
|
-
* (env / beside the binary / cache / homebrew). ~52 MB, resumable,
|
|
832
|
-
* sha256-verified. Must run BEFORE importing anything that dlopens
|
|
833
|
-
* (./server, ./generate — they load libmlxc at module scope). */
|
|
834
|
-
async function ensureNative(s?: import("./tui").Step): Promise<void> {
|
|
835
|
-
const { nativeRuntimeDir, ensureNativeRuntime, NATIVE_PACK_VERSION } = await import("./native-pack");
|
|
836
|
-
if (nativeRuntimeDir()) return;
|
|
837
|
-
s?.update(`downloading the MLX native runtime (v${NATIVE_PACK_VERSION}, ~52 MB)`);
|
|
838
|
-
await ensureNativeRuntime({
|
|
839
|
-
onProgress: (received, total) => {
|
|
840
|
-
const pct = total ? Math.floor((received / total) * 100) : 0;
|
|
841
|
-
const line = `MLX native runtime ${gb(received)} / ${gb(total)} (${pct}%)`;
|
|
842
|
-
if (s) s.update(line); else process.stdout.write(`\r ${line} `);
|
|
843
|
-
},
|
|
844
|
-
});
|
|
845
|
-
if (!s) process.stdout.write("\n");
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
/** AppleScript that focuses the first tab whose URL contains `match` in a
|
|
849
|
-
* literal-named browser, returning "ok"/"miss". The app name MUST be a literal
|
|
850
|
-
* (not a variable): AppleScript loads an app's scripting dictionary by literal
|
|
851
|
-
* name at COMPILE time, so `tell application someVar` can't resolve app-specific
|
|
852
|
-
* terms like `active tab index` (fails with -2740). Referencing an uninstalled
|
|
853
|
-
* app by literal name is also a compile error a `try` can't catch — hence we
|
|
854
|
-
* only ever run a browser's script when pgrep says it's running (= installed). */
|
|
855
|
-
function focusTabScript(app: string, kind: "chromium" | "safari", match: string): string {
|
|
856
|
-
const m = JSON.stringify(match); // safe AppleScript string literal
|
|
857
|
-
if (kind === "safari") {
|
|
858
|
-
return `tell application "${app}"
|
|
859
|
-
repeat with w in windows
|
|
860
|
-
repeat with t in tabs of w
|
|
861
|
-
if URL of t contains ${m} then
|
|
862
|
-
set current tab of w to t
|
|
863
|
-
activate
|
|
864
|
-
return "ok"
|
|
865
|
-
end if
|
|
866
|
-
end repeat
|
|
867
|
-
end repeat
|
|
868
|
-
end tell
|
|
869
|
-
return "miss"`;
|
|
870
|
-
}
|
|
871
|
-
return `tell application "${app}"
|
|
872
|
-
repeat with w in windows
|
|
873
|
-
set k to 0
|
|
874
|
-
repeat with t in tabs of w
|
|
875
|
-
set k to k + 1
|
|
876
|
-
if URL of t contains ${m} then
|
|
877
|
-
set active tab index of w to k
|
|
878
|
-
set index of w to 1
|
|
879
|
-
activate
|
|
880
|
-
return "ok"
|
|
881
|
-
end if
|
|
882
|
-
end repeat
|
|
883
|
-
end repeat
|
|
884
|
-
end tell
|
|
885
|
-
return "miss"`;
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
/** Open the chat UI, reusing an already-open tab on this host:port instead of
|
|
889
|
-
* spawning a duplicate. Each browser is scripted via its own literal-named
|
|
890
|
-
* AppleScript, run ONLY when pgrep confirms it's running (so the literal `tell`
|
|
891
|
-
* always compiles). Falls back to a plain `open` (new tab) when no running
|
|
892
|
-
* browser has the tab, for browsers we don't script (e.g. Firefox), or if
|
|
893
|
-
* AppleScript is blocked. Best-effort and fire-and-forget: any failure just
|
|
894
|
-
* means a fresh tab, so it never regresses the plain-open path. The first focus
|
|
895
|
-
* may trigger a one-time macOS "control your browser" permission prompt;
|
|
896
|
-
* declining it just falls back to opening a new tab. */
|
|
897
|
-
function openChatUi(url: string, hostPort: string): void {
|
|
898
|
-
const browsers: Array<{ proc: string; kind: "chromium" | "safari" }> = [
|
|
899
|
-
{ proc: "Google Chrome", kind: "chromium" },
|
|
900
|
-
{ proc: "Arc", kind: "chromium" },
|
|
901
|
-
{ proc: "Brave Browser", kind: "chromium" },
|
|
902
|
-
{ proc: "Microsoft Edge", kind: "chromium" },
|
|
903
|
-
{ proc: "Safari", kind: "safari" },
|
|
904
|
-
];
|
|
905
|
-
void (async () => {
|
|
906
|
-
for (const b of browsers) {
|
|
907
|
-
try {
|
|
908
|
-
const running = Bun.spawn(["pgrep", "-x", b.proc], { stdout: "ignore", stderr: "ignore" });
|
|
909
|
-
if ((await running.exited) !== 0) continue; // not running → skip (and don't compile its tell)
|
|
910
|
-
const script = focusTabScript(b.proc, b.kind, hostPort);
|
|
911
|
-
const p = Bun.spawn(["osascript", "-e", script], { stdout: "pipe", stderr: "ignore", timeout: 8000 });
|
|
912
|
-
const out = await new Response(p.stdout).text();
|
|
913
|
-
if ((await p.exited) === 0 && out.trim() === "ok") return; // focused an existing tab
|
|
914
|
-
} catch { /* try the next browser */ }
|
|
915
|
-
}
|
|
916
|
-
try { Bun.spawn(["open", url], { stdout: "ignore", stderr: "ignore" }); } catch { /* best-effort */ }
|
|
917
|
-
})();
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
/** Server/runtime flags shared by every mode that loads a model
|
|
921
|
-
* (serve, pi). Runtime overrides are installed here before the
|
|
922
|
-
* generate/compiled-decode modules read them. */
|
|
923
|
-
/** Resolve the decode ROUTE: a tier alias (--l1/--l2) sets the whole route,
|
|
924
|
-
* and an explicit per-fork flag (--kv-quant/--fused-sdpa/…) overrides the
|
|
925
|
-
* alias. Installs decode runtime options and returns the kv-quant mode. See
|
|
926
|
-
* docs/design/unified-engine-frontier-plan.md + unified-engine-frontier-plan.md.
|
|
927
|
-
* Each tier is a GUARANTEE about which reference you reproduce bit-for-bit:
|
|
928
|
-
* --l1 bit-for-bit IDENTICAL to mlx-lm — drop-in replacement for mlx-lm
|
|
929
|
-
* (bf16 KV; THE naked default since 2026-07-05)
|
|
930
|
-
* --l2 bit-for-bit IDENTICAL to mlx-optiq — drop-in replacement for
|
|
931
|
-
* mlx-optiq: quantized KV per kv_config.json + fused N-tiled SDPA for
|
|
932
|
-
* L>1 + STOCK unfused L=1 decode (the optiq-golden composition,
|
|
933
|
-
* scripts/regen.ts kvq + regen-mixed-kv-goldens.ts).
|
|
934
|
-
* There is no --l3: output-changing experiments live in the Lab (env flags
|
|
935
|
-
* with a bench + expiry) until one beats the L1 baseline in a paired A/B —
|
|
936
|
-
* docs/design/unified-engine-frontier-plan.md §6-7. */
|
|
937
|
-
function applyDecodeRoute(): { kvQuant?: "off" | "config" | number; turboQuant?: TurboQuantScheme } {
|
|
938
|
-
const onOff = (name: string): boolean | null => {
|
|
939
|
-
const v = opt(name); if (v == null) return null;
|
|
940
|
-
if (v === "on" || v === "1" || v === "true") return true;
|
|
941
|
-
if (v === "off" || v === "0" || v === "false") return false;
|
|
942
|
-
console.error(`--${name} expects on|off (got "${v}")`); process.exit(1);
|
|
943
|
-
};
|
|
944
|
-
if (flag("l3")) {
|
|
945
|
-
console.error(
|
|
946
|
-
"--l3 was removed 2026-07-05: no output-changing kernel beat the L1 baseline " +
|
|
947
|
-
"in a paired A/B, so the tier had no content. Use --l2 for the optiq route, " +
|
|
948
|
-
"or the Lab (docs/design/unified-engine-frontier-plan.md) for experiments.",
|
|
949
|
-
);
|
|
950
|
-
process.exit(1);
|
|
951
|
-
}
|
|
952
|
-
type Preset = { kv: "off" | "config"; fusedSdpa: boolean; compiled: boolean; compiledAct: boolean };
|
|
953
|
-
const TIERS: Record<string, Preset> = {
|
|
954
|
-
// A tier is the GUARANTEE; within it we default to the FAST kernel that still
|
|
955
|
-
// holds it, and expose the slow one as an opt-in. compiled-decode is BIT-EXACT
|
|
956
|
-
// with uncompiled (compiled-decode.test: on==off) → free speed, so it's ON in
|
|
957
|
-
// EVERY tier (--compiled-decode off is the slow, same-tier opt-in). compiled
|
|
958
|
-
// ACTIVATIONS (geglu/swiglu) go through the SAME libmlx as mlx-lm's `@mx.compile`
|
|
959
|
-
// → same kernel, bit-exact AND faster → ON in every tier (--compiled-activations
|
|
960
|
-
// off is the slower same-parity opt-in). fused-sdpa matches OPTIQ bit-for-bit
|
|
961
|
-
// (tier-a goldens, fused-sdpa.test) and is quantized-KV only → it's the L2 bridge,
|
|
962
|
-
// a no-op on L1's bf16.
|
|
963
|
-
l1: { kv: "off", fusedSdpa: false, compiled: true, compiledAct: true }, // = mlx-lm bit-for-bit (bf16)
|
|
964
|
-
l2: { kv: "config", fusedSdpa: true, compiled: true, compiledAct: true }, // = mlx-optiq bit-for-bit
|
|
965
|
-
};
|
|
966
|
-
// NAKED DEFAULT = L1 (decided 2026-07-05): with no tier flag you get the
|
|
967
|
-
// mlx-lm bit-for-bit route. The 2026-07-05 benchmark pass showed the L1
|
|
968
|
-
// faithful kernel set matches mlx-lm 1.00× on every model while every
|
|
969
|
-
// output-changing lever failed to beat it in a paired A/B (the losers were
|
|
970
|
-
// deleted the same day — Phase 1 of unified-engine-frontier-plan.md).
|
|
971
|
-
// Quantized KV costs 5–20% decode at ≤16k on BOTH stacks and pays only in
|
|
972
|
-
// memory/context headroom → explicit opt-in (--kv-quant / --l2).
|
|
973
|
-
const tier = flag("l2") ? "l2" : "l1";
|
|
974
|
-
const p = TIERS[tier]!;
|
|
975
|
-
const kv = opt("kv-quant"); // explicit --kv-quant overrides the tier
|
|
976
|
-
const pick = (name: string, base: boolean | undefined): boolean | null => {
|
|
977
|
-
const ex = onOff(name); return ex !== null ? ex : (base ?? null); // explicit flag wins, else tier preset
|
|
978
|
-
};
|
|
979
|
-
const set = (name: RuntimeKey, val: boolean | null, invert = false) => {
|
|
980
|
-
if (val !== null)
|
|
981
|
-
configureRuntime({ [name]: (invert ? !val : val) ? "1" : "0" });
|
|
982
|
-
};
|
|
983
|
-
// Explicit --kv-quant picks the composition its ORACLE uses: config (the
|
|
984
|
-
// optiq mixed scheme) runs optiq's fused SDPA (the L2 golden composition);
|
|
985
|
-
// uniform 4/8 runs UNFUSED (mlx-lm's quantized_scaled_dot_product_attention
|
|
986
|
-
// — the bit-exact L1-eligible scheme). --fused-sdpa still overrides either.
|
|
987
|
-
const fusedSdpaBase = kv === "config" ? true : kv != null ? false : p.fusedSdpa;
|
|
988
|
-
set("MLX_BUN_COMPILED_DECODE", pick("compiled-decode", p.compiled));
|
|
989
|
-
set("MLX_BUN_NO_FUSED_SDPA", pick("fused-sdpa", fusedSdpaBase), true); // inverted env
|
|
990
|
-
// Compiled activations (the faithful geglu/swiglu — mlx-lm's @mx.compile kernel).
|
|
991
|
-
// One fork drives both runtime options; qwen3/qwen3.5/universal compile unconditionally,
|
|
992
|
-
// so this toggles gemma geglu + minicpm5 swiglu (the only unfused-capable sites).
|
|
993
|
-
set("MLX_BUN_COMPILED_GEGLU", pick("compiled-activations", p.compiledAct));
|
|
994
|
-
set("MLX_BUN_COMPILED_SWIGLU", pick("compiled-activations", p.compiledAct));
|
|
995
|
-
if (kv === "off") return { kvQuant: "off" };
|
|
996
|
-
if (kv === "config") return { kvQuant: "config" };
|
|
997
|
-
if (kv) {
|
|
998
|
-
// TurboQuant (docs/design/turboquant.md): a distinct axis from the
|
|
999
|
-
// affine kvQuant above — dequantize-on-fetch, stock unfused ops.sdpa,
|
|
1000
|
-
// no kvQuant value at all (mutually exclusive with kvBits/kvConfig).
|
|
1001
|
-
let turboQuant: TurboQuantScheme | null;
|
|
1002
|
-
try {
|
|
1003
|
-
turboQuant = parseTurboQuantScheme(kv);
|
|
1004
|
-
} catch (e) {
|
|
1005
|
-
console.error((e as Error).message);
|
|
1006
|
-
process.exit(1);
|
|
1007
|
-
}
|
|
1008
|
-
if (turboQuant) return { turboQuant };
|
|
1009
|
-
const bits = Number(kv);
|
|
1010
|
-
if (![4, 8].includes(bits)) { console.error(`--kv-quant expects config|off|4|8|turbo (got "${kv}")`); process.exit(1); }
|
|
1011
|
-
return { kvQuant: bits };
|
|
1012
|
-
}
|
|
1013
|
-
return { kvQuant: p.kv };
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
function serverRuntimeFlags(): { port: number; serverOptions: import("./server").ServerOptions } {
|
|
1017
|
-
const onOff = (name: string): boolean | null => {
|
|
1018
|
-
const v = opt(name);
|
|
1019
|
-
if (v === null) return null;
|
|
1020
|
-
if (v === "on" || v === "1" || v === "true") return true;
|
|
1021
|
-
if (v === "off" || v === "0" || v === "false") return false;
|
|
1022
|
-
console.error(`--${name} expects on|off (got "${v}")`);
|
|
1023
|
-
process.exit(1);
|
|
1024
|
-
};
|
|
1025
|
-
const route = applyDecodeRoute(); // --l1/--l2 tier alias, with per-fork flags overriding
|
|
1026
|
-
if (flag("force-wire")) configureRuntime({ MLX_BUN_FORCE_WIRE: "1" });
|
|
1027
|
-
// Remote image_url/audio_url media fetches block private/loopback/link-
|
|
1028
|
-
// local destinations by default (SSRF — src/media-fetch.ts); this is the
|
|
1029
|
-
// LAN-hosts escape hatch.
|
|
1030
|
-
if (flag("allow-private-media"))
|
|
1031
|
-
configureRuntime({ MLX_BUN_ALLOW_PRIVATE_MEDIA: "1" });
|
|
1032
|
-
|
|
1033
|
-
const serverOptions: import("./server").ServerOptions = {};
|
|
1034
|
-
const budgetGB = Number(opt("memory-budget", "0"));
|
|
1035
|
-
if (budgetGB > 0) serverOptions.memoryBudgetBytes = budgetGB * 1e9;
|
|
1036
|
-
// Aggregate KV budget across batch rows (batching-perf-path P3 admission).
|
|
1037
|
-
const kvBudgetGB = Number(opt("kv-budget", "0"));
|
|
1038
|
-
if (kvBudgetGB > 0) serverOptions.kvBudgetBytes = kvBudgetGB * 1e9;
|
|
1039
|
-
const pcRaw = opt("prompt-cache"); // null if absent; an explicit value (incl. 0) wins → `--prompt-cache 0` DISABLES the cache
|
|
1040
|
-
if (pcRaw !== null) serverOptions.promptCacheBytes = Math.max(0, Number(pcRaw)) * 2 ** 30;
|
|
1041
|
-
// Engine-child mode (docs/reference/server-config.md): listen on a unix socket
|
|
1042
|
-
// instead of TCP — the isolated parent proxies here. Internal flag,
|
|
1043
|
-
// spawned by --isolate; usable directly for socket-level integration.
|
|
1044
|
-
const unixRaw = opt("unix");
|
|
1045
|
-
if (unixRaw !== null) serverOptions.unixSocket = unixRaw;
|
|
1046
|
-
// SSD cold tier (docs/design/kv-cache.md) — off unless a dir is given.
|
|
1047
|
-
const ssdDir = opt("ssd-cache");
|
|
1048
|
-
if (ssdDir !== null) {
|
|
1049
|
-
serverOptions.ssdCacheDir = ssdDir;
|
|
1050
|
-
const maxRaw = opt("ssd-cache-max");
|
|
1051
|
-
if (maxRaw !== null) {
|
|
1052
|
-
const maxGb = Number(maxRaw);
|
|
1053
|
-
if (!Number.isFinite(maxGb) || maxGb < 0) {
|
|
1054
|
-
console.error(`--ssd-cache-max expects a number >= 0 (got "${maxRaw}")`);
|
|
1055
|
-
process.exit(1);
|
|
1056
|
-
}
|
|
1057
|
-
serverOptions.ssdCacheMaxBytes = maxGb === 0
|
|
1058
|
-
? Number.POSITIVE_INFINITY
|
|
1059
|
-
: maxGb * 2 ** 30;
|
|
1060
|
-
}
|
|
1061
|
-
if (flag("ssd-cache-verify")) serverOptions.ssdCacheVerify = true;
|
|
1062
|
-
const demoteRaw = opt("ssd-demote-idle");
|
|
1063
|
-
if (demoteRaw !== null) serverOptions.ssdDemoteIdleSec = Math.max(0, Number(demoteRaw));
|
|
1064
|
-
const checkpointRaw = opt("generation-checkpoint");
|
|
1065
|
-
if (checkpointRaw !== null) {
|
|
1066
|
-
const n = Number(checkpointRaw);
|
|
1067
|
-
if (!Number.isInteger(n) || n < 1) {
|
|
1068
|
-
console.error(`--generation-checkpoint expects an integer >= 1 (got "${checkpointRaw}")`);
|
|
1069
|
-
process.exit(1);
|
|
1070
|
-
}
|
|
1071
|
-
serverOptions.generationCheckpointTokens = n;
|
|
1072
|
-
}
|
|
1073
|
-
} else {
|
|
1074
|
-
// Refuse-loudly house rule: sub-flags of an absent tier must not vanish.
|
|
1075
|
-
for (const f of ["ssd-cache-max", "ssd-demote-idle", "generation-checkpoint"] as const)
|
|
1076
|
-
if (opt(f) !== null) console.warn(`--${f} has no effect without --ssd-cache — ignored`);
|
|
1077
|
-
if (flag("ssd-cache-verify"))
|
|
1078
|
-
console.warn("--ssd-cache-verify has no effect without --ssd-cache — ignored");
|
|
1079
|
-
}
|
|
1080
|
-
// --batch N: max concurrent requests batched through the mlx-lm-parity
|
|
1081
|
-
// engine (N=1 pins the strict serial executor). --decode-concurrency is accepted for
|
|
1082
|
-
// drop-in compatibility with mlx_lm.server, but the semantics differ: there
|
|
1083
|
-
// it caps per-BatchGenerator decode parallelism (default 32); in mlx-bun it
|
|
1084
|
-
// caps continuous batching (default 8; `--batch 1` pins the serial path —
|
|
1085
|
-
// the unified engine is bit-parity with it and costs ~nothing at B=1).
|
|
1086
|
-
const batchRaw = opt("batch") ?? opt("decode-concurrency");
|
|
1087
|
-
if (batchRaw !== null) {
|
|
1088
|
-
const n = Number(batchRaw);
|
|
1089
|
-
if (!Number.isInteger(n) || n < 1) {
|
|
1090
|
-
console.error(`--batch expects an integer >= 1 (got "${batchRaw}")`);
|
|
1091
|
-
process.exit(1);
|
|
1092
|
-
}
|
|
1093
|
-
serverOptions.batch = n;
|
|
1094
|
-
}
|
|
1095
|
-
if (route.kvQuant !== undefined) serverOptions.kvQuant = route.kvQuant;
|
|
1096
|
-
if (route.turboQuant !== undefined) serverOptions.turboQuant = route.turboQuant;
|
|
1097
|
-
// --paged-kv (env: MLX_BUN_PAGED_KV=1): OPTIONAL vLLM-style block-pool KV
|
|
1098
|
-
// storage, default off (docs/design/kv-cache.md). The state policy owns
|
|
1099
|
-
// block pools and cache reuse; paging preserves the selected batch size.
|
|
1100
|
-
if (flag("paged-kv") || runtimeValue("MLX_BUN_PAGED_KV") === "1") {
|
|
1101
|
-
const bsRaw = opt("paged-kv-block-size");
|
|
1102
|
-
serverOptions.pagedKv = bsRaw !== null ? { blockSize: Number(bsRaw) } : {};
|
|
1103
|
-
}
|
|
1104
|
-
// Bind loopback unless asked otherwise (mlx_lm.server parity); --host
|
|
1105
|
-
// 0.0.0.0 is the explicit opt-in for LAN exposure. The chat-UI open and
|
|
1106
|
-
// `mlx-bun pi` attach both go through localhost, so loopback-only is
|
|
1107
|
-
// transparent to them.
|
|
1108
|
-
serverOptions.hostname = opt("host", "127.0.0.1")!;
|
|
1109
|
-
// Speech-to-text companion: explicit checkpoint + residency policy.
|
|
1110
|
-
const whisperIdleRaw = opt("whisper-idle-unload");
|
|
1111
|
-
if (whisperIdleRaw !== null) {
|
|
1112
|
-
const n = Number(whisperIdleRaw);
|
|
1113
|
-
if (!Number.isFinite(n) || n < 0) {
|
|
1114
|
-
console.error(`--whisper-idle-unload expects seconds >= 0 (got "${whisperIdleRaw}")`);
|
|
1115
|
-
process.exit(1);
|
|
1116
|
-
}
|
|
1117
|
-
serverOptions.whisperIdleUnloadSec = n;
|
|
1118
|
-
}
|
|
1119
|
-
if (flag("whisper-resident")) serverOptions.whisperResident = true;
|
|
1120
|
-
// Server-wide default for the chat template's enable_thinking variable
|
|
1121
|
-
// (CPM/MiniCPM5 and other hybrid-reasoning models). Unset ⇒ the model's
|
|
1122
|
-
// own default (false for MiniCPM5). onOff accepts true|false|on|off|1|0.
|
|
1123
|
-
const thinking = onOff("thinking");
|
|
1124
|
-
if (thinking !== null) serverOptions.defaultThinking = thinking;
|
|
1125
|
-
// Server-wide sampling defaults. Unset ⇒ the model's generation_config.json
|
|
1126
|
-
// (MiniCPM5 ships 0.9 / 0.95 — its think recipe). A per-request field still
|
|
1127
|
-
// overrides these; the browser chat sends none, so this is its only lever.
|
|
1128
|
-
const numFlag = (name: string, lo: number, hi: number): number | null => {
|
|
1129
|
-
const v = opt(name);
|
|
1130
|
-
if (v === null) return null;
|
|
1131
|
-
const n = Number(v);
|
|
1132
|
-
if (!Number.isFinite(n) || n < lo || n > hi) {
|
|
1133
|
-
console.error(`--${name} expects a number in [${lo}, ${hi}] (got "${v}")`);
|
|
1134
|
-
process.exit(1);
|
|
1135
|
-
}
|
|
1136
|
-
return n;
|
|
1137
|
-
};
|
|
1138
|
-
// --temp is the mlx_lm.server spelling; explicit --temperature wins.
|
|
1139
|
-
const temp = numFlag("temperature", 0, 5) ?? numFlag("temp", 0, 5);
|
|
1140
|
-
if (temp !== null) serverOptions.defaultTemperature = temp;
|
|
1141
|
-
const topP = numFlag("top-p", 0, 1);
|
|
1142
|
-
if (topP !== null) serverOptions.defaultTopP = topP;
|
|
1143
|
-
const topK = numFlag("top-k", 0, 1_000_000);
|
|
1144
|
-
if (topK !== null) serverOptions.defaultTopK = topK;
|
|
1145
|
-
// mlx_lm.server's --max-tokens: the completion cap when a request omits
|
|
1146
|
-
// max_tokens. Our built-in default stays 65,536 (chat) / 512 (raw text
|
|
1147
|
-
// completion); `--max-tokens 512` reproduces mlx-lm's behavior exactly.
|
|
1148
|
-
const maxTok = numFlag("max-tokens", 1, 10_000_000);
|
|
1149
|
-
if (maxTok !== null) serverOptions.defaultMaxTokens = Math.floor(maxTok);
|
|
1150
|
-
// HLG sampling — piecewise tone curve on the logprobs (default off). Knobs in
|
|
1151
|
-
// nats; the mid gain folds from --temperature. docs/archive/hlg-sampling.md.
|
|
1152
|
-
if (onOff("hlg-sampling") === true) {
|
|
1153
|
-
serverOptions.hlg = {
|
|
1154
|
-
enabled: true,
|
|
1155
|
-
width: numFlag("hlg-width", 0, 100) ?? 4,
|
|
1156
|
-
shoulder: numFlag("hlg-shoulder", 0, 100) ?? 4,
|
|
1157
|
-
toe: numFlag("hlg-toe", 0, 100) ?? 6,
|
|
1158
|
-
pivotOffset: numFlag("hlg-pivot-offset", 0, 100) ?? 6,
|
|
1159
|
-
pivot: "top",
|
|
1160
|
-
};
|
|
1161
|
-
}
|
|
1162
|
-
return { port: Number(opt("port", "8080")), serverOptions };
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
/** Direct Colibri runtime defaults shared by `serve` and the embedded `pi`
|
|
1166
|
-
* server. Other model families ignore this load-only option block. */
|
|
1167
|
-
function glm52RuntimeFlags(
|
|
1168
|
-
serverOptions: import("./server").ServerOptions,
|
|
1169
|
-
): import("./model/factory").Glm52RuntimeOpenOptions {
|
|
1170
|
-
const mtpRaw = opt("mtp");
|
|
1171
|
-
let enableMtp: boolean | undefined;
|
|
1172
|
-
if (mtpRaw !== null) {
|
|
1173
|
-
if (["on", "1", "true"].includes(mtpRaw)) enableMtp = true;
|
|
1174
|
-
else if (["off", "0", "false"].includes(mtpRaw)) enableMtp = false;
|
|
1175
|
-
else {
|
|
1176
|
-
console.error(`--mtp expects on|off (got "${mtpRaw}")`);
|
|
1177
|
-
process.exit(1);
|
|
1178
|
-
}
|
|
1179
|
-
}
|
|
1180
|
-
const contextRaw = opt("context-length");
|
|
1181
|
-
let contextTokens: number | undefined;
|
|
1182
|
-
if (contextRaw !== null) {
|
|
1183
|
-
contextTokens = Number(contextRaw);
|
|
1184
|
-
if (!Number.isSafeInteger(contextTokens) || contextTokens < 1) {
|
|
1185
|
-
console.error(`--context-length expects an integer >= 1 (got "${contextRaw}")`);
|
|
1186
|
-
process.exit(1);
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
return {
|
|
1190
|
-
memoryBudgetBytes: serverOptions.memoryBudgetBytes,
|
|
1191
|
-
batchSize: serverOptions.batch ?? 8,
|
|
1192
|
-
maxGenerationTokens: serverOptions.defaultMaxTokens ?? 128,
|
|
1193
|
-
...(contextTokens !== undefined ? { contextTokens } : {}),
|
|
1194
|
-
...(enableMtp !== undefined ? { enableMtp } : {}),
|
|
1195
|
-
};
|
|
1196
|
-
}
|
|
1197
|
-
|
|
1198
|
-
/** `--adapter <dir>` (alias `--adapter-path`, mlx_lm.server's spelling):
|
|
1199
|
-
* mount a LoRA adapter at startup — the same machinery as POST /v1/adapters —
|
|
1200
|
-
* and make it the default for requests that don't select one (a request's
|
|
1201
|
-
* explicit `adapter`, including "none", still wins; hot-swap unchanged).
|
|
1202
|
-
* This is the flag `mlx-bun train`'s completion message points at. Shared by
|
|
1203
|
-
* `serve` and the `pi` session server. Exits on a bad adapter — better a
|
|
1204
|
-
* loud startup failure than silently serving the base model. */
|
|
1205
|
-
async function mountStartupAdapter(
|
|
1206
|
-
ctx: import("./server").ServerContext,
|
|
1207
|
-
serverOptions: import("./server").ServerOptions,
|
|
1208
|
-
): Promise<void> {
|
|
1209
|
-
const adapterDir = opt("adapter") ?? opt("adapter-path");
|
|
1210
|
-
if (!adapterDir) return;
|
|
1211
|
-
const { step, style } = await import("./tui");
|
|
1212
|
-
const sAdp = step("mounting adapter");
|
|
1213
|
-
try {
|
|
1214
|
-
const adapterId = adapterDir.replace(/\/+$/, "").split("/").pop()!;
|
|
1215
|
-
const info = await ctx.adapters.mount(adapterId, adapterDir);
|
|
1216
|
-
serverOptions.defaultAdapter = info.id;
|
|
1217
|
-
sAdp.done(`adapter ${style.bold(info.id)} ${style.dim(`· ${info.mountedLayers} layers · default for requests (select others via \`adapter\`)`)}`);
|
|
1218
|
-
} catch (e) {
|
|
1219
|
-
sAdp.fail(`adapter mount failed: ${(e as Error).message}`);
|
|
1220
|
-
process.exit(1);
|
|
1221
|
-
}
|
|
1222
|
-
}
|
|
1223
|
-
|
|
1224
|
-
/** One-line summary of the active runtime levers for the ready card. */
|
|
1225
|
-
function runtimeSummary(o: import("./server").ServerOptions): string {
|
|
1226
|
-
const kv = o.turboQuant ? `turbo-k${o.turboQuant.kBits}v${o.turboQuant.vBits}`
|
|
1227
|
-
: o.kvQuant === "off" ? "off" : typeof o.kvQuant === "number" ? `kv${o.kvQuant}` : "config";
|
|
1228
|
-
const lever = (name: RuntimeKey, dflt: string) => runtimeValue(name) ?? dflt;
|
|
1229
|
-
return `kv-quant ${kv} · compiled-decode ${lever("MLX_BUN_COMPILED_DECODE", "1") === "1" ? "on" : "off"}` +
|
|
1230
|
-
(lever("MLX_BUN_COMPILED_GEGLU", "1") === "0" ? " · compiled-activations off" : "") +
|
|
1231
|
-
(o.batch && o.batch > 1 ? ` · batch ${o.batch}` : "") +
|
|
1232
|
-
(o.defaultThinking !== undefined ? ` · thinking ${o.defaultThinking ? "on" : "off"}` : "") +
|
|
1233
|
-
(o.defaultTemperature !== undefined ? ` · temp ${o.defaultTemperature}` : "") +
|
|
1234
|
-
(o.defaultTopP !== undefined ? ` · top-p ${o.defaultTopP}` : "") +
|
|
1235
|
-
(o.defaultTopK !== undefined ? ` · top-k ${o.defaultTopK}` : "");
|
|
1236
|
-
}
|
|
1237
|
-
|
|
1238
|
-
/** Shared model resolution: explicit query wins; otherwise e4b (the default
|
|
1239
|
-
* model everywhere) if it's downloaded, else the largest supported model that
|
|
1240
|
-
* fits this machine. Downloads the recommended model first on a fresh install.
|
|
1241
|
-
* A query that is a DIRECTORY containing config.json loads directly via
|
|
1242
|
-
* scanSnapshot (no registry dependency) — the `--model <path>` bench/CI shape. */
|
|
1243
|
-
async function resolveModelAuto(query: string | null): Promise<{ m: import("./registry").ModelRecord; picked: boolean }> {
|
|
1244
|
-
if (query && existsSync(join(query, "config.json"))) {
|
|
1245
|
-
const { scanSnapshot } = await import("./registry");
|
|
1246
|
-
const dir = resolve(query);
|
|
1247
|
-
// Recover the repo id from an HF-cache path (…/models--org--name/snapshots/<rev>);
|
|
1248
|
-
// otherwise the directory basename is the display id.
|
|
1249
|
-
const hf = /models--([^/]+)--([^/]+)\/snapshots\//.exec(dir);
|
|
1250
|
-
const repoId = hf ? `${hf[1]}/${hf[2]}` : basename(dir);
|
|
1251
|
-
const m = await scanSnapshot(dir, repoId);
|
|
1252
|
-
if (!m) throw new Error(`${dir} has config.json but no weights (*.safetensors)`);
|
|
1253
|
-
return { m, picked: false };
|
|
1254
|
-
}
|
|
1255
|
-
const reg = new Registry();
|
|
1256
|
-
if (reg.list().length === 0) await reg.scan();
|
|
1257
|
-
if (query) return { m: reg.resolve(query), picked: false };
|
|
1258
|
-
const { recommendedRepoId } = await import("./fit");
|
|
1259
|
-
let candidates = reg.list().filter((r) => isSupportedModelRecord(r.modelType, r.repoId));
|
|
1260
|
-
if (candidates.length === 0) {
|
|
1261
|
-
// First run: starter-model flow (PRODUCT_ROADMAP / PLAN Phase 16).
|
|
1262
|
-
// Download the sub-GB starter first so the user is chatting quickly,
|
|
1263
|
-
// then stream the recommended Gemma for this Mac in the background —
|
|
1264
|
-
// it becomes the auto-pick next run.
|
|
1265
|
-
const starterRepo = "mlx-community/MiniCPM5-1B-OptiQ-4bit";
|
|
1266
|
-
const recRepo = recommendedRepoId();
|
|
1267
|
-
const { step, style } = await import("./tui");
|
|
1268
|
-
const { downloadModel } = await import("./download");
|
|
1269
|
-
const s = step(`downloading starter model ${starterRepo}`);
|
|
1270
|
-
await downloadModel(starterRepo, {
|
|
1271
|
-
onProgress: (file, received, total) => {
|
|
1272
|
-
const pct = total ? Math.floor((received / total) * 100) : 0;
|
|
1273
|
-
s.update(`starter model ${starterRepo} — ${file} ${gb(received)} / ${gb(total)} (${pct}%)`);
|
|
1274
|
-
},
|
|
1275
|
-
});
|
|
1276
|
-
s.done(`starter model ready ${style.dim(`· ${starterRepo}`)}`);
|
|
1277
|
-
await reg.scan();
|
|
1278
|
-
if (recRepo !== starterRepo) {
|
|
1279
|
-
console.log(` ${style.dim(`downloading ${recRepo} (recommended for this Mac) in the background —`)}`);
|
|
1280
|
-
console.log(` ${style.dim("it becomes the default next run · progress in the web app's Status view · resumable")}`);
|
|
1281
|
-
downloadModel(recRepo)
|
|
1282
|
-
.then(async () => {
|
|
1283
|
-
await new Registry().scan();
|
|
1284
|
-
console.log(` ${style.dim(`background download complete: ${recRepo} (used on next run)`)}`);
|
|
1285
|
-
})
|
|
1286
|
-
.catch(() => { /* resumable — the next run continues it */ });
|
|
1287
|
-
}
|
|
1288
|
-
candidates = reg.list().filter((r) => isSupportedModelRecord(r.modelType, r.repoId));
|
|
1289
|
-
}
|
|
1290
|
-
// No query: e4b by default; otherwise the largest model that still leaves
|
|
1291
|
-
// the machine usable for other apps. Never auto-grab the 26B "dedicate the
|
|
1292
|
-
// machine" model on a small-RAM Mac — that's an explicit --query choice —
|
|
1293
|
-
// but allow it as a last resort if it's all the user has downloaded.
|
|
1294
|
-
const { DEFAULT_REPO_ID, COEXIST_FRACTION, chooseAutoModel, thisMachine } = await import("./fit");
|
|
1295
|
-
const machine = thisMachine();
|
|
1296
|
-
const fitsFull = new Map<string, boolean>();
|
|
1297
|
-
const fitsCoexist = new Map<string, boolean>();
|
|
1298
|
-
for (const r of candidates) {
|
|
1299
|
-
const config = await loadModelConfig(r.path);
|
|
1300
|
-
fitsFull.set(r.repoId, fit(config, r.sizeBytes, 8192, machine, undefined, r.expertsBytes).fits);
|
|
1301
|
-
fitsCoexist.set(
|
|
1302
|
-
r.repoId,
|
|
1303
|
-
fit(config, r.sizeBytes, 8192, machine, undefined, r.expertsBytes, machine.ramBytes * COEXIST_FRACTION).fits,
|
|
1304
|
-
);
|
|
1305
|
-
}
|
|
1306
|
-
const chosen = chooseAutoModel(
|
|
1307
|
-
candidates,
|
|
1308
|
-
DEFAULT_REPO_ID,
|
|
1309
|
-
(c) => fitsFull.get(c.repoId) ?? false,
|
|
1310
|
-
(c) => fitsCoexist.get(c.repoId) ?? false,
|
|
1311
|
-
);
|
|
1312
|
-
if (chosen) return { m: chosen, picked: true };
|
|
1313
|
-
console.error("no downloaded supported model fits this machine — pick one explicitly (mlx-bun ls)");
|
|
1314
|
-
process.exit(1);
|
|
1315
|
-
}
|
|
1316
|
-
|
|
1317
|
-
function parseSize(s: string): number {
|
|
1318
|
-
const m = /^([\d.]+)\s*(GB|MB|G|M)?$/i.exec(s.trim());
|
|
1319
|
-
if (!m) throw new Error(`bad size: ${s}`);
|
|
1320
|
-
const mult = /^m/i.test(m[2] ?? "G") ? 2 ** 20 : 2 ** 30;
|
|
1321
|
-
return Number(m[1]) * mult;
|
|
1322
|
-
}
|
|
1323
|
-
|
|
1324
|
-
switch (cmd) {
|
|
1325
|
-
case "get": {
|
|
1326
|
-
let repoId = positional(0);
|
|
1327
|
-
if (!repoId) {
|
|
1328
|
-
console.error("usage: mlx-bun get <org/repo | substring> [--revision main]");
|
|
1329
|
-
process.exit(1);
|
|
1330
|
-
}
|
|
1331
|
-
if (!repoId.includes("/")) {
|
|
1332
|
-
// No org/name — treat it as a registry query over already-downloaded
|
|
1333
|
-
// repos (re-get/refresh, e.g. `mlx-bun get 12B` after an upstream push).
|
|
1334
|
-
try {
|
|
1335
|
-
const regQ = new Registry();
|
|
1336
|
-
if (regQ.list().length === 0) await regQ.scan();
|
|
1337
|
-
repoId = regQ.resolve(repoId).repoId;
|
|
1338
|
-
} catch (e) {
|
|
1339
|
-
console.error((e as Error).message);
|
|
1340
|
-
console.error(`usage: mlx-bun get <org/repo> [--revision main] — or a substring of a downloaded repo (try \`mlx-bun ls ${repoId}\`)`);
|
|
1341
|
-
process.exit(1);
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
const { downloadModel } = await import("./download");
|
|
1345
|
-
const { step, style } = await import("./tui");
|
|
1346
|
-
const { join: joinPath, basename } = await import("node:path");
|
|
1347
|
-
const { readdirSync, existsSync } = await import("node:fs");
|
|
1348
|
-
const { DEFAULT_HUB, planRepoGc } = await import("./registry");
|
|
1349
|
-
const repoDir = joinPath(DEFAULT_HUB, `models--${repoId.replaceAll("/", "--")}`);
|
|
1350
|
-
const priorSnapshots = existsSync(joinPath(repoDir, "snapshots"))
|
|
1351
|
-
? readdirSync(joinPath(repoDir, "snapshots"))
|
|
1352
|
-
: [];
|
|
1353
|
-
const s = step(`downloading ${repoId}`);
|
|
1354
|
-
const snap = await downloadModel(repoId, {
|
|
1355
|
-
revision: opt("revision", "main")!,
|
|
1356
|
-
onProgress: (file, received, total) => {
|
|
1357
|
-
const pct = total ? Math.floor((received / total) * 100) : 0;
|
|
1358
|
-
s.update(`${style.bold(repoId)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
|
|
1359
|
-
},
|
|
1360
|
-
});
|
|
1361
|
-
s.done(`${style.bold(repoId)} ${style.dim("downloaded · verified")}`);
|
|
1362
|
-
const sScan = step("updating registry");
|
|
1363
|
-
const reg = new Registry();
|
|
1364
|
-
await reg.scan();
|
|
1365
|
-
sScan.done(`registry updated ${style.dim(`· ${snap}`)}`);
|
|
1366
|
-
// Upstream pushed a new revision → this get created a NEW snapshots/<sha>
|
|
1367
|
-
// dir; the old one stays on disk (that's how ~25 GB of dead blobs pile
|
|
1368
|
-
// up). Say so, with what `gc` would get back.
|
|
1369
|
-
if (priorSnapshots.length > 0 && !priorSnapshots.includes(basename(snap))) {
|
|
1370
|
-
const plan = planRepoGc(repoDir);
|
|
1371
|
-
const note = plan.skippedSnapshots.length > 0
|
|
1372
|
-
? " (a previous snapshot has files this revision lacks — gc will warn; --force to prune it)"
|
|
1373
|
-
: "";
|
|
1374
|
-
console.log(style.dim(
|
|
1375
|
-
` previous revision kept on disk — \`mlx-bun gc\` reclaims ~${gb(plan.reclaimBytes)}${note}`,
|
|
1376
|
-
));
|
|
1377
|
-
}
|
|
1378
|
-
break;
|
|
1379
|
-
}
|
|
1380
|
-
|
|
1381
|
-
case "scan": {
|
|
1382
|
-
const { step } = await import("./tui");
|
|
1383
|
-
const s = step("scanning the Hugging Face cache");
|
|
1384
|
-
const reg = new Registry();
|
|
1385
|
-
const n = await reg.scan();
|
|
1386
|
-
s.done(`indexed ${n} model snapshot(s)`);
|
|
1387
|
-
break;
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
case "ls": {
|
|
1391
|
-
const reg = new Registry();
|
|
1392
|
-
if ((reg.list().length ?? 0) === 0) await reg.scan();
|
|
1393
|
-
const maxSize = opt("max-size");
|
|
1394
|
-
const allRevisions = flag("all-revisions");
|
|
1395
|
-
const lsFilter = {
|
|
1396
|
-
vision: flag("vision") ? true : undefined,
|
|
1397
|
-
maxBytes: maxSize ? parseSize(maxSize) : undefined,
|
|
1398
|
-
query: positional(0),
|
|
1399
|
-
};
|
|
1400
|
-
// One row per repo by default: the HF cache keeps a snapshots/<commit>
|
|
1401
|
-
// dir per downloaded revision, and re-getting after an upstream push
|
|
1402
|
-
// strands the old one — those are duplicates, not separate models.
|
|
1403
|
-
// --all-revisions shows the per-snapshot truth (canonical marked *).
|
|
1404
|
-
const models = allRevisions ? reg.list(lsFilter) : reg.listCanonical(lsFilter);
|
|
1405
|
-
if (models.length === 0) {
|
|
1406
|
-
console.log("no models match (try `mlx-bun scan`)");
|
|
1407
|
-
break;
|
|
1408
|
-
}
|
|
1409
|
-
const { visionCapable } = await import("./registry");
|
|
1410
|
-
const { supportTier } = await import("./model/support");
|
|
1411
|
-
const canonicalPaths = new Set(reg.listCanonical(lsFilter).map((m) => m.path));
|
|
1412
|
-
const capabilities = (m: (typeof models)[number]) => {
|
|
1413
|
-
const tier = supportTier(m.modelType, m.repoId);
|
|
1414
|
-
return [
|
|
1415
|
-
tier ? `supported (${tier})` : `unsupported (${m.modelType})`,
|
|
1416
|
-
visionCapable(m) ? "vision" : null,
|
|
1417
|
-
m.hasToolTemplate ? "tools" : null,
|
|
1418
|
-
m.hasKvConfig ? "kv-quant" : null,
|
|
1419
|
-
].filter(Boolean).join(" · ");
|
|
1420
|
-
};
|
|
1421
|
-
const { table, style, h1 } = await import("./tui");
|
|
1422
|
-
h1("library");
|
|
1423
|
-
console.log();
|
|
1424
|
-
table(
|
|
1425
|
-
[
|
|
1426
|
-
{ header: "model", paint: (c) => style.bold(c) },
|
|
1427
|
-
...(allRevisions ? [{ header: "revision" }] : []),
|
|
1428
|
-
{ header: "size", align: "right" as const },
|
|
1429
|
-
{ header: "params", align: "right" as const },
|
|
1430
|
-
{ header: "quant" },
|
|
1431
|
-
{ header: "license", paint: (c: string) => style.dim(c) },
|
|
1432
|
-
{ header: "capabilities", paint: (c: string) => style.dim(c) },
|
|
1433
|
-
],
|
|
1434
|
-
models.map((m) => [
|
|
1435
|
-
m.repoId,
|
|
1436
|
-
...(allRevisions
|
|
1437
|
-
? [`${(m.path.split("/snapshots/")[1] ?? "").slice(0, 12)}${canonicalPaths.has(m.path) ? " *" : ""}`]
|
|
1438
|
-
: []),
|
|
1439
|
-
gb(m.sizeBytes),
|
|
1440
|
-
m.paramCount ? `${(m.paramCount / 1e9).toFixed(1)}B` : "?",
|
|
1441
|
-
m.quantBits ? `${m.quantBits}-bit g${m.quantGroupSize}` : "full",
|
|
1442
|
-
m.license ?? "?",
|
|
1443
|
-
capabilities(m),
|
|
1444
|
-
]),
|
|
1445
|
-
);
|
|
1446
|
-
console.log();
|
|
1447
|
-
if (allRevisions) {
|
|
1448
|
-
console.log(style.dim(
|
|
1449
|
-
` ${models.length} snapshot(s) · * = canonical (refs/main) · superseded snapshots: \`mlx-bun gc\``,
|
|
1450
|
-
));
|
|
1451
|
-
} else {
|
|
1452
|
-
console.log(style.dim(` ${models.length} model(s) · mlx-bun fit <query> for a memory assessment`));
|
|
1453
|
-
}
|
|
1454
|
-
break;
|
|
1455
|
-
}
|
|
1456
|
-
|
|
1457
|
-
case "gc": {
|
|
1458
|
-
const { planGc, executeGc, DEFAULT_HUB } = await import("./registry");
|
|
1459
|
-
const { table, style, h1 } = await import("./tui");
|
|
1460
|
-
const force = flag("force");
|
|
1461
|
-
const plans = planGc(DEFAULT_HUB, { force }).filter(
|
|
1462
|
-
(p) => p.pruneSnapshots.length || p.skippedSnapshots.length || p.deadBlobs.length,
|
|
1463
|
-
);
|
|
1464
|
-
h1("gc — superseded snapshots + dead blobs");
|
|
1465
|
-
console.log();
|
|
1466
|
-
if (plans.length === 0) {
|
|
1467
|
-
console.log(" nothing to reclaim — every snapshot is referenced by refs/*");
|
|
1468
|
-
break;
|
|
1469
|
-
}
|
|
1470
|
-
table(
|
|
1471
|
-
[
|
|
1472
|
-
{ header: "repo", paint: (c) => style.bold(c) },
|
|
1473
|
-
{ header: "keep", align: "right" },
|
|
1474
|
-
{ header: "prune", align: "right" },
|
|
1475
|
-
{ header: "skip", align: "right" },
|
|
1476
|
-
{ header: "reclaims", align: "right" },
|
|
1477
|
-
],
|
|
1478
|
-
plans.map((p) => [
|
|
1479
|
-
p.repoId,
|
|
1480
|
-
String(p.keepSnapshots.length),
|
|
1481
|
-
String(p.pruneSnapshots.length),
|
|
1482
|
-
String(p.skippedSnapshots.length),
|
|
1483
|
-
gb(p.reclaimBytes),
|
|
1484
|
-
]),
|
|
1485
|
-
);
|
|
1486
|
-
console.log();
|
|
1487
|
-
for (const p of plans) {
|
|
1488
|
-
for (const s of p.skippedSnapshots) {
|
|
1489
|
-
const rev = (s.path.split("/snapshots/")[1] ?? s.path).slice(0, 12);
|
|
1490
|
-
console.log(style.accent(
|
|
1491
|
-
` WARNING ${p.repoId}@${rev}: superseded snapshot has files the canonical revision lacks — skipped.`,
|
|
1492
|
-
));
|
|
1493
|
-
console.log(style.dim(` only copy of: ${s.extraFiles.join(", ")}`));
|
|
1494
|
-
console.log(style.dim(" deleting it loses those files — rerun with --force if that's intended."));
|
|
1495
|
-
}
|
|
1496
|
-
}
|
|
1497
|
-
const totalReclaim = plans.reduce((a, p) => a + p.reclaimBytes, 0);
|
|
1498
|
-
const totalSnaps = plans.reduce((a, p) => a + p.pruneSnapshots.length, 0);
|
|
1499
|
-
const totalBlobs = plans.reduce((a, p) => a + p.deadBlobs.length, 0);
|
|
1500
|
-
console.log(
|
|
1501
|
-
` total: ${totalSnaps} snapshot(s), ${totalBlobs} blob(s), ${style.bold(gb(totalReclaim))}`,
|
|
1502
|
-
);
|
|
1503
|
-
if (flag("dry-run") || !flag("yes")) {
|
|
1504
|
-
console.log(style.dim(" nothing deleted — rerun with --yes to delete (destructive)."));
|
|
1505
|
-
break;
|
|
1506
|
-
}
|
|
1507
|
-
const res = executeGc(plans);
|
|
1508
|
-
console.log(
|
|
1509
|
-
` deleted ${res.snapshots} snapshot(s) + ${res.blobs} blob(s) — reclaimed ${style.bold(gb(res.reclaimedBytes))}`,
|
|
1510
|
-
);
|
|
1511
|
-
const reg = new Registry();
|
|
1512
|
-
await reg.scan(); // reap deleted snapshots from the registry
|
|
1513
|
-
break;
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
case "fit": {
|
|
1517
|
-
const query = positional(0);
|
|
1518
|
-
if (!query) { console.error("usage: mlx-bun fit <query> [--ctx N] [--kv-quant 4|8|config] [--skus]"); process.exit(1); }
|
|
1519
|
-
const reg = new Registry();
|
|
1520
|
-
if (reg.list().length === 0) await reg.scan();
|
|
1521
|
-
const m = reg.resolve(query);
|
|
1522
|
-
const config = await loadModelConfig(m.path);
|
|
1523
|
-
const { box, table, style, h1, gradient } = await import("./tui");
|
|
1524
|
-
const glm = isGlm52Config(config);
|
|
1525
|
-
const ctx = Number(opt("ctx", glm ? "4096" : "8192"));
|
|
1526
|
-
if (glm) {
|
|
1527
|
-
const machine = thisMachine();
|
|
1528
|
-
let plan;
|
|
1529
|
-
try {
|
|
1530
|
-
plan = await planGlm52MemoryForArtifact(m.path, {
|
|
1531
|
-
machineBytes: machine.ramBytes,
|
|
1532
|
-
processLimitBytes: Math.min(
|
|
1533
|
-
GLM52_G5_DEFAULT_PROCESS_LIMIT_BYTES,
|
|
1534
|
-
machine.ramBytes,
|
|
1535
|
-
),
|
|
1536
|
-
contextTokens: ctx,
|
|
1537
|
-
maxGenerationTokens: Math.min(
|
|
1538
|
-
ctx,
|
|
1539
|
-
GLM52_G5_DEFAULT_MAX_GENERATION_TOKENS,
|
|
1540
|
-
),
|
|
1541
|
-
batchSize: 1,
|
|
1542
|
-
enableMtp: true,
|
|
1543
|
-
});
|
|
1544
|
-
} catch (error) {
|
|
1545
|
-
console.error(
|
|
1546
|
-
`GLM-5.2 resource plan refused: ` +
|
|
1547
|
-
`${error instanceof Error ? error.message : String(error)}`,
|
|
1548
|
-
);
|
|
1549
|
-
process.exitCode = 1;
|
|
1550
|
-
break;
|
|
1551
|
-
}
|
|
1552
|
-
const gib = (bytes: number): string => `${(bytes / 2 ** 30).toFixed(2)} GiB`;
|
|
1553
|
-
const li = plan.lineItems;
|
|
1554
|
-
const kvBytes = li.targetKvBytes + li.mtpKvBytes;
|
|
1555
|
-
const otherBytes = plan.plannedProcessBytes - li.residentWeightsBytes -
|
|
1556
|
-
li.mainExpertSlabBytes - li.mtpExpertSlabBytes - kvBytes;
|
|
1557
|
-
h1("will it fit?");
|
|
1558
|
-
console.log(` ${style.bold(m.repoId)} ${style.dim(`@ ${ctx.toLocaleString()} context · streamed GLM plan`)}`);
|
|
1559
|
-
console.log();
|
|
1560
|
-
box([
|
|
1561
|
-
`artifact on disk ${gib(m.sizeBytes).padStart(10)} ${style.dim("(streamed, not resident)")}`,
|
|
1562
|
-
`resident weights ${gib(li.residentWeightsBytes).padStart(10)}`,
|
|
1563
|
-
`main expert slab ${gib(li.mainExpertSlabBytes).padStart(10)}`,
|
|
1564
|
-
`MTP expert slab ${gib(li.mtpExpertSlabBytes).padStart(10)}`,
|
|
1565
|
-
`target + MTP KV ${gib(kvBytes).padStart(10)}`,
|
|
1566
|
-
`runtime reserves ${gib(otherBytes).padStart(10)}`,
|
|
1567
|
-
`process plan ${gib(plan.plannedProcessBytes).padStart(10)} ${style.dim(`of ${gib(plan.processLimitBytes)}`)} ${style.green(style.bold("FITS"))}`,
|
|
1568
|
-
`macOS headroom ${gib(plan.machineHeadroomBytes).padStart(10)}`,
|
|
1569
|
-
"",
|
|
1570
|
-
`max safe context ${style.bold(plan.contextTokens.toLocaleString())} tokens`,
|
|
1571
|
-
`max generation ${style.bold(plan.maxGenerationTokens.toLocaleString())} tokens`,
|
|
1572
|
-
`measured warm ${gradient(`${GLM52_G5_MEASURED_WARM_DECODE_TPS.toFixed(3)} tok/s`)} ${style.dim("(quality-preserving, MTP on)")}`,
|
|
1573
|
-
`direct oracle ${GLM52_G5_DIRECT_ORACLE_WARM_DECODE_TPS.toFixed(2)} tok/s`,
|
|
1574
|
-
`aspirational ${GLM52_G5_ASPIRATIONAL_DECODE_TPS.toFixed(1)} tok/s ${style.dim("(not a fit prediction or release gate)")}`,
|
|
1575
|
-
]);
|
|
1576
|
-
if (flag("skus")) {
|
|
1577
|
-
console.log(style.dim(
|
|
1578
|
-
" GLM-5.2's direct-container preset is validated on the M1 Max 32 GB; " +
|
|
1579
|
-
"the generic resident-weight SKU projection does not apply.",
|
|
1580
|
-
));
|
|
1581
|
-
}
|
|
1582
|
-
console.log();
|
|
1583
|
-
break;
|
|
1584
|
-
}
|
|
1585
|
-
// --kv-quant mirrors serve: bill the quantized cache's true bytes so
|
|
1586
|
-
// the reported window matches what serving with the same flag admits.
|
|
1587
|
-
const kvQuantOpt = opt("kv-quant");
|
|
1588
|
-
let fitKvScheme: FitKvScheme | undefined;
|
|
1589
|
-
if (kvQuantOpt === "4" || kvQuantOpt === "8") {
|
|
1590
|
-
fitKvScheme = resolveKvScheme({ override: Number(kvQuantOpt) }).fitOptions;
|
|
1591
|
-
} else if (kvQuantOpt === "config") {
|
|
1592
|
-
if (!config.kvQuant?.length) {
|
|
1593
|
-
console.error(`${m.repoId} ships no per-layer kv-quant config — use --kv-quant 4|8`);
|
|
1594
|
-
process.exit(1);
|
|
1595
|
-
}
|
|
1596
|
-
fitKvScheme = resolveKvScheme({
|
|
1597
|
-
override: "config",
|
|
1598
|
-
config: config.kvQuant,
|
|
1599
|
-
missingConfig: "error",
|
|
1600
|
-
}).fitOptions;
|
|
1601
|
-
} else if (kvQuantOpt != null && kvQuantOpt !== "off") {
|
|
1602
|
-
console.error(`--kv-quant ${kvQuantOpt}: expected 4, 8, config, or off`);
|
|
1603
|
-
process.exit(1);
|
|
1604
|
-
}
|
|
1605
|
-
const r = fit(config, m.sizeBytes, ctx, thisMachine(), undefined, m.expertsBytes, undefined, fitKvScheme);
|
|
1606
|
-
h1("will it fit?");
|
|
1607
|
-
const kvNote = fitKvScheme
|
|
1608
|
-
? style.dim(` · kv-quant ${fitKvScheme.kvBits ? `${fitKvScheme.kvBits}-bit` : "config"}`)
|
|
1609
|
-
: "";
|
|
1610
|
-
console.log(` ${style.bold(m.repoId)} ${style.dim(`@ ${ctx.toLocaleString()} context · this machine`)}${kvNote}`);
|
|
1611
|
-
console.log();
|
|
1612
|
-
const expertsNote = config.text.enableMoeBlock && m.expertsBytes > 0
|
|
1613
|
-
? style.dim(` (experts ${gb(m.expertsBytes)}; top ${config.text.topKExperts}/${config.text.numExperts} read per token)`)
|
|
1614
|
-
: "";
|
|
1615
|
-
box([
|
|
1616
|
-
`weights ${gb(r.weightsBytes).padStart(9)}${expertsNote}`,
|
|
1617
|
-
...(m.sidecarBytes > 0
|
|
1618
|
-
? [style.dim(` + vision sidecar ${gb(m.sidecarBytes)} (bf16, loads only for vision)`)] : []),
|
|
1619
|
-
`kv cache ${gb(r.kvBytes).padStart(9)}`,
|
|
1620
|
-
`transient ${gb(r.transientBytes).padStart(9)}`,
|
|
1621
|
-
`total ${gb(r.totalBytes).padStart(9)} ${style.dim(`of ${gb(r.usableBytes)} usable`)} ${r.fits ? style.green(style.bold("FITS")) : style.bold("DOES NOT FIT")}`,
|
|
1622
|
-
"",
|
|
1623
|
-
`max safe context ${style.bold(r.maxSafeContext.toLocaleString())} tokens`,
|
|
1624
|
-
`predicted decode ${gradient(`${r.predictedDecodeTps.toFixed(1)} tok/s`)}`,
|
|
1625
|
-
]);
|
|
1626
|
-
if (flag("skus")) {
|
|
1627
|
-
h1("apple silicon matrix");
|
|
1628
|
-
console.log();
|
|
1629
|
-
table(
|
|
1630
|
-
[
|
|
1631
|
-
{ header: "chip" }, { header: "ram", align: "right" },
|
|
1632
|
-
{ header: "fits", paint: (c) => (c.includes("fits") ? style.green(c) : style.dim(c)) },
|
|
1633
|
-
{ header: "max context", align: "right" }, { header: "decode", align: "right" },
|
|
1634
|
-
],
|
|
1635
|
-
skuMatrix(config, m.sizeBytes, ctx, m.expertsBytes, fitKvScheme).map((row) => [
|
|
1636
|
-
row.sku, `${row.ramGB} GB`, row.fits ? "fits" : "—",
|
|
1637
|
-
row.fits ? row.maxContext.toLocaleString() : "—",
|
|
1638
|
-
row.fits ? `~${row.decodeTps.toFixed(0)} tok/s` : "—",
|
|
1639
|
-
]),
|
|
1640
|
-
);
|
|
1641
|
-
}
|
|
1642
|
-
console.log();
|
|
1643
|
-
break;
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
case "serve": {
|
|
1647
|
-
const { banner, step, box, style } = await import("./tui");
|
|
1648
|
-
banner(pkg.version);
|
|
1649
|
-
if (noSubcommand)
|
|
1650
|
-
console.log(style.dim(" no command given — starting the server · mlx-bun --help for all commands"));
|
|
1651
|
-
const rt = serverRuntimeFlags();
|
|
1652
|
-
// Friendly collision check before loading gigabytes of weights.
|
|
1653
|
-
if (!rt.serverOptions.unixSocket) {
|
|
1654
|
-
const { probeServer } = await import("./harness-pi");
|
|
1655
|
-
const running = await probeServer(`http://localhost:${rt.port}/v1`);
|
|
1656
|
-
if (running) {
|
|
1657
|
-
console.error(`port ${rt.port} is already serving ${running.map((m) => m.id).join(", ")}.`);
|
|
1658
|
-
console.error("reuse it (mlx-bun pi attaches automatically), stop it, or pick --port <other>.");
|
|
1659
|
-
console.error("NOTE: a second server is a second model in memory — check `mlx-bun fit` first.");
|
|
1660
|
-
process.exit(1);
|
|
1661
|
-
}
|
|
1662
|
-
}
|
|
1663
|
-
// Runtime isolation (--isolate, docs/reference/server-config.md): THIS process
|
|
1664
|
-
// becomes a pure reactor — it binds the TCP port and reverse-proxies to
|
|
1665
|
-
// an engine CHILD (the full server on a unix socket, spawned from the
|
|
1666
|
-
// same argv minus parent-only flags). Zero MLX calls in the parent: the
|
|
1667
|
-
// UI stays instant under any GPU load, and an engine crash respawns
|
|
1668
|
-
// without taking the session down.
|
|
1669
|
-
if (!flag("isolate") && opt("model-pool") !== null)
|
|
1670
|
-
console.warn("--model-pool has no effect without --isolate (child-per-model pool) — ignored");
|
|
1671
|
-
if (flag("isolate")) {
|
|
1672
|
-
const { startProxyServer, engineArgvForModel, defaultSocketPath } = await import("./serve/isolate");
|
|
1673
|
-
const sock = defaultSocketPath();
|
|
1674
|
-
// Re-spawn ourselves: compiled binary = execPath alone (argv[1] is a
|
|
1675
|
-
// $bunfs virtual path); dev = bun + the script path.
|
|
1676
|
-
const selfArgv = process.argv[1] && !process.argv[1].includes("$bunfs")
|
|
1677
|
-
? [process.execPath, process.argv[1]]
|
|
1678
|
-
: [process.execPath];
|
|
1679
|
-
const rawArgs = process.argv.slice(2);
|
|
1680
|
-
// Resolve the DEFAULT model in the parent (registry lookup — no
|
|
1681
|
-
// weights load) so the pool has its key and the child gets a pinned
|
|
1682
|
-
// --model path instead of re-running fuzzy resolution.
|
|
1683
|
-
const { m: m0 } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
|
|
1684
|
-
// STRICT resolver for the pool (P2): only EXACT /v1/models ids
|
|
1685
|
-
// switch models. Fuzzy matching here would hijack drop-in clients
|
|
1686
|
-
// that send model:"gpt-4"-style strings (which must keep today's
|
|
1687
|
-
// ignored-field semantics and ride the default engine).
|
|
1688
|
-
const { Registry } = await import("./registry");
|
|
1689
|
-
const poolResolve = (query: string): { repoId: string; path: string } | null => {
|
|
1690
|
-
if (query === m0.repoId) return { repoId: m0.repoId, path: m0.path };
|
|
1691
|
-
try {
|
|
1692
|
-
const reg = new Registry();
|
|
1693
|
-
try {
|
|
1694
|
-
const hit = reg.listCanonical().find((r) => r.repoId === query);
|
|
1695
|
-
return hit ? { repoId: hit.repoId, path: hit.path } : null;
|
|
1696
|
-
} finally {
|
|
1697
|
-
reg.close();
|
|
1698
|
-
}
|
|
1699
|
-
} catch {
|
|
1700
|
-
return null;
|
|
1701
|
-
}
|
|
1702
|
-
};
|
|
1703
|
-
const poolMaxRaw = opt("model-pool");
|
|
1704
|
-
const poolMax = Math.max(1, Number(poolMaxRaw ?? 1) || 1);
|
|
1705
|
-
const { engine, close } = startProxyServer({
|
|
1706
|
-
port: rt.port,
|
|
1707
|
-
...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
|
|
1708
|
-
engine: {
|
|
1709
|
-
argv: [...selfArgv, ...engineArgvForModel(rawArgs, sock, m0.path)],
|
|
1710
|
-
socketPath: sock,
|
|
1711
|
-
},
|
|
1712
|
-
pool: {
|
|
1713
|
-
rawArgs, selfArgv, poolMax,
|
|
1714
|
-
resolve: poolResolve,
|
|
1715
|
-
defaultKey: m0.repoId,
|
|
1716
|
-
},
|
|
1717
|
-
});
|
|
1718
|
-
const shownHost0 = rt.serverOptions.hostname ?? "localhost";
|
|
1719
|
-
console.log(style.dim(` isolated mode: engine child starting (socket ${sock})`));
|
|
1720
|
-
console.log(` ${style.green("●")} ${style.bold("proxy up")} http://${shownHost0}:${rt.port} ${style.dim(`· ${m0.repoId} loading behind it · model pool ${poolMax}`)}`);
|
|
1721
|
-
const onExit = () => { void close().finally(() => process.exit(0)); };
|
|
1722
|
-
process.on("SIGINT", onExit);
|
|
1723
|
-
process.on("SIGTERM", onExit);
|
|
1724
|
-
await engine.ready;
|
|
1725
|
-
console.log(` ${style.green("●")} ${style.bold("engine ready")} ${style.dim(`pid ${engine.pid}`)}`);
|
|
1726
|
-
await new Promise(() => {}); // serve until killed
|
|
1727
|
-
}
|
|
1728
|
-
// --model <path-or-query> is the explicit override (mlx_lm.server's
|
|
1729
|
-
// spelling — previously accepted-but-IGNORED here, which silently
|
|
1730
|
-
// auto-picked the wrong model). A directory with config.json loads
|
|
1731
|
-
// straight from that path; anything else resolves like the positional
|
|
1732
|
-
// fuzzy query. Precedence: --model > positional > --query > auto-pick.
|
|
1733
|
-
const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
|
|
1734
|
-
if (m.modelType === "whisper") {
|
|
1735
|
-
// Transcription-only server: no chat model, weights paged in on demand.
|
|
1736
|
-
const sNative0 = step("native runtime");
|
|
1737
|
-
await ensureNative(sNative0);
|
|
1738
|
-
sNative0.done("native runtime ready");
|
|
1739
|
-
const { createTranscriptionServer } = await import("./serve/transcription-server");
|
|
1740
|
-
const { server, service } = await createTranscriptionServer({
|
|
1741
|
-
modelDir: m.path, modelId: m.repoId, port: rt.port,
|
|
1742
|
-
...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
|
|
1743
|
-
...(rt.serverOptions.unixSocket ? { unixSocket: rt.serverOptions.unixSocket } : {}),
|
|
1744
|
-
idleUnloadSec: rt.serverOptions.whisperIdleUnloadSec,
|
|
1745
|
-
resident: rt.serverOptions.whisperResident,
|
|
1746
|
-
preload: flag("preload"),
|
|
1747
|
-
});
|
|
1748
|
-
const where = rt.serverOptions.unixSocket ?? `http://${rt.serverOptions.hostname === "0.0.0.0" ? "localhost" : rt.serverOptions.hostname}:${server.port}`;
|
|
1749
|
-
box([
|
|
1750
|
-
`${style.bold(m.repoId)} ${style.dim("· transcription-only server")}`,
|
|
1751
|
-
`POST ${where}/v1/audio/transcriptions ${style.dim(`· ${service.stats.idle_unload_sec === null ? "always resident" : service.idleUnloadSec === 0 ? "released after every take" : `idle unload ${service.idleUnloadSec}s`}${service.resident ? " · loaded" : " · loads on first request"}`)}`,
|
|
1752
|
-
]);
|
|
1753
|
-
const stop = () => { service.dispose(); server.stop(true); process.exit(0); };
|
|
1754
|
-
process.once("SIGINT", stop);
|
|
1755
|
-
process.once("SIGTERM", stop);
|
|
1756
|
-
break;
|
|
1757
|
-
}
|
|
1758
|
-
const whisperQuery = opt("whisper-model");
|
|
1759
|
-
if (whisperQuery !== null) {
|
|
1760
|
-
const { m: wm } = await resolveModelAuto(whisperQuery);
|
|
1761
|
-
if (wm.modelType !== "whisper") {
|
|
1762
|
-
console.error(`--whisper-model ${whisperQuery} resolved to ${wm.repoId} (model_type ${wm.modelType}), not a Whisper checkpoint`);
|
|
1763
|
-
process.exit(1);
|
|
1764
|
-
}
|
|
1765
|
-
rt.serverOptions.whisperModelDir = wm.path;
|
|
1766
|
-
rt.serverOptions.whisperModelId = wm.repoId;
|
|
1767
|
-
}
|
|
1768
|
-
const sFit = step("assessing fit");
|
|
1769
|
-
const config = await loadModelConfig(m.path);
|
|
1770
|
-
const report = fit(config, m.sizeBytes, 8192, undefined, undefined, m.expertsBytes);
|
|
1771
|
-
sFit.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)} · ~${report.predictedDecodeTps.toFixed(0)} tok/s predicted`)}${picked ? style.dim(" · auto-picked (override: mlx-bun serve <query>)") : ""}`);
|
|
1772
|
-
const sNative = step("native runtime");
|
|
1773
|
-
await ensureNative(sNative);
|
|
1774
|
-
sNative.done("native runtime ready");
|
|
1775
|
-
const { createServer, loadContext, shutdownServer } = await import("./server");
|
|
1776
|
-
if (flag("expert-offload")) {
|
|
1777
|
-
if (m.expertsBytes === 0) {
|
|
1778
|
-
console.error(style.dim("--expert-offload ignored: this model has no experts (dense)"));
|
|
1779
|
-
} else {
|
|
1780
|
-
const sOff = step("expert offload");
|
|
1781
|
-
const { ensureOffloadFile } = await import("./expert-offload-build");
|
|
1782
|
-
const { activateExpertOffload } = await import("./expert-offload");
|
|
1783
|
-
// builds <model>/.mlx-bun-offload on first use (~one-time), then reuses
|
|
1784
|
-
const dir = await ensureOffloadFile(m.path, (msg) => sOff.update(msg));
|
|
1785
|
-
activateExpertOffload(dir);
|
|
1786
|
-
sOff.done(`experts mmap'd ${style.dim(dir)} ${style.dim("· phys_footprint ≈ core")}`);
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
// Speculative decoding (mlx_lm.server parity): --draft-model <path|query>
|
|
1790
|
-
// resolves like the main model; --num-draft-tokens defaults to 3 (the
|
|
1791
|
-
// server's default upstream — mlx_lm.generate's is 2). Qualified providers
|
|
1792
|
-
// use shared execution; ordinary requests retain their own placement.
|
|
1793
|
-
const draftQuery = opt("draft-model");
|
|
1794
|
-
let draftModelDir: string | undefined;
|
|
1795
|
-
if (draftQuery !== null) {
|
|
1796
|
-
const { m: dm } = await resolveModelAuto(draftQuery);
|
|
1797
|
-
draftModelDir = dm.path;
|
|
1798
|
-
}
|
|
1799
|
-
const numDraftRaw = opt("num-draft-tokens");
|
|
1800
|
-
const numDraftTokens = numDraftRaw !== null ? Number(numDraftRaw) : undefined;
|
|
1801
|
-
if (numDraftTokens !== undefined && (!Number.isInteger(numDraftTokens) || numDraftTokens < 1)) {
|
|
1802
|
-
console.error(`--num-draft-tokens expects an integer >= 1 (got "${numDraftRaw}")`);
|
|
1803
|
-
process.exit(1);
|
|
1804
|
-
}
|
|
1805
|
-
const draftKindRaw = opt("draft-kind");
|
|
1806
|
-
const draftKind = (draftKindRaw ?? undefined) as
|
|
1807
|
-
| "dspark" | "deepspec" | "assistant" | "two-model" | "ngram" | "mtp" | undefined;
|
|
1808
|
-
if (draftKind !== undefined && !["dspark", "deepspec", "assistant", "two-model", "ngram", "mtp"].includes(draftKind)) {
|
|
1809
|
-
console.error(`--draft-kind expects two-model|assistant|dspark|deepspec|mtp|ngram (got "${draftKindRaw}")`);
|
|
1810
|
-
process.exit(1);
|
|
1811
|
-
}
|
|
1812
|
-
// Prompt-lookup window bounds (ngram kind only; validated here so a typo
|
|
1813
|
-
// fails the launch, not the first request).
|
|
1814
|
-
const ngramInt = (name: string): number | undefined => {
|
|
1815
|
-
const raw = opt(name);
|
|
1816
|
-
if (raw === null) return undefined;
|
|
1817
|
-
const v = Number(raw);
|
|
1818
|
-
if (!Number.isInteger(v) || v < 1) {
|
|
1819
|
-
console.error(`--${name} expects an integer >= 1 (got "${raw}")`);
|
|
1820
|
-
process.exit(1);
|
|
1821
|
-
}
|
|
1822
|
-
return v;
|
|
1823
|
-
};
|
|
1824
|
-
const ngramMax = ngramInt("ngram-max");
|
|
1825
|
-
const ngramMin = ngramInt("ngram-min");
|
|
1826
|
-
if (ngramMax !== undefined && ngramMin !== undefined && ngramMin > ngramMax) {
|
|
1827
|
-
console.error(`--ngram-min (${ngramMin}) must be <= --ngram-max (${ngramMax})`);
|
|
1828
|
-
process.exit(1);
|
|
1829
|
-
}
|
|
1830
|
-
if ((ngramMax !== undefined || ngramMin !== undefined) && draftKind !== "ngram")
|
|
1831
|
-
console.warn("--ngram-max/--ngram-min only apply with --draft-kind ngram — ignored");
|
|
1832
|
-
const sLoad = step("loading weights");
|
|
1833
|
-
const t0 = performance.now();
|
|
1834
|
-
const ctx = await loadContext(m.path, m.repoId, {
|
|
1835
|
-
memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
|
|
1836
|
-
glm: glm52RuntimeFlags(rt.serverOptions),
|
|
1837
|
-
// ngram is model-free; mtp resolves the bundled <model>/mtp/ companion
|
|
1838
|
-
// in loadContext when no --draft-model is given (cli.md contract —
|
|
1839
|
-
// this gate silently dropped that case until 2026-08-20).
|
|
1840
|
-
...(draftModelDir || draftKind === "ngram" || draftKind === "mtp"
|
|
1841
|
-
? {
|
|
1842
|
-
...(draftModelDir ? { draftModelDir } : {}),
|
|
1843
|
-
numDraftTokens,
|
|
1844
|
-
...(draftKind ? { draftKind } : {}),
|
|
1845
|
-
...(ngramMax !== undefined ? { ngramMax } : {}),
|
|
1846
|
-
...(ngramMin !== undefined ? { ngramMin } : {}),
|
|
1847
|
-
}
|
|
1848
|
-
: {}),
|
|
1849
|
-
});
|
|
1850
|
-
sLoad.done(`weights loaded ${style.dim(`in ${(performance.now() - t0).toFixed(0)} ms`)}${draftModelDir ? style.dim(` · draft: ${draftModelDir.split("/").filter(Boolean).at(-1)}`) : draftKind === "ngram" ? style.dim(" · draft: ngram (prompt lookup)") : ""}`);
|
|
1851
|
-
await mountStartupAdapter(ctx, rt.serverOptions);
|
|
1852
|
-
const server = createServer(ctx, rt.port, { ...rt.serverOptions, owner: "serve" });
|
|
1853
|
-
let shutdownStarted = false;
|
|
1854
|
-
const onServeSignal = (signal: "SIGINT" | "SIGTERM"): void => {
|
|
1855
|
-
if (shutdownStarted) return;
|
|
1856
|
-
shutdownStarted = true;
|
|
1857
|
-
const rawTimeout = Number(runtimeValue("MLX_BUN_SHUTDOWN_TIMEOUT_MS"));
|
|
1858
|
-
const timeoutMs = Number.isFinite(rawTimeout) && rawTimeout > 0 ? rawTimeout : 120_000;
|
|
1859
|
-
console.log(`\n[server] ${signal}: flushing prompt-cache durability`);
|
|
1860
|
-
void shutdownServer(server, timeoutMs).then((result) => {
|
|
1861
|
-
if (!result.durability.durable) {
|
|
1862
|
-
const d = result.durability;
|
|
1863
|
-
console.warn(
|
|
1864
|
-
`[server] cache flush incomplete after ${Math.round(d.elapsedMs)} ms: ` +
|
|
1865
|
-
`${d.pendingSnapshots} snapshots, ${d.pendingSpills} spills, ` +
|
|
1866
|
-
`${d.droppedSpills} dropped, ${d.failedSpills} failed`,
|
|
1867
|
-
);
|
|
1868
|
-
}
|
|
1869
|
-
process.exit(0);
|
|
1870
|
-
}).catch((error) => {
|
|
1871
|
-
console.warn(`[server] shutdown flush failed: ${(error as Error).message}`);
|
|
1872
|
-
process.exit(0);
|
|
1873
|
-
});
|
|
1874
|
-
};
|
|
1875
|
-
process.once("SIGINT", () => onServeSignal("SIGINT"));
|
|
1876
|
-
process.once("SIGTERM", () => onServeSignal("SIGTERM"));
|
|
1877
|
-
const shownHost = rt.serverOptions.hostname ?? "localhost";
|
|
1878
|
-
console.log();
|
|
1879
|
-
box([
|
|
1880
|
-
`${style.green("●")} ${style.bold("serving")} ${m.repoId}`,
|
|
1881
|
-
"",
|
|
1882
|
-
`API ${style.url(`http://${shownHost}:${server.port}/v1`)} ${style.dim("(OpenAI · Anthropic · Responses)")}`,
|
|
1883
|
-
`App ${style.url(`http://${shownHost}:${server.port}/`)} ${style.dim("(chat · quantize · finetune · dataset · status)")}`,
|
|
1884
|
-
...(rt.serverOptions.memoryBudgetBytes
|
|
1885
|
-
? [`Mem ${gb(rt.serverOptions.memoryBudgetBytes)} budget ${style.dim("(admission control on)")}`] : []),
|
|
1886
|
-
`Perf ${style.dim(runtimeSummary(rt.serverOptions))}`,
|
|
1887
|
-
"",
|
|
1888
|
-
style.dim("agent session: mlx-bun pi stop: Ctrl+C"),
|
|
1889
|
-
]);
|
|
1890
|
-
// Open the chat UI for interactive runs (--no-open, or a non-TTY such as
|
|
1891
|
-
// a piped/headless/benchmark run, skips it). 0.0.0.0/:: bind → localhost
|
|
1892
|
-
// for the browser. Reuses an existing tab on this host:port when possible
|
|
1893
|
-
// (see openChatUi); the URL is printed above either way.
|
|
1894
|
-
if (!flag("no-open") && process.stdout.isTTY) {
|
|
1895
|
-
const browserHost = shownHost === "0.0.0.0" || shownHost === "::" ? "localhost" : shownHost;
|
|
1896
|
-
openChatUi(`http://${browserHost}:${server.port}/#/chat`, `${browserHost}:${server.port}`);
|
|
1897
|
-
}
|
|
1898
|
-
break;
|
|
1899
|
-
}
|
|
1900
|
-
|
|
1901
|
-
case "gen":
|
|
1902
|
-
case "generate": {
|
|
1903
|
-
// RAW generate — a direct one-shot entry point (no server, no chat UI). The
|
|
1904
|
-
// three entry points (raw / OpenAI API / chat UI) differ only in how params
|
|
1905
|
-
// are POPULATED; here every param is explicit. Decode-path levers mirror
|
|
1906
|
-
// serve so you can pin the route (mlx-lm compat = --l1).
|
|
1907
|
-
// --l1/--l2 + per-fork overrides set the decode runtime options; the
|
|
1908
|
-
// returned KV scheme is applied below so `generate --l2` runs the same
|
|
1909
|
-
// quantized-KV route (and produces the same tokens) as `serve --l2`.
|
|
1910
|
-
const route = applyDecodeRoute();
|
|
1911
|
-
|
|
1912
|
-
const prompt = opt("prompt") ?? positional(1);
|
|
1913
|
-
if (!prompt) {
|
|
1914
|
-
console.error('usage: mlx-bun generate [query] --prompt "…" [--raw] [--max-tokens N]');
|
|
1915
|
-
console.error(' sampling: --temperature N (alias --temp) --top-p N --top-k N --seed N');
|
|
1916
|
-
console.error(' decode path: --l1 (= mlx-lm compat, the default) / --l2, or per-fork flags');
|
|
1917
|
-
process.exit(1);
|
|
1918
|
-
}
|
|
1919
|
-
const { loadTaskModel, generateText } = await import("./eval/runner");
|
|
1920
|
-
const maxTokens = Number(opt("max-tokens", "256"));
|
|
1921
|
-
const tm = await loadTaskModel(
|
|
1922
|
-
positional(0) ?? opt("query") ?? "",
|
|
1923
|
-
undefined,
|
|
1924
|
-
{ maxGenerationTokens: maxTokens, enableMtp: false },
|
|
1925
|
-
);
|
|
1926
|
-
// Resolve the tier/--kv-quant KV scheme exactly the way serve does
|
|
1927
|
-
// (server.ts kvScheme): "config" = the model's kv_config.json (absent →
|
|
1928
|
-
// bf16, same silent fallback as serve), N = uniform bits from decode
|
|
1929
|
-
// start. Unset (no tier, no --kv-quant) keeps generate's historical
|
|
1930
|
-
// default: bf16 bit-exact greedy (generateText's parity path).
|
|
1931
|
-
const kvScheme = route.turboQuant || route.kvQuant !== undefined
|
|
1932
|
-
? resolveKvScheme({
|
|
1933
|
-
override: route.kvQuant,
|
|
1934
|
-
turboQuant: route.turboQuant,
|
|
1935
|
-
config: tm.config.kvQuant,
|
|
1936
|
-
}).generationOptions
|
|
1937
|
-
: undefined;
|
|
1938
|
-
const num = (n: string): number | undefined => { const v = opt(n); return v == null ? undefined : Number(v); };
|
|
1939
|
-
const text = await generateText(tm, prompt, {
|
|
1940
|
-
maxTokens,
|
|
1941
|
-
useChat: !flag("raw"),
|
|
1942
|
-
...(kvScheme !== undefined ? { kvScheme } : {}),
|
|
1943
|
-
sampler: { temperature: num("temperature") ?? num("temp"), topP: num("top-p"), topK: num("top-k"), seed: num("seed") },
|
|
1944
|
-
});
|
|
1945
|
-
process.stdout.write(text.endsWith("\n") ? text : text + "\n");
|
|
1946
|
-
break;
|
|
1947
|
-
}
|
|
1948
|
-
|
|
1949
|
-
case "embed": {
|
|
1950
|
-
// Raw text embeddings — one-shot, no server. Text from --text, or one per
|
|
1951
|
-
// line on stdin. Uses the same embedPooled path the parity test verifies.
|
|
1952
|
-
const { Weights } = await import("./weights");
|
|
1953
|
-
const { createModel } = await import("./model/factory");
|
|
1954
|
-
const { loadTokenizer } = await import("./tokenizer");
|
|
1955
|
-
const { embedMany, isEmbeddingModel } = await import("./embed");
|
|
1956
|
-
|
|
1957
|
-
let texts: string[];
|
|
1958
|
-
const oneText = opt("text") ?? positional(1);
|
|
1959
|
-
if (oneText != null) {
|
|
1960
|
-
texts = [oneText];
|
|
1961
|
-
} else if (!process.stdin.isTTY) {
|
|
1962
|
-
texts = (await Bun.stdin.text()).split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
|
|
1963
|
-
} else {
|
|
1964
|
-
texts = [];
|
|
1965
|
-
}
|
|
1966
|
-
if (texts.length === 0) {
|
|
1967
|
-
console.error('usage: mlx-bun embed [query] --text "…" (or pipe text, one per line)');
|
|
1968
|
-
console.error(' --instruct "…" query instruction · --json OpenAI-style output');
|
|
1969
|
-
process.exit(1);
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
|
-
// Model resolution: an explicit query resolves normally; the no-query
|
|
1973
|
-
// path auto-picks the first DOWNLOADED embedding model (plain Qwen3 —
|
|
1974
|
-
// the record-level equivalent of isEmbeddingModel, which the factory
|
|
1975
|
-
// maps to Qwen3Model). Never the chat default, and never the fresh-
|
|
1976
|
-
// install starter download — a chat model can't embed, so downloading
|
|
1977
|
-
// one here only to fail the isEmbeddingModel check would be a trap.
|
|
1978
|
-
const embedQuery = positional(0) ?? opt("query");
|
|
1979
|
-
let m: import("./registry").ModelRecord;
|
|
1980
|
-
if (embedQuery) {
|
|
1981
|
-
({ m } = await resolveModelAuto(embedQuery));
|
|
1982
|
-
} else {
|
|
1983
|
-
const reg = new Registry();
|
|
1984
|
-
if (reg.list().length === 0) await reg.scan();
|
|
1985
|
-
const embedder = reg.list().find((r) => r.modelType === "qwen3");
|
|
1986
|
-
if (!embedder) {
|
|
1987
|
-
console.error("no embedding model downloaded — try: mlx-bun get mlx-community/Qwen3-Embedding-4B-4bit-DWQ");
|
|
1988
|
-
process.exit(1);
|
|
1989
|
-
}
|
|
1990
|
-
m = embedder;
|
|
1991
|
-
}
|
|
1992
|
-
const config = await loadModelConfig(m.path);
|
|
1993
|
-
const model = createModel(await Weights.open(m.path), config);
|
|
1994
|
-
if (!isEmbeddingModel(model)) {
|
|
1995
|
-
console.error(`"${m.repoId}" is not an embedding model (need plain Qwen3, e.g. Qwen3-Embedding).`);
|
|
1996
|
-
process.exit(1);
|
|
1997
|
-
}
|
|
1998
|
-
const tok = await loadTokenizer(m.path);
|
|
1999
|
-
const results = embedMany(model, tok, texts, opt("instruct") ?? undefined);
|
|
2000
|
-
|
|
2001
|
-
if (flag("json")) {
|
|
2002
|
-
let total = 0;
|
|
2003
|
-
const data = results.map((r, index) => { total += r.tokens; return { object: "embedding", index, embedding: Array.from(r.vector) }; });
|
|
2004
|
-
process.stdout.write(JSON.stringify({
|
|
2005
|
-
object: "list", data, model: m.repoId,
|
|
2006
|
-
usage: { prompt_tokens: total, total_tokens: total },
|
|
2007
|
-
}) + "\n");
|
|
2008
|
-
} else {
|
|
2009
|
-
for (const r of results) process.stdout.write(JSON.stringify(Array.from(r.vector)) + "\n");
|
|
2010
|
-
}
|
|
2011
|
-
break;
|
|
2012
|
-
}
|
|
2013
|
-
|
|
2014
|
-
case "transcribe": {
|
|
2015
|
-
// Speech-to-text, one-shot, no server: audio file → Whisper → text.
|
|
2016
|
-
const { openWhisperModel } = await import("./model/factory");
|
|
2017
|
-
const { loadWhisperTokenizer } = await import("./audio/whisper-tokenizer");
|
|
2018
|
-
const { WhisperTranscriber } = await import("./audio/whisper-transcribe");
|
|
2019
|
-
const { formatTranscription, TRANSCRIPTION_FORMATS } = await import("./audio/whisper-format");
|
|
2020
|
-
const { TranscriptionService } = await import("./serve/transcription-service");
|
|
2021
|
-
const { formatTimestamp } = await import("./audio/whisper-format");
|
|
2022
|
-
|
|
2023
|
-
const audioPath = positional(0);
|
|
2024
|
-
if (!audioPath) {
|
|
2025
|
-
console.error("usage: mlx-bun transcribe <audio-file> [query] [--language en] [--format text|json|verbose_json|srt|vtt]");
|
|
2026
|
-
process.exit(1);
|
|
2027
|
-
}
|
|
2028
|
-
const format = (opt("format") ?? "text") as import("./audio/whisper-format").TranscriptionFormat;
|
|
2029
|
-
if (!TRANSCRIPTION_FORMATS.includes(format)) {
|
|
2030
|
-
console.error(`--format must be one of ${TRANSCRIPTION_FORMATS.join(", ")}`);
|
|
2031
|
-
process.exit(1);
|
|
2032
|
-
}
|
|
2033
|
-
const bytes = new Uint8Array(await Bun.file(audioPath).arrayBuffer());
|
|
2034
|
-
const samples = await TranscriptionService.decodeAudio(bytes);
|
|
2035
|
-
const durationSeconds = samples.length / 16_000;
|
|
2036
|
-
|
|
2037
|
-
const modelQuery = opt("model") ?? positional(1) ?? opt("query");
|
|
2038
|
-
let modelDir: string;
|
|
2039
|
-
if (modelQuery && existsSync(join(modelQuery, "config.json"))) modelDir = modelQuery;
|
|
2040
|
-
else if (modelQuery) modelDir = (await resolveModelAuto(modelQuery)).m.path;
|
|
2041
|
-
else {
|
|
2042
|
-
const reg = new Registry();
|
|
2043
|
-
if (reg.list().length === 0) await reg.scan();
|
|
2044
|
-
const w = reg.list().find((r) => r.modelType === "whisper");
|
|
2045
|
-
if (!w) {
|
|
2046
|
-
console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo");
|
|
2047
|
-
process.exit(1);
|
|
2048
|
-
}
|
|
2049
|
-
modelDir = w.path;
|
|
2050
|
-
}
|
|
2051
|
-
let vadGate: { segments: { start: number; end: number }[] } | null = null;
|
|
2052
|
-
if (flag("vad")) {
|
|
2053
|
-
const { SileroVad } = await import("./audio/silero-vad");
|
|
2054
|
-
const tv = performance.now();
|
|
2055
|
-
const vad = SileroVad.load(opt("vad-model"));
|
|
2056
|
-
const { segments } = vad.detect(samples, { threshold: Number(opt("vad-threshold") ?? 0.5) });
|
|
2057
|
-
vadGate = { segments };
|
|
2058
|
-
if (flag("verbose")) console.error(`vad: ${segments.length} speech segment(s) in ${(performance.now() - tv).toFixed(0)} ms`);
|
|
2059
|
-
if (segments.length === 0) { process.stdout.write(format === "text" ? "\n" : JSON.stringify({ text: "", vad: { speech: false, segments: [] } }) + "\n"); break; }
|
|
2060
|
-
}
|
|
2061
|
-
const { model } = await openWhisperModel(modelDir);
|
|
2062
|
-
const tok = await loadWhisperTokenizer(modelDir, model.dims.nVocab);
|
|
2063
|
-
const transcriber = new WhisperTranscriber(model, tok);
|
|
2064
|
-
void vadGate;
|
|
2065
|
-
const langOpt = opt("language");
|
|
2066
|
-
const task = opt("task") === "translate" ? "translate" as const : "transcribe" as const;
|
|
2067
|
-
const beam = opt("beam-size");
|
|
2068
|
-
const temperature = opt("temperature");
|
|
2069
|
-
const verbose = flag("verbose");
|
|
2070
|
-
const t0 = performance.now();
|
|
2071
|
-
const result = await transcriber.transcribe(samples, {
|
|
2072
|
-
language: !langOpt || langOpt === "auto" ? null : langOpt,
|
|
2073
|
-
task,
|
|
2074
|
-
beamSize: beam ? Number(beam) : null,
|
|
2075
|
-
...(temperature ? { temperature: Number(temperature) } : flag("no-fallback") ? { temperature: 0 } : {}),
|
|
2076
|
-
initialPrompt: opt("prompt"),
|
|
2077
|
-
withoutTimestamps: flag("no-timestamps"),
|
|
2078
|
-
conditionOnPreviousText: !flag("no-condition"),
|
|
2079
|
-
fast: !flag("faithful"),
|
|
2080
|
-
wordTimestamps: flag("word-timestamps"),
|
|
2081
|
-
audioCtx: opt("audio-ctx") ? Number(opt("audio-ctx")) : null,
|
|
2082
|
-
onSegment: verbose
|
|
2083
|
-
? (s) => console.error(`[${formatTimestamp(s.start, ".")} --> ${formatTimestamp(s.end, ".")}] ${s.text.trim()}`)
|
|
2084
|
-
: undefined,
|
|
2085
|
-
});
|
|
2086
|
-
const elapsed = (performance.now() - t0) / 1000;
|
|
2087
|
-
if (verbose)
|
|
2088
|
-
console.error(`${durationSeconds.toFixed(2)} s audio in ${elapsed.toFixed(3)} s (${(durationSeconds / elapsed).toFixed(1)}× realtime), language ${result.language}`);
|
|
2089
|
-
process.stdout.write(formatTranscription(result, format, durationSeconds, task));
|
|
2090
|
-
transcriber.dispose();
|
|
2091
|
-
model.dispose();
|
|
2092
|
-
break;
|
|
2093
|
-
}
|
|
2094
|
-
|
|
2095
|
-
case "dictate": {
|
|
2096
|
-
const { startMicCapture } = await import("./audio/mic-capture");
|
|
2097
|
-
const { TranscriptionService, defaultWhisperRecord } = await import("./serve/transcription-service");
|
|
2098
|
-
const { style } = await import("./tui");
|
|
2099
|
-
const serverUrl = opt("server");
|
|
2100
|
-
const hotkeyFlag = argv.includes("--hotkey");
|
|
2101
|
-
const hotkey = hotkeyFlag ? Number(opt("hotkey") ?? 61) || 61 : null;
|
|
2102
|
-
const language = opt("language") ?? "en";
|
|
2103
|
-
const beam = opt("beam-size");
|
|
2104
|
-
const vocabulary = opt("vocabulary")?.split(",").map((s) => s.trim()).filter(Boolean);
|
|
2105
|
-
const params = {
|
|
2106
|
-
language: language === "auto" ? null : language,
|
|
2107
|
-
beamSize: beam ? Number(beam) : null,
|
|
2108
|
-
temperature: 0,
|
|
2109
|
-
prompt: opt("prompt"),
|
|
2110
|
-
vocabulary,
|
|
2111
|
-
vad: flag("no-vad") ? null : { threshold: 0.5, minSpeechMs: 120 },
|
|
2112
|
-
};
|
|
2113
|
-
const typeDelay = Number(opt("type-delay") ?? (hotkey ? 0 : 1));
|
|
2114
|
-
|
|
2115
|
-
// backend: in-process service or a running server's sessions
|
|
2116
|
-
let service: InstanceType<typeof TranscriptionService> | null = null;
|
|
2117
|
-
if (!serverUrl) {
|
|
2118
|
-
const q = opt("model") ?? positional(0) ?? opt("query");
|
|
2119
|
-
let modelDir: string; let modelId: string;
|
|
2120
|
-
if (q && existsSync(join(q, "config.json"))) { modelDir = q; modelId = q; }
|
|
2121
|
-
else if (q) { const { m } = await resolveModelAuto(q); modelDir = m.path; modelId = m.repoId; }
|
|
2122
|
-
else {
|
|
2123
|
-
const rec = await defaultWhisperRecord();
|
|
2124
|
-
if (!rec) { console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo"); process.exit(1); }
|
|
2125
|
-
modelDir = rec.path; modelId = rec.repoId;
|
|
2126
|
-
}
|
|
2127
|
-
const idle = opt("idle-unload");
|
|
2128
|
-
service = new TranscriptionService({ modelDir, modelId, idleUnloadSec: idle !== null ? Number(idle) : 30, resident: flag("resident"), log: () => {} });
|
|
2129
|
-
process.stderr.write(style.dim("loading whisper… "));
|
|
2130
|
-
await service.ensureLoaded();
|
|
2131
|
-
service.vad();
|
|
2132
|
-
process.stderr.write(style.dim("ready\n"));
|
|
2133
|
-
}
|
|
2134
|
-
const base = serverUrl?.replace(/\/$/, "");
|
|
2135
|
-
interface Take { feed(pcm: Float32Array): Promise<void>; finish(): Promise<{ text: string; ms: number }>; }
|
|
2136
|
-
const startTake = async (): Promise<Take> => {
|
|
2137
|
-
if (service) {
|
|
2138
|
-
const session = await service.createSession(params);
|
|
2139
|
-
return {
|
|
2140
|
-
feed: async (pcm) => { await session.append(pcm); },
|
|
2141
|
-
finish: async () => { const t0 = performance.now(); const o = await session.finish(); return { text: o.result.text.trim(), ms: performance.now() - t0 }; },
|
|
2142
|
-
};
|
|
2143
|
-
}
|
|
2144
|
-
const r = await fetch(`${base}/v1/audio/sessions`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ language: params.language, beam_size: params.beamSize, temperature: 0, prompt: params.prompt, vocabulary, vad: !!params.vad, vad_min_speech_ms: 120 }) });
|
|
2145
|
-
if (!r.ok) throw new Error(`session create failed: ${r.status} ${await r.text()}`);
|
|
2146
|
-
const { id } = await r.json() as { id: string };
|
|
2147
|
-
return {
|
|
2148
|
-
feed: async (pcm) => { await fetch(`${base}/v1/audio/sessions/${id}/audio`, { method: "POST", headers: { "content-type": "audio/pcm;rate=16000" }, body: new Uint8Array(pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength) as ArrayBuffer) }); },
|
|
2149
|
-
finish: async () => { const t0 = performance.now(); const f = await fetch(`${base}/v1/audio/sessions/${id}/finish`, { method: "POST" }); const b = await f.json() as { text: string }; return { text: (b.text ?? "").trim(), ms: performance.now() - t0 }; },
|
|
2150
|
-
};
|
|
2151
|
-
};
|
|
2152
|
-
|
|
2153
|
-
const deliver = async (text: string) => {
|
|
2154
|
-
if (!text) { console.log(style.dim("(no speech)")); return; }
|
|
2155
|
-
console.log(text);
|
|
2156
|
-
if (flag("copy")) { const p = Bun.spawn(["pbcopy"], { stdin: "pipe" }); p.stdin.write(text); p.stdin.end(); await p.exited; }
|
|
2157
|
-
if (flag("type")) {
|
|
2158
|
-
if (typeDelay > 0) await Bun.sleep(typeDelay * 1000);
|
|
2159
|
-
const escaped = text.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
2160
|
-
const r = Bun.spawnSync(["osascript", "-e", `tell application "System Events" to keystroke "${escaped}"`], { stderr: "pipe" });
|
|
2161
|
-
if (r.exitCode !== 0) console.error(style.dim(`typing failed (grant Accessibility to your terminal): ${r.stderr.toString().trim()}`));
|
|
2162
|
-
}
|
|
2163
|
-
};
|
|
2164
|
-
|
|
2165
|
-
const mic = await startMicCapture({ hotkey });
|
|
2166
|
-
let take: Take | null = null;
|
|
2167
|
-
let pending: Float32Array[] = []; let pendingLen = 0;
|
|
2168
|
-
let feeding: Promise<void> = Promise.resolve();
|
|
2169
|
-
const flush = () => {
|
|
2170
|
-
if (!take || pendingLen === 0) return;
|
|
2171
|
-
const buf = new Float32Array(pendingLen); let o = 0; for (const p of pending) { buf.set(p, o); o += p.length; }
|
|
2172
|
-
pending = []; pendingLen = 0;
|
|
2173
|
-
const t = take;
|
|
2174
|
-
feeding = feeding.then(() => t.feed(buf)).catch((e) => console.error(style.dim(`feed failed: ${(e as Error).message}`)));
|
|
2175
|
-
};
|
|
2176
|
-
const begin = async () => { if (take) return; take = await startTake(); pending = []; pendingLen = 0; process.stderr.write(style.bold("● recording") + style.dim(hotkey ? " (release to stop)\n" : " (Enter to stop)\n")); };
|
|
2177
|
-
const end = async () => {
|
|
2178
|
-
if (!take) return;
|
|
2179
|
-
flush();
|
|
2180
|
-
await feeding;
|
|
2181
|
-
const t = take; take = null;
|
|
2182
|
-
process.stderr.write(style.dim("transcribing… "));
|
|
2183
|
-
try { const { text, ms } = await t.finish(); process.stderr.write(style.dim(`${ms.toFixed(0)} ms\n`)); await deliver(text); }
|
|
2184
|
-
catch (e) { console.error(`transcription failed: ${(e as Error).message}`); }
|
|
2185
|
-
};
|
|
2186
|
-
if (!hotkey) {
|
|
2187
|
-
console.error(style.dim("Enter: start/stop a take · q: quit"));
|
|
2188
|
-
(async () => {
|
|
2189
|
-
for await (const line of console) {
|
|
2190
|
-
if (line.trim() === "q") { mic.stop(); process.exit(0); }
|
|
2191
|
-
if (take) await end(); else await begin();
|
|
2192
|
-
}
|
|
2193
|
-
})();
|
|
2194
|
-
}
|
|
2195
|
-
process.once("SIGINT", () => { mic.stop(); process.exit(0); });
|
|
2196
|
-
for await (const ev of mic.events) {
|
|
2197
|
-
if (ev.kind === "ready") { if (hotkey) console.error(style.dim(`hold key ${hotkey} to talk · Ctrl-C quits · mic ${ev.info}`)); }
|
|
2198
|
-
else if (ev.kind === "error") { console.error(ev.message); mic.stop(); process.exit(1); }
|
|
2199
|
-
else if (ev.kind === "hotkey") { if (ev.down) await begin(); else await end(); }
|
|
2200
|
-
else if (ev.kind === "pcm" && take) {
|
|
2201
|
-
pending.push(ev.samples); pendingLen += ev.samples.length;
|
|
2202
|
-
if (pendingLen >= 16_000 / 4) flush(); // every 250 ms of audio
|
|
2203
|
-
}
|
|
2204
|
-
}
|
|
2205
|
-
break;
|
|
2206
|
-
}
|
|
2207
|
-
|
|
2208
|
-
case "bench":
|
|
2209
|
-
case "benchmark": {
|
|
2210
|
-
const { banner, step, box, style } = await import("./tui");
|
|
2211
|
-
banner(pkg.version);
|
|
2212
|
-
serverRuntimeFlags(); // decode options (--compiled-decode, --fused-sdpa, ...) apply to the run
|
|
2213
|
-
const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
|
|
2214
|
-
const tokens = Number(opt("tokens", "256"));
|
|
2215
|
-
const runs = Number(opt("runs", "3"));
|
|
2216
|
-
const promptTokens = Number(opt("prompt-tokens", "0"));
|
|
2217
|
-
const kvMode = opt("kv-quant", "off")!;
|
|
2218
|
-
console.log(` ${style.dim("measures THIS machine, OUR stack only — no other systems needed.")}`);
|
|
2219
|
-
console.log(` ${style.dim("numbers on a loaded machine are not quotable; bun scripts/bench-serve.ts all is the preflight-gated harness.")}`);
|
|
2220
|
-
const sNative = step("native runtime");
|
|
2221
|
-
await ensureNative(sNative);
|
|
2222
|
-
sNative.done("native runtime ready");
|
|
2223
|
-
const sLoad = step(`loading ${m.repoId}${picked ? " (auto-picked)" : ""}`);
|
|
2224
|
-
const { Weights } = await import("./weights");
|
|
2225
|
-
const { createModel } = await import("./model/factory");
|
|
2226
|
-
const { generate } = await import("./generate");
|
|
2227
|
-
const { ChatTemplate } = await import("./chat-template");
|
|
2228
|
-
const { loadTokenizer } = await import("./tokenizer");
|
|
2229
|
-
const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
|
|
2230
|
-
const config = await loadModelConfig(m.path);
|
|
2231
|
-
const weights = await Weights.open(m.path);
|
|
2232
|
-
const model = createModel(weights, config);
|
|
2233
|
-
const tok = await loadTokenizer(m.path);
|
|
2234
|
-
const template = await ChatTemplate.load(m.path);
|
|
2235
|
-
sLoad.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)}`)}`);
|
|
2236
|
-
|
|
2237
|
-
let userMsg = "Write a detailed essay about the history of computing, starting with mechanical calculators.";
|
|
2238
|
-
if (promptTokens > 0) {
|
|
2239
|
-
const filler = "Background context: the history of computation spans mechanical " +
|
|
2240
|
-
"calculators, electromechanical relays, vacuum tubes, transistors, " +
|
|
2241
|
-
"integrated circuits, and modern accelerators. ";
|
|
2242
|
-
while (tok.encode(userMsg).length < promptTokens - 24) userMsg = filler + userMsg;
|
|
2243
|
-
}
|
|
2244
|
-
const rendered = template.render([{ role: "user", content: userMsg }]);
|
|
2245
|
-
const ids = tok.encode(rendered);
|
|
2246
|
-
const promptIds = ids[0] === ids[1] && ids[0] === tok.bosTokenId ? ids.slice(1) : ids;
|
|
2247
|
-
let turboScheme: TurboQuantScheme | null;
|
|
2248
|
-
try {
|
|
2249
|
-
turboScheme = parseTurboQuantScheme(kvMode);
|
|
2250
|
-
} catch (e) {
|
|
2251
|
-
console.error((e as Error).message);
|
|
2252
|
-
process.exit(1);
|
|
2253
|
-
}
|
|
2254
|
-
const kvOptions = turboScheme
|
|
2255
|
-
? { turboQuant: turboScheme, quantizedKvStart: 0 }
|
|
2256
|
-
: kvMode === "config"
|
|
2257
|
-
? (() => {
|
|
2258
|
-
if (!config.kvQuant?.length) { console.error("model has no kv_config.json (--kv-quant config)"); process.exit(1); }
|
|
2259
|
-
return { kvConfig: config.kvQuant };
|
|
2260
|
-
})()
|
|
2261
|
-
: kvMode !== "off" ? { kvBits: Number(kvMode), quantizedKvStart: 0 } : {};
|
|
2262
|
-
|
|
2263
|
-
// Warmup materializes weights so prefill timing measures prefill,
|
|
2264
|
-
// not lazy page-in (same protocol as bench.ts (deleted 2026-08-23; git history)).
|
|
2265
|
-
const sWarm = step("warmup");
|
|
2266
|
-
{
|
|
2267
|
-
const wCache = model.makeCache();
|
|
2268
|
-
const wGen = generate(model, promptIds.slice(0, Math.min(8, promptIds.length - 1)), {
|
|
2269
|
-
maxTokens: 1, temperature: 0, cache: wCache,
|
|
2270
|
-
});
|
|
2271
|
-
for await (const _ of wGen) { /* discard */ }
|
|
2272
|
-
for (const c of wCache) c.dispose();
|
|
2273
|
-
}
|
|
2274
|
-
resetPeakMemory();
|
|
2275
|
-
sWarm.done("warmup complete");
|
|
2276
|
-
|
|
2277
|
-
const decodes: number[] = [];
|
|
2278
|
-
const prefills: number[] = [];
|
|
2279
|
-
let lastStats: { promptTokens: number; generatedTokens: number } = { promptTokens: 0, generatedTokens: 0 };
|
|
2280
|
-
for (let i = 0; i < runs; i++) {
|
|
2281
|
-
const sRun = step(`run ${i + 1}/${runs} · ${tokens} tokens`);
|
|
2282
|
-
const gen = generate(model, promptIds, { maxTokens: tokens, temperature: 0, ...kvOptions });
|
|
2283
|
-
for await (const _ of gen) { /* timing only */ }
|
|
2284
|
-
const s = gen.stats!;
|
|
2285
|
-
decodes.push(s.decodeTps);
|
|
2286
|
-
prefills.push(s.prefillTps);
|
|
2287
|
-
lastStats = s;
|
|
2288
|
-
sRun.done(`run ${i + 1}/${runs} ${style.dim("·")} ${style.green(style.bold(`${s.decodeTps.toFixed(1)} tok/s`))} ${style.dim(`decode · prefill ${s.prefillTps.toFixed(0)} tok/s`)}`);
|
|
2289
|
-
}
|
|
2290
|
-
const median = (a: number[]) => [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)]!;
|
|
2291
|
-
const peak = peakMemory();
|
|
2292
|
-
|
|
2293
|
-
const { EvalDB: DB, gitCommit } = await import("./evaldb");
|
|
2294
|
-
new DB().record({
|
|
2295
|
-
modelPath: m.path,
|
|
2296
|
-
commitSha: gitCommit() ?? undefined,
|
|
2297
|
-
promptTokens: lastStats.promptTokens, cachedTokens: 0,
|
|
2298
|
-
generatedTokens: lastStats.generatedTokens,
|
|
2299
|
-
prefillTps: median(prefills), decodeTps: median(decodes),
|
|
2300
|
-
peakBytes: peak,
|
|
2301
|
-
notes: `cli-bench median-of-${runs} ${tokens}tok kv=${kvMode}${promptTokens ? ` ctx=${lastStats.promptTokens}` : ""} decode[${decodes.map((d) => d.toFixed(1)).join(",")}]`,
|
|
2302
|
-
});
|
|
2303
|
-
console.log();
|
|
2304
|
-
box([
|
|
2305
|
-
`${style.green("●")} ${style.bold("benchmark complete")} ${style.dim(`· ${m.repoId}`)}`,
|
|
2306
|
-
"",
|
|
2307
|
-
`decode ${style.green(style.bold(`${median(decodes).toFixed(1)} tok/s`))} ${style.dim(`median of ${runs} · [${decodes.map((d) => d.toFixed(1)).join(", ")}]`)}`,
|
|
2308
|
-
`prefill ${style.bold(`${median(prefills).toFixed(0)} tok/s`)} ${style.dim(`· ${lastStats.promptTokens} prompt tokens`)}`,
|
|
2309
|
-
`peak mem ${style.bold(gb(peak))} ${style.dim("(generation only)")}`,
|
|
2310
|
-
`levers ${style.dim(runtimeSummary(turboScheme ? { turboQuant: turboScheme } : { kvQuant: kvMode === "off" ? "off" : kvMode === "config" ? undefined : Number(kvMode) }))}`,
|
|
2311
|
-
"",
|
|
2312
|
-
style.dim("recorded to the eval DB — see mlx-bun evals"),
|
|
2313
|
-
]);
|
|
2314
|
-
break;
|
|
2315
|
-
}
|
|
2316
|
-
|
|
2317
|
-
case "evals": {
|
|
2318
|
-
if (flag("clear")) {
|
|
2319
|
-
// Eval rows are real data — archive, never destroy.
|
|
2320
|
-
const { DEFAULT_EVAL_DB } = await import("./evaldb");
|
|
2321
|
-
const { existsSync, renameSync } = await import("node:fs");
|
|
2322
|
-
if (!existsSync(DEFAULT_EVAL_DB)) { console.log("no eval DB to clear"); break; }
|
|
2323
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
2324
|
-
const backup = DEFAULT_EVAL_DB.replace(/\.sqlite$/, `-${stamp}.sqlite`);
|
|
2325
|
-
renameSync(DEFAULT_EVAL_DB, backup);
|
|
2326
|
-
console.log(`eval DB archived to ${backup}`);
|
|
2327
|
-
console.log("fresh DB starts with the next mlx-bun benchmark run");
|
|
2328
|
-
break;
|
|
2329
|
-
}
|
|
2330
|
-
const db = new EvalDB();
|
|
2331
|
-
const limit = Number(opt("limit", "20"));
|
|
2332
|
-
const rows = db.recent(limit);
|
|
2333
|
-
if (flag("raw")) {
|
|
2334
|
-
for (const r of rows) console.log(JSON.stringify(r));
|
|
2335
|
-
break;
|
|
2336
|
-
}
|
|
2337
|
-
const { style } = await import("./tui");
|
|
2338
|
-
if (rows.length === 0) {
|
|
2339
|
-
console.log("no eval runs recorded yet — bun scripts/bench-serve.ts all writes them");
|
|
2340
|
-
break;
|
|
2341
|
-
}
|
|
2342
|
-
const ago = (ts: number) => {
|
|
2343
|
-
const s = (Date.now() - ts) / 1000;
|
|
2344
|
-
if (s < 90) return `${Math.round(s)}s ago`;
|
|
2345
|
-
if (s < 5400) return `${Math.round(s / 60)}m ago`;
|
|
2346
|
-
if (s < 129600) return `${Math.round(s / 3600)}h ago`;
|
|
2347
|
-
return `${Math.round(s / 86400)}d ago`;
|
|
2348
|
-
};
|
|
2349
|
-
const modelName = (p: string) =>
|
|
2350
|
-
/models--[^/]+--([^/]+)/.exec(p)?.[1] ?? p.split("/").at(-1) ?? p;
|
|
2351
|
-
const note = (r: Record<string, unknown>, key: string) =>
|
|
2352
|
-
new RegExp(`${key}=([\\w.]+)`).exec((r.notes as string) ?? "")?.[1] ?? "";
|
|
2353
|
-
const BENCH_LABELS: Record<string, string> = {
|
|
2354
|
-
"h2h-server": "server e2e",
|
|
2355
|
-
"h2h-direct": "direct decode",
|
|
2356
|
-
"bench.ts": "decode script",
|
|
2357
|
-
"cli-bench": "cli benchmark",
|
|
2358
|
-
};
|
|
2359
|
-
const table = rows.map((r) => ({
|
|
2360
|
-
when: ago(r.ts as number),
|
|
2361
|
-
model: modelName(r.model_path as string).replace(/-OptiQ-4bit$/, "").replace(/^gemma-4-/, "g4-"),
|
|
2362
|
-
bench: BENCH_LABELS[((r.notes as string) ?? "").split(" ")[0] ?? ""] ?? ((r.notes as string) ?? "").split(" ")[0] ?? "",
|
|
2363
|
-
kv: note(r, "kv"),
|
|
2364
|
-
stack: (r.stack as string) ?? "mlx-bun",
|
|
2365
|
-
prefill: (r.prefill_tps as number) > 0 ? `${(r.prefill_tps as number).toFixed(0)}` : "",
|
|
2366
|
-
decode: (r.decode_tps as number).toFixed(1),
|
|
2367
|
-
ttft: note(r, "ttft_ms") ? `${note(r, "ttft_ms")}ms` : "",
|
|
2368
|
-
peak: gb(r.peak_bytes as number),
|
|
2369
|
-
commit: (r.commit_sha as string) ?? "",
|
|
2370
|
-
}));
|
|
2371
|
-
type Row = (typeof table)[number];
|
|
2372
|
-
const cols: Array<[string, keyof Row, "left" | "right"]> = [
|
|
2373
|
-
["WHEN", "when", "right"], ["STACK", "stack", "left"], ["MODEL", "model", "left"],
|
|
2374
|
-
["BENCH", "bench", "left"], ["KV", "kv", "left"],
|
|
2375
|
-
["PREFILL", "prefill", "right"], ["DECODE", "decode", "right"], ["TTFT", "ttft", "right"],
|
|
2376
|
-
["PEAK", "peak", "right"], ["COMMIT", "commit", "left"],
|
|
2377
|
-
];
|
|
2378
|
-
const widths = cols.map(([h, k]) => Math.max(h.length, ...table.map((t) => t[k].length)));
|
|
2379
|
-
console.log();
|
|
2380
|
-
console.log(" " + cols.map(([h], i) => style.dim(h.padEnd(widths[i]!))).join(" "));
|
|
2381
|
-
for (const t of table) {
|
|
2382
|
-
console.log(" " + cols.map(([, k, align], i) => {
|
|
2383
|
-
const cell = align === "right" ? t[k].padStart(widths[i]!) : t[k].padEnd(widths[i]!);
|
|
2384
|
-
if (k === "decode") return style.green(style.bold(cell));
|
|
2385
|
-
if (k === "model") return style.bold(cell);
|
|
2386
|
-
if (k === "stack") return cell.trimEnd() === "mlx-bun" ? style.accent(cell) : style.dim(cell);
|
|
2387
|
-
if (k === "commit" || k === "when") return style.dim(cell);
|
|
2388
|
-
return cell;
|
|
2389
|
-
}).join(" "));
|
|
2390
|
-
}
|
|
2391
|
-
console.log();
|
|
2392
|
-
console.log(style.dim(" STACK engine that produced the number (mlx-bun = us; mlx-lm/optiq = Python references)"));
|
|
2393
|
-
console.log(style.dim(" BENCH server e2e = full HTTP round-trip · direct decode = engine only, no server"));
|
|
2394
|
-
console.log(style.dim(" decode script / cli benchmark = single-stack decode measurement"));
|
|
2395
|
-
console.log(style.dim(" KV KV-cache quantization (off = bf16, config = per-layer kv_config.json)"));
|
|
2396
|
-
console.log(style.dim(" PREFILL prompt ingestion tok/s (in-process benches) · DECODE generation tok/s"));
|
|
2397
|
-
console.log(style.dim(" TTFT request → first token, incl. server overhead (server benches)"));
|
|
2398
|
-
console.log(style.dim(" PEAK max memory during generation"));
|
|
2399
|
-
console.log();
|
|
2400
|
-
console.log(style.dim(` ${rows.length} run(s) · --limit <n> for more · --raw for full records · mlx-bun benchmark to add a run`));
|
|
2401
|
-
break;
|
|
2402
|
-
}
|
|
2403
|
-
|
|
2404
|
-
case "pi": {
|
|
2405
|
-
// mlx-bun consumes its own flags; the rest describe how the built-in
|
|
2406
|
-
// agent runs. `mlx-bun pi` always drives pi's own TUI in-process
|
|
2407
|
-
// (src/pi-terminal.ts) — pi is bundled, nothing to install. Users who
|
|
2408
|
-
// already run their own pi connect it with `mlx-bun harness pi`.
|
|
2409
|
-
const OURS_VAL = new Set([
|
|
2410
|
-
"--query", "-q", "--model", "--port", "--host", "--memory-budget", "--kv-budget", "--prompt-cache",
|
|
2411
|
-
"--ssd-cache", "--ssd-cache-max", "--ssd-demote-idle", "--generation-checkpoint", "--kv-quant", "--unix",
|
|
2412
|
-
"--batch", "--decode-concurrency", "--adapter", "--adapter-path",
|
|
2413
|
-
"--draft-model", "--num-draft-tokens", "--draft-kind", "--ngram-max", "--ngram-min",
|
|
2414
|
-
"--compiled-decode", "--compiled-activations", "--fused-sdpa", "--thinking",
|
|
2415
|
-
"--temperature", "--temp", "--top-p", "--top-k", "--max-tokens",
|
|
2416
|
-
"--mtp", "--context-length",
|
|
2417
|
-
"--hlg-sampling", "--hlg-width", "--hlg-shoulder", "--hlg-toe", "--hlg-pivot-offset",
|
|
2418
|
-
"--paged-kv-block-size",
|
|
2419
|
-
]);
|
|
2420
|
-
const OURS_BOOL = new Set(["--force-wire", "--expert-offload", "--no-open", "--ssd-cache-verify", "--l1", "--l2", "--l3", "--isolate", "--paged-kv"]);
|
|
2421
|
-
const passthrough: string[] = [];
|
|
2422
|
-
for (let i = 1; i < argv.length; i++) {
|
|
2423
|
-
const a = argv[i]!;
|
|
2424
|
-
if (OURS_VAL.has(a)) { i++; continue; }
|
|
2425
|
-
if (OURS_BOOL.has(a)) continue;
|
|
2426
|
-
passthrough.push(a);
|
|
2427
|
-
}
|
|
2428
|
-
|
|
2429
|
-
const { parsePiArgs } = await import("./pi-terminal");
|
|
2430
|
-
const stdinPiped = !process.stdin.isTTY;
|
|
2431
|
-
const parsed = parsePiArgs(passthrough, stdinPiped);
|
|
2432
|
-
|
|
2433
|
-
// Headless machine-readable runs (`-p`, `--mode json`, `--mode rpc`) must
|
|
2434
|
-
// keep stdout pristine for the response/JSONL stream — route our bring-up
|
|
2435
|
-
// logging to stderr. Interactive keeps the pretty banner on stdout.
|
|
2436
|
-
const quiet = parsed.mode !== "interactive";
|
|
2437
|
-
const origStdoutWrite = process.stdout.write.bind(process.stdout);
|
|
2438
|
-
if (quiet) {
|
|
2439
|
-
(process.stdout as unknown as { write: typeof process.stderr.write }).write =
|
|
2440
|
-
process.stderr.write.bind(process.stderr);
|
|
2441
|
-
}
|
|
2442
|
-
const restoreStdout = () => {
|
|
2443
|
-
if (quiet) (process.stdout as unknown as { write: typeof origStdoutWrite }).write = origStdoutWrite;
|
|
2444
|
-
};
|
|
2445
|
-
|
|
2446
|
-
const rt = serverRuntimeFlags();
|
|
2447
|
-
const port = rt.port;
|
|
2448
|
-
const baseUrl = `http://127.0.0.1:${port}/v1`;
|
|
2449
|
-
const { probeServer } = await import("./harness-pi");
|
|
2450
|
-
let models = await probeServer(baseUrl);
|
|
2451
|
-
let startedServer = false;
|
|
2452
|
-
if (models) {
|
|
2453
|
-
console.log(`reusing running server at ${baseUrl} (${models.map((m) => m.id).join(", ")})`);
|
|
2454
|
-
// Server-shaping flags can't apply to a server we didn't start —
|
|
2455
|
-
// drop them loudly rather than silently or fatally.
|
|
2456
|
-
const ignored = [...OURS_VAL, ...OURS_BOOL].filter((f) => f !== "--port" && argv.includes(f));
|
|
2457
|
-
if (ignored.length > 0) {
|
|
2458
|
-
const { style } = await import("./tui");
|
|
2459
|
-
console.log(style.dim(` ignoring ${ignored.join(", ")} — the running server keeps its own configuration`));
|
|
2460
|
-
console.log(style.dim(" (stop it or use --port <other> to start a fresh server with these flags)"));
|
|
2461
|
-
}
|
|
2462
|
-
try {
|
|
2463
|
-
const stats = await (await fetch(`http://127.0.0.1:${port}/stats`)).json() as
|
|
2464
|
-
{ server?: { owner?: string } };
|
|
2465
|
-
if (stats.server?.owner === "pi-session") {
|
|
2466
|
-
console.log("note: that server belongs to another `mlx-bun pi` session and ends when it does —");
|
|
2467
|
-
console.log(" run `mlx-bun serve` for a persistent server.");
|
|
2468
|
-
}
|
|
2469
|
-
} catch {}
|
|
2470
|
-
} else {
|
|
2471
|
-
const { banner, step, style } = await import("./tui");
|
|
2472
|
-
banner(pkg.version);
|
|
2473
|
-
const { m, picked } = await resolveModelAuto(opt("query") ?? optShort("q"));
|
|
2474
|
-
if (picked)
|
|
2475
|
-
console.log(` ${style.dim(`auto-picked ${m.repoId} (largest supported model that fits; override with --query)`)}`);
|
|
2476
|
-
const sNative = step("native runtime");
|
|
2477
|
-
await ensureNative(sNative);
|
|
2478
|
-
sNative.done("native runtime ready");
|
|
2479
|
-
const { createServer, loadContext } = await import("./server");
|
|
2480
|
-
const sLoad = step(`loading ${m.repoId}`);
|
|
2481
|
-
const t0 = performance.now();
|
|
2482
|
-
const ctx = await loadContext(m.path, m.repoId, {
|
|
2483
|
-
memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
|
|
2484
|
-
glm: glm52RuntimeFlags(rt.serverOptions),
|
|
2485
|
-
});
|
|
2486
|
-
await mountStartupAdapter(ctx, rt.serverOptions);
|
|
2487
|
-
createServer(ctx, port, { ...rt.serverOptions, owner: "pi-session" });
|
|
2488
|
-
startedServer = true;
|
|
2489
|
-
sLoad.done(`serving ${style.bold(m.repoId)} ${style.dim(`at ${baseUrl} · ready in ${(performance.now() - t0).toFixed(0)} ms`)}`);
|
|
2490
|
-
models = await probeServer(baseUrl);
|
|
2491
|
-
if (!models) { restoreStdout(); console.error("server started but /v1/models probe failed"); process.exit(1); }
|
|
2492
|
-
}
|
|
2493
|
-
const exitNote = () => {
|
|
2494
|
-
// The server we started lives exactly as long as the pi session — any
|
|
2495
|
-
// exit tears it down. A standalone server belongs to `mlx-bun serve`
|
|
2496
|
-
// (which we reuse and never stop).
|
|
2497
|
-
if (startedServer && parsed.mode === "interactive")
|
|
2498
|
-
console.log("pi exited — shutting down (use `mlx-bun serve` for a persistent server)");
|
|
2499
|
-
};
|
|
2500
|
-
|
|
2501
|
-
// pi's own TUI embedded in-process — nothing to install.
|
|
2502
|
-
const { runEmbeddedPi } = await import("./pi-terminal");
|
|
2503
|
-
let message = parsed.message;
|
|
2504
|
-
if (parsed.mode === "print" && stdinPiped) {
|
|
2505
|
-
const piped = (await Bun.stdin.text()).trim();
|
|
2506
|
-
if (piped) message = message ? `${message}\n\n${piped}` : piped;
|
|
2507
|
-
}
|
|
2508
|
-
if (parsed.mode === "interactive")
|
|
2509
|
-
console.log(`launching mlx-bun pi (model ${models[0]!.id}) — /help for commands, double-Ctrl+C to exit`);
|
|
2510
|
-
restoreStdout();
|
|
2511
|
-
if (parsed.ignored.length) {
|
|
2512
|
-
const { style } = await import("./tui");
|
|
2513
|
-
console.log(style.dim(` ignoring pi flags: ${parsed.ignored.join(", ")}`));
|
|
2514
|
-
}
|
|
2515
|
-
const code = await runEmbeddedPi({
|
|
2516
|
-
baseUrl,
|
|
2517
|
-
modelLabel: models[0]!.id,
|
|
2518
|
-
contextWindow: models[0]!.contextWindow,
|
|
2519
|
-
reasoning: models[0]!.reasoning,
|
|
2520
|
-
vision: models[0]!.vision,
|
|
2521
|
-
verbose: parsed.verbose,
|
|
2522
|
-
mode: parsed.mode,
|
|
2523
|
-
printFormat: parsed.printFormat,
|
|
2524
|
-
initialMessage: message,
|
|
2525
|
-
});
|
|
2526
|
-
exitNote();
|
|
2527
|
-
process.exit(code);
|
|
2528
|
-
}
|
|
2529
|
-
|
|
2530
|
-
case "harness": {
|
|
2531
|
-
const target = positional(0);
|
|
2532
|
-
if (target !== "pi") {
|
|
2533
|
-
console.error("usage: mlx-bun harness pi [--base-url <url>] [--remove]");
|
|
2534
|
-
process.exit(1);
|
|
2535
|
-
}
|
|
2536
|
-
const { detectPi, installPiExtension, removePiExtension, DEFAULT_BASE_URL } =
|
|
2537
|
-
await import("./harness-pi");
|
|
2538
|
-
if (flag("remove")) {
|
|
2539
|
-
console.log(removePiExtension() ? "mlx-bun provider extension removed" : "nothing to remove");
|
|
2540
|
-
break;
|
|
2541
|
-
}
|
|
2542
|
-
const pi = detectPi();
|
|
2543
|
-
if (!pi.found) {
|
|
2544
|
-
console.error("pi not found on PATH. Install it first:");
|
|
2545
|
-
console.error(" curl -fsSL https://pi.dev/install.sh | sh");
|
|
2546
|
-
console.error("then re-run: mlx-bun harness pi");
|
|
2547
|
-
process.exit(1);
|
|
2548
|
-
}
|
|
2549
|
-
const { step, box, style } = await import("./tui");
|
|
2550
|
-
const sPi = step("detecting pi");
|
|
2551
|
-
sPi.done(`pi ${style.dim(`${pi.binPath}${pi.version ? ` · v${pi.version}` : ""}`)}`);
|
|
2552
|
-
const baseUrl = opt("base-url", DEFAULT_BASE_URL)!;
|
|
2553
|
-
const sExt = step("installing provider extension");
|
|
2554
|
-
const result = await installPiExtension(baseUrl);
|
|
2555
|
-
sExt.done(`provider extension installed ${style.dim(`· ${result.path}`)}`);
|
|
2556
|
-
const sProbe = step("probing server");
|
|
2557
|
-
if (result.serverReachable) sProbe.done(`server live ${style.dim(`· ${result.bakedModels.join(", ")}`)}`);
|
|
2558
|
-
else sProbe.done(`server not running ${style.dim(`· models discovered when pi starts against ${baseUrl}`)}`);
|
|
2559
|
-
console.log();
|
|
2560
|
-
box([
|
|
2561
|
-
`${style.green("●")} ${style.bold("your pi is connected to mlx-bun")}`,
|
|
2562
|
-
"",
|
|
2563
|
-
`launch ${style.accent("pi --provider mlx-bun")}`,
|
|
2564
|
-
`select ${style.dim("/model inside pi · or scope cycling:")} ${style.accent('pi --models "mlx-bun/*"')}`,
|
|
2565
|
-
`undo ${style.dim("mlx-bun harness pi --remove")}`,
|
|
2566
|
-
]);
|
|
2567
|
-
break;
|
|
2568
|
-
}
|
|
2569
|
-
|
|
2570
|
-
case "train": {
|
|
2571
|
-
const { banner, step, box, style } = await import("./tui");
|
|
2572
|
-
const query = positional(0) ?? opt("query");
|
|
2573
|
-
const dataDir = opt("data");
|
|
2574
|
-
if (!dataDir) {
|
|
2575
|
-
console.error("usage: mlx-bun train <model> --data <dir> (see: mlx-bun help train)");
|
|
2576
|
-
process.exit(1);
|
|
2577
|
-
}
|
|
2578
|
-
if (!(await Bun.file(`${dataDir}/train.jsonl`).exists())) {
|
|
2579
|
-
console.error(`no train.jsonl in ${dataDir}`);
|
|
2580
|
-
process.exit(1);
|
|
2581
|
-
}
|
|
2582
|
-
const method = opt("method", "orpo")!;
|
|
2583
|
-
if (method !== "sft" && method !== "dpo" && method !== "orpo") {
|
|
2584
|
-
console.error(`--method must be sft | dpo | orpo (got "${method}")`);
|
|
2585
|
-
process.exit(1);
|
|
2586
|
-
}
|
|
2587
|
-
const isOrpo = method === "orpo";
|
|
2588
|
-
// ORPO chosen-NLL scope (trainer sftScope; job.ts sft_scope). "full" =
|
|
2589
|
-
// paper/TRL-faithful prompt+response CE (the default); "response" =
|
|
2590
|
-
// response-only, bit-exact to pre-2026-07 runs. Validated here so junk
|
|
2591
|
-
// fails fast, before any model resolution or weight loading.
|
|
2592
|
-
const sftScope = opt("sft-scope");
|
|
2593
|
-
if (sftScope !== null && sftScope !== "full" && sftScope !== "response") {
|
|
2594
|
-
console.error(`--sft-scope must be full | response (got "${sftScope}")`);
|
|
2595
|
-
process.exit(1);
|
|
2596
|
-
}
|
|
2597
|
-
// Validated numeric flag: default if absent, hard-exit on a non-number.
|
|
2598
|
-
const numFlag = (name: string, dflt: number): number => {
|
|
2599
|
-
const v = opt(name);
|
|
2600
|
-
if (v == null) return dflt;
|
|
2601
|
-
const n = Number(v);
|
|
2602
|
-
if (!Number.isFinite(n)) { console.error(`--${name} expects a number (got "${v}")`); process.exit(1); }
|
|
2603
|
-
return n;
|
|
2604
|
-
};
|
|
2605
|
-
|
|
2606
|
-
banner(pkg.version);
|
|
2607
|
-
const sNative = step("native runtime");
|
|
2608
|
-
await ensureNative(sNative);
|
|
2609
|
-
sNative.done("native runtime ready");
|
|
2610
|
-
|
|
2611
|
-
const { m, picked } = await resolveModelAuto(query);
|
|
2612
|
-
|
|
2613
|
-
const isGemma = (await Bun.file(`${m.path}/config.json`).text()).toLowerCase().includes("gemma");
|
|
2614
|
-
|
|
2615
|
-
const modelTag = isGemma ? "e4b" : "cpm5";
|
|
2616
|
-
const adapter = opt("adapter") ?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/${method}-${modelTag}`;
|
|
2617
|
-
const iters = numFlag("iters", 100);
|
|
2618
|
-
const seq = numFlag("seq", isGemma ? 8192 : 4096);
|
|
2619
|
-
const seg = flag("no-segment") ? 0 : numFlag("seg", isOrpo ? 2 : 0);
|
|
2620
|
-
const saveEvery = numFlag("save-every", 0);
|
|
2621
|
-
const flashOn = !flag("no-flash");
|
|
2622
|
-
const prefixOn = !flag("no-prefix");
|
|
2623
|
-
const resume = opt("resume") ?? "";
|
|
2624
|
-
|
|
2625
|
-
// Build the snake_case submit record — the exact shape the server hands to
|
|
2626
|
-
// the finetune job runner (src/train/job.ts parseConfig).
|
|
2627
|
-
const cfg: Record<string, unknown> = {
|
|
2628
|
-
model_dir: m.path,
|
|
2629
|
-
data_dir: dataDir,
|
|
2630
|
-
adapter_path: adapter,
|
|
2631
|
-
method,
|
|
2632
|
-
rank: numFlag("rank", isOrpo ? 16 : 8),
|
|
2633
|
-
scale: numFlag("scale", isOrpo ? 2.0 : 1.0),
|
|
2634
|
-
rank_scaling: "by_bits",
|
|
2635
|
-
num_layers: -1,
|
|
2636
|
-
iters,
|
|
2637
|
-
learning_rate: numFlag("lr", isOrpo ? 1e-5 : method === "dpo" ? 5e-5 : 2e-4),
|
|
2638
|
-
max_seq_length: seq,
|
|
2639
|
-
batch_size: numFlag("batch", 1),
|
|
2640
|
-
grad_accumulation_steps: numFlag("grad-accum", 1),
|
|
2641
|
-
seed: numFlag("seed", 0),
|
|
2642
|
-
steps_per_report: 1,
|
|
2643
|
-
steps_per_eval: saveEvery > 0 ? saveEvery : 1_000_000,
|
|
2644
|
-
save_checkpoints: saveEvery > 0,
|
|
2645
|
-
segment_size: seg,
|
|
2646
|
-
grad_clip_norm: numFlag("grad-clip", 1.0),
|
|
2647
|
-
val_max_examples: numFlag("val-size", 256),
|
|
2648
|
-
warm_start_adapter: resume,
|
|
2649
|
-
...(sftScope ? { sft_scope: sftScope } : {}),
|
|
2650
|
-
...(isOrpo ? {
|
|
2651
|
-
orpo_lambda: numFlag("lambda", 0.1),
|
|
2652
|
-
orpo_lr_schedule: "cosine",
|
|
2653
|
-
orpo_warmup_iters: Math.min(10, Math.floor(iters / 10)),
|
|
2654
|
-
orpo_chunk_size: 512,
|
|
2655
|
-
orpo_flash_ce: flashOn,
|
|
2656
|
-
orpo_fused_ce: !flashOn,
|
|
2657
|
-
orpo_prefix_shared: prefixOn,
|
|
2658
|
-
} : {}),
|
|
2659
|
-
};
|
|
2660
|
-
|
|
2661
|
-
// Pre-flight: dataset counts + detected format (bail before loading the model).
|
|
2662
|
-
const { inspectDataset } = await import("./train/job");
|
|
2663
|
-
const ds = await inspectDataset(dataDir);
|
|
2664
|
-
if (!ds.ok) { console.error(`dataset: ${ds.error}`); process.exit(1); }
|
|
2665
|
-
|
|
2666
|
-
const planLines = [
|
|
2667
|
-
`${style.green("●")} ${style.bold(`train ${method}`)} ${style.dim(`· ${m.repoId}${picked ? " (auto-picked)" : ""}${isGemma ? " · e4b defaults" : ""}`)}`,
|
|
2668
|
-
"",
|
|
2669
|
-
`data ${style.bold(`${ds.n_train} train`)}${ds.n_valid ? ` · ${ds.n_valid} valid` : ""} ${style.dim(`· format ${ds.format}`)}`,
|
|
2670
|
-
`loop ${style.dim(`iters ${iters} · lr ${cfg.learning_rate} · rank ${cfg.rank} · scale ${cfg.scale} · seq ${seq} · batch ${cfg.batch_size}`)}`,
|
|
2671
|
-
];
|
|
2672
|
-
if (isOrpo)
|
|
2673
|
-
planLines.push(
|
|
2674
|
-
`head ${style.dim(flashOn ? "flash-CCE Metal ([M,vocab]-free)" : "MLX fused linear-CE")}`,
|
|
2675
|
-
`stack ${style.dim(`prefix-share ${prefixOn ? "on" : "off"} · segmented ${seg > 0 ? `${seg}/seg` : "off"} · λ ${cfg.orpo_lambda}`)}`,
|
|
2676
|
-
`stability ${style.dim(`grad-clip ${cfg.grad_clip_norm || "off"} · val-size ${cfg.val_max_examples}${(cfg.grad_accumulation_steps as number) > 1 ? ` · grad-accum ${cfg.grad_accumulation_steps} (eff batch ${(cfg.batch_size as number) * (cfg.grad_accumulation_steps as number)})` : ""}`)}`,
|
|
2677
|
-
);
|
|
2678
|
-
else planLines.push(`stack ${style.dim(`segmented ${seg > 0 ? `${seg}/seg` : "off"}`)}`);
|
|
2679
|
-
if (resume) planLines.push(`warm-start ${style.dim(`from ${resume} (weights only)`)}`);
|
|
2680
|
-
if (saveEvery > 0) planLines.push(`checkpoint ${style.dim(`every ${saveEvery} steps`)}`);
|
|
2681
|
-
planLines.push("", `adapter ${style.dim(adapter)}`);
|
|
2682
|
-
console.log();
|
|
2683
|
-
box(planLines);
|
|
2684
|
-
console.log();
|
|
2685
|
-
console.log(` ${style.dim("watch live (other tab):")} ${style.accent(`mlx-bun train-watch ${adapter}`)}`);
|
|
2686
|
-
console.log();
|
|
2687
|
-
|
|
2688
|
-
if (flag("dry-run")) { console.log(` ${style.dim("dry run — not training.")}`); break; }
|
|
2689
|
-
|
|
2690
|
-
// Run the finetune job runner IN-PROCESS (foreground), streaming metrics to
|
|
2691
|
-
// the terminal — the same runner the server drives via submitSubprocess.
|
|
2692
|
-
// Run long jobs detached from your own shell: nohup mlx-bun train … &
|
|
2693
|
-
const { finetuneRunner } = await import("./train/job");
|
|
2694
|
-
const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
|
|
2695
|
-
const losses: number[] = [];
|
|
2696
|
-
const stepMs: number[] = [];
|
|
2697
|
-
let lastStepT = Date.now();
|
|
2698
|
-
resetPeakMemory();
|
|
2699
|
-
const emit = (e: JobEvent) => {
|
|
2700
|
-
if (e.type === "stage" && e.message) console.log(` ${style.dim("·")} ${e.message}`);
|
|
2701
|
-
else if (e.type === "metric" && e.kind === "train") {
|
|
2702
|
-
const now = Date.now(); stepMs.push(now - lastStepT); lastStepT = now;
|
|
2703
|
-
losses.push(e.loss);
|
|
2704
|
-
const n = losses.length;
|
|
2705
|
-
if (n <= 3 || n % 10 === 0)
|
|
2706
|
-
console.log(` step ${n}/${iters}: loss ${style.bold(e.loss.toFixed(4))} ${style.dim(`(${(stepMs[stepMs.length - 1]! / 1000).toFixed(1)}s/step · peak ${gb(peakMemory())})`)}`);
|
|
2707
|
-
}
|
|
2708
|
-
};
|
|
2709
|
-
try {
|
|
2710
|
-
await finetuneRunner(emit, cfg);
|
|
2711
|
-
} catch (err) {
|
|
2712
|
-
console.error(`\n training failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2713
|
-
process.exit(1);
|
|
2714
|
-
}
|
|
2715
|
-
const finite = losses.every((l) => Number.isFinite(l));
|
|
2716
|
-
const sorted = stepMs.slice(1).sort((a, b) => a - b);
|
|
2717
|
-
const med = sorted[Math.floor(sorted.length / 2)] ?? stepMs[0] ?? 0;
|
|
2718
|
-
console.log();
|
|
2719
|
-
box([
|
|
2720
|
-
`${style.green("●")} ${style.bold("training complete")} ${style.dim(`· ${losses.length} steps`)}`,
|
|
2721
|
-
"",
|
|
2722
|
-
`loss ${style.bold(`${losses[0]?.toFixed(4) ?? "—"} → ${losses[losses.length - 1]?.toFixed(4) ?? "—"}`)}${finite ? "" : " (NON-FINITE!)"}`,
|
|
2723
|
-
`speed ${style.dim(`${(med / 1000).toFixed(1)}s/step median · peak ${gb(peakMemory())}`)}`,
|
|
2724
|
-
"",
|
|
2725
|
-
`adapter ${style.bold(adapter)}`,
|
|
2726
|
-
`serve it ${style.accent(`mlx-bun serve ${m.repoId} --adapter ${adapter}`)}`,
|
|
2727
|
-
]);
|
|
2728
|
-
break;
|
|
2729
|
-
}
|
|
2730
|
-
|
|
2731
|
-
case "train-watch": {
|
|
2732
|
-
// Live mactop-style dashboard for a training run: tails <adapter>/metrics.jsonl.
|
|
2733
|
-
const dir = positional(0) ?? opt("adapter")
|
|
2734
|
-
?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5`;
|
|
2735
|
-
const { runWatch } = await import("./train/watch");
|
|
2736
|
-
await runWatch(dir);
|
|
2737
|
-
break;
|
|
2738
|
-
}
|
|
2739
|
-
|
|
2740
|
-
case "fuse": {
|
|
2741
|
-
// mlx_lm.fuse counterpart over our native machinery (src/train/fuse.ts):
|
|
2742
|
-
// fold a LoRA adapter into the base weights → standalone snapshot.
|
|
2743
|
-
const unsupported = ["--de-quantize", "--dequantize", "--export-gguf", "--gguf-path", "--upload-repo"]
|
|
2744
|
-
.filter((f) => argv.includes(f));
|
|
2745
|
-
if (unsupported.length > 0) {
|
|
2746
|
-
console.error(`${unsupported.join(", ")}: not supported (see: mlx-bun help fuse)`);
|
|
2747
|
-
process.exit(1);
|
|
2748
|
-
}
|
|
2749
|
-
const modelArg = positional(0) ?? opt("model");
|
|
2750
|
-
if (!modelArg) {
|
|
2751
|
-
console.error("usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [--save-path <dir>]");
|
|
2752
|
-
process.exit(1);
|
|
2753
|
-
}
|
|
2754
|
-
const adapterDir = opt("adapter") ?? opt("adapter-path") ?? "adapters";
|
|
2755
|
-
const savePath = opt("save-path") ?? "fused_model";
|
|
2756
|
-
const { existsSync } = await import("node:fs");
|
|
2757
|
-
if (!existsSync(adapterDir)) {
|
|
2758
|
-
console.error(`adapter dir not found: ${adapterDir}`);
|
|
2759
|
-
process.exit(1);
|
|
2760
|
-
}
|
|
2761
|
-
let modelDir = modelArg;
|
|
2762
|
-
if (!existsSync(`${modelArg}/config.json`)) {
|
|
2763
|
-
const reg = new Registry();
|
|
2764
|
-
if (reg.list().length === 0) await reg.scan();
|
|
2765
|
-
modelDir = reg.resolve(modelArg).path;
|
|
2766
|
-
}
|
|
2767
|
-
const { banner, step, box, style } = await import("./tui");
|
|
2768
|
-
banner(pkg.version);
|
|
2769
|
-
const sNative = step("native runtime");
|
|
2770
|
-
await ensureNative(sNative);
|
|
2771
|
-
sNative.done("native runtime ready");
|
|
2772
|
-
const s = step(`fusing ${adapterDir} into ${modelDir}`);
|
|
2773
|
-
const { fuseAdapter } = await import("./train");
|
|
2774
|
-
try {
|
|
2775
|
-
const stats = await fuseAdapter(modelDir, adapterDir, savePath, (e) => s.update(e.message));
|
|
2776
|
-
s.done(`fused ${stats.fusedModules} module(s) ${style.dim(`· ${stats.totalTensors} tensors written`)}`);
|
|
2777
|
-
console.log();
|
|
2778
|
-
box([
|
|
2779
|
-
`${style.green("●")} ${style.bold("fuse complete")}`,
|
|
2780
|
-
"",
|
|
2781
|
-
`base ${style.dim(modelDir)}`,
|
|
2782
|
-
`adapter ${style.dim(adapterDir)}`,
|
|
2783
|
-
`model ${style.bold(stats.outDir)}`,
|
|
2784
|
-
...(stats.skippedAdapterTensors > 0
|
|
2785
|
-
? [`skipped ${style.dim(`${stats.skippedAdapterTensors} adapter tensor(s) with no matching base weight`)}`] : []),
|
|
2786
|
-
"",
|
|
2787
|
-
`serve it ${style.accent(`mlx-bun serve ${stats.outDir}`)}`,
|
|
2788
|
-
]);
|
|
2789
|
-
} catch (e) {
|
|
2790
|
-
s.fail(`fuse failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2791
|
-
process.exit(1);
|
|
2792
|
-
}
|
|
2793
|
-
break;
|
|
2794
|
-
}
|
|
2795
|
-
|
|
2796
|
-
case "convert": {
|
|
2797
|
-
// mlx_lm.convert counterpart over our native quantize pipeline (the same
|
|
2798
|
-
// engine as the web /api/quantize): uniform affine 4/8-bit, or the OptiQ
|
|
2799
|
-
// mixed-precision path via --target-bpw.
|
|
2800
|
-
// --upload-repo (mlx_lm.convert parity): push the converted model to the
|
|
2801
|
-
// Hub afterwards, via the same engine as `mlx-bun upload`. The token is
|
|
2802
|
-
// resolved BEFORE any conversion work — failing after minutes of
|
|
2803
|
-
// quantizing because auth is missing would be cruel.
|
|
2804
|
-
const uploadRepo = ((): string | null => {
|
|
2805
|
-
if (!argv.includes("--upload-repo")) return null;
|
|
2806
|
-
const v = opt("upload-repo");
|
|
2807
|
-
if (!v || v.startsWith("-")) {
|
|
2808
|
-
console.error("--upload-repo expects a repo id (org/name)");
|
|
2809
|
-
process.exit(1);
|
|
2810
|
-
}
|
|
2811
|
-
return v;
|
|
2812
|
-
})();
|
|
2813
|
-
let uploadToken: string | null = null;
|
|
2814
|
-
if (uploadRepo) {
|
|
2815
|
-
const { getHfToken } = await import("./hf-push");
|
|
2816
|
-
uploadToken = getHfToken();
|
|
2817
|
-
if (!uploadToken) {
|
|
2818
|
-
console.error("--upload-repo needs a Hugging Face WRITE token and none was found —");
|
|
2819
|
-
console.error("run `hf auth login`, export HF_TOKEN, or save one in the web UI (Settings → Hugging Face).");
|
|
2820
|
-
process.exit(1);
|
|
2821
|
-
}
|
|
2822
|
-
}
|
|
2823
|
-
const unsupported = ["--dtype", "-d", "--dequantize", "--quant-predicate"]
|
|
2824
|
-
.filter((f) => argv.includes(f));
|
|
2825
|
-
if (unsupported.length > 0) {
|
|
2826
|
-
console.error(`${unsupported.join(", ")}: not supported (mixed precision: --target-bpw; see: mlx-bun help convert)`);
|
|
2827
|
-
process.exit(1);
|
|
2828
|
-
}
|
|
2829
|
-
const qMode = opt("q-mode", "affine")!;
|
|
2830
|
-
if (qMode !== "affine") {
|
|
2831
|
-
console.error(`--q-mode ${qMode}: only "affine" is supported`);
|
|
2832
|
-
process.exit(1);
|
|
2833
|
-
}
|
|
2834
|
-
const hfPath = opt("hf-path") ?? opt("model") ?? positional(0);
|
|
2835
|
-
if (!hfPath) {
|
|
2836
|
-
console.error("usage: mlx-bun convert --hf-path <repo-or-path> -q [--q-bits N] [--q-group-size N] [--mlx-path <dir>] [--target-bpw F]");
|
|
2837
|
-
process.exit(1);
|
|
2838
|
-
}
|
|
2839
|
-
const targetBpwRaw = opt("target-bpw");
|
|
2840
|
-
if (!argv.includes("-q") && !argv.includes("--quantize") && targetBpwRaw === null) {
|
|
2841
|
-
console.error("plain (non-quantizing) conversion is not supported yet — pass -q or --target-bpw");
|
|
2842
|
-
process.exit(1);
|
|
2843
|
-
}
|
|
2844
|
-
const targetBpw = targetBpwRaw !== null ? Number(targetBpwRaw) : undefined;
|
|
2845
|
-
if (targetBpwRaw !== null && (!Number.isFinite(targetBpw!) || targetBpw! <= 0)) {
|
|
2846
|
-
console.error(`--target-bpw expects a positive number (got "${targetBpwRaw}")`);
|
|
2847
|
-
process.exit(1);
|
|
2848
|
-
}
|
|
2849
|
-
const qBits = Number(opt("q-bits", "4"));
|
|
2850
|
-
if (qBits !== 4 && qBits !== 8) {
|
|
2851
|
-
console.error(`--q-bits must be 4 or 8 (got "${opt("q-bits")}")`);
|
|
2852
|
-
process.exit(1);
|
|
2853
|
-
}
|
|
2854
|
-
const qGroup = Number(opt("q-group-size", "64"));
|
|
2855
|
-
if (qGroup !== 32 && qGroup !== 64) {
|
|
2856
|
-
console.error(`--q-group-size must be 32 or 64 (got "${opt("q-group-size")}")`);
|
|
2857
|
-
process.exit(1);
|
|
2858
|
-
}
|
|
2859
|
-
const candidateBits = opt("candidate-bits")?.split(",").map((s) => Number(s.trim()));
|
|
2860
|
-
if (candidateBits && candidateBits.some((b) => !Number.isInteger(b) || b < 2 || b > 8)) {
|
|
2861
|
-
console.error(`--candidate-bits expects a comma list of integers in [2, 8] (got "${opt("candidate-bits")}")`);
|
|
2862
|
-
process.exit(1);
|
|
2863
|
-
}
|
|
2864
|
-
const rotateWeights = flag("rotate-weights");
|
|
2865
|
-
const rotationSeed = Number(opt("rotation-seed", "42"));
|
|
2866
|
-
if (!Number.isInteger(rotationSeed)) {
|
|
2867
|
-
console.error(`--rotation-seed expects an integer (got "${opt("rotation-seed")}")`);
|
|
2868
|
-
process.exit(1);
|
|
2869
|
-
}
|
|
2870
|
-
const mlxPath = opt("mlx-path", "mlx_model")!;
|
|
2871
|
-
const { existsSync } = await import("node:fs");
|
|
2872
|
-
if (existsSync(mlxPath)) {
|
|
2873
|
-
console.error(`Cannot save to the path ${mlxPath} as it already exists — delete it or pass a fresh --mlx-path.`);
|
|
2874
|
-
process.exit(1);
|
|
2875
|
-
}
|
|
2876
|
-
const { banner, step, box, style } = await import("./tui");
|
|
2877
|
-
banner(pkg.version);
|
|
2878
|
-
const sNative = step("native runtime");
|
|
2879
|
-
await ensureNative(sNative);
|
|
2880
|
-
sNative.done("native runtime ready");
|
|
2881
|
-
// Resolve the source: local dir → itself; downloaded model → registry;
|
|
2882
|
-
// otherwise an HF repo id, downloaded first (the point of the verb).
|
|
2883
|
-
let srcDir = hfPath;
|
|
2884
|
-
if (!existsSync(`${hfPath}/config.json`)) {
|
|
2885
|
-
const reg = new Registry();
|
|
2886
|
-
if (reg.list().length === 0) await reg.scan();
|
|
2887
|
-
try {
|
|
2888
|
-
srcDir = reg.resolve(hfPath).path;
|
|
2889
|
-
} catch (e) {
|
|
2890
|
-
if (!/^[\w.-]+\/[\w.-]+$/.test(hfPath)) {
|
|
2891
|
-
console.error(e instanceof Error ? e.message : String(e));
|
|
2892
|
-
process.exit(1);
|
|
2893
|
-
}
|
|
2894
|
-
const { downloadModel } = await import("./download");
|
|
2895
|
-
const sDl = step(`downloading ${hfPath}`);
|
|
2896
|
-
srcDir = await downloadModel(hfPath, {
|
|
2897
|
-
onProgress: (file, received, total) => {
|
|
2898
|
-
const pct = total ? Math.floor((received / total) * 100) : 0;
|
|
2899
|
-
sDl.update(`${style.bold(hfPath)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
|
|
2900
|
-
},
|
|
2901
|
-
});
|
|
2902
|
-
sDl.done(`${style.bold(hfPath)} ${style.dim("downloaded · verified")}`);
|
|
2903
|
-
await reg.scan();
|
|
2904
|
-
}
|
|
2905
|
-
}
|
|
2906
|
-
const sQ = step(
|
|
2907
|
-
targetBpw !== undefined
|
|
2908
|
-
? `quantizing (mixed, target ${targetBpw} bpw — sensitivity sweep, ~minutes)`
|
|
2909
|
-
: `quantizing (${qBits}-bit, group ${qGroup})`,
|
|
2910
|
-
);
|
|
2911
|
-
const { quantizeModelDir, automaticRotationWeightTransform } = await import("./quantize");
|
|
2912
|
-
try {
|
|
2913
|
-
const r = await quantizeModelDir(
|
|
2914
|
-
srcDir,
|
|
2915
|
-
mlxPath,
|
|
2916
|
-
{
|
|
2917
|
-
bits: qBits as 4 | 8,
|
|
2918
|
-
groupSize: qGroup as 32 | 64,
|
|
2919
|
-
mode: "affine",
|
|
2920
|
-
...(targetBpw !== undefined ? { targetBpw } : {}),
|
|
2921
|
-
...(candidateBits ? { candidateBits } : {}),
|
|
2922
|
-
...(opt("calibration-mix") ? { calibrationMix: opt("calibration-mix")! } : {}),
|
|
2923
|
-
...(opt("n-calibration") ? { nCalibration: Number(opt("n-calibration")) } : {}),
|
|
2924
|
-
...(rotateWeights
|
|
2925
|
-
? { weightTransform: automaticRotationWeightTransform({ seed: rotationSeed }) }
|
|
2926
|
-
: {}),
|
|
2927
|
-
},
|
|
2928
|
-
(e) => sQ.update(e.message),
|
|
2929
|
-
);
|
|
2930
|
-
sQ.done(`quantized ${r.nQuantized} module(s) ${style.dim(`· ${r.achievedBpw.toFixed(2)} bpw achieved`)}`);
|
|
2931
|
-
console.log();
|
|
2932
|
-
box([
|
|
2933
|
-
`${style.green("●")} ${style.bold("convert complete")}`,
|
|
2934
|
-
"",
|
|
2935
|
-
`source ${style.dim(srcDir)}`,
|
|
2936
|
-
`model ${style.bold(r.outDir)} ${style.dim(`· ${gb(r.write.totalSize)}`)}`,
|
|
2937
|
-
`quant ${style.dim(targetBpw !== undefined ? `mixed ${r.achievedBpw.toFixed(2)} bpw (target ${targetBpw})` : `${qBits}-bit g${qGroup} affine`)}`,
|
|
2938
|
-
...(rotateWeights ? [`transform ${style.dim(`TurboQuant rotation seed ${rotationSeed}`)}`] : []),
|
|
2939
|
-
"",
|
|
2940
|
-
`serve it ${style.accent(`mlx-bun serve ${r.outDir}`)}`,
|
|
2941
|
-
]);
|
|
2942
|
-
if (uploadRepo) {
|
|
2943
|
-
const sUp = step(`uploading ${r.outDir} → ${uploadRepo}`);
|
|
2944
|
-
try {
|
|
2945
|
-
const { uploadFolder } = await import("./hf-push");
|
|
2946
|
-
const up = await uploadFolder(r.outDir, uploadRepo, {
|
|
2947
|
-
repoType: "model",
|
|
2948
|
-
token: uploadToken,
|
|
2949
|
-
commitMessage: "Upload with mlx-bun convert",
|
|
2950
|
-
onProgress: (file, sent, total) =>
|
|
2951
|
-
sUp.update(`${style.bold(uploadRepo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
|
|
2952
|
-
});
|
|
2953
|
-
sUp.done(`uploaded ${style.bold(up.url)}`);
|
|
2954
|
-
} catch (e) {
|
|
2955
|
-
sUp.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2956
|
-
console.error(`the converted model is intact at ${r.outDir} — retry with: mlx-bun upload --path ${r.outDir} --upload-repo ${uploadRepo}`);
|
|
2957
|
-
process.exit(1);
|
|
2958
|
-
}
|
|
2959
|
-
}
|
|
2960
|
-
} catch (e) {
|
|
2961
|
-
sQ.fail(`convert failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2962
|
-
process.exit(1);
|
|
2963
|
-
}
|
|
2964
|
-
break;
|
|
2965
|
-
}
|
|
2966
|
-
|
|
2967
|
-
case "upload": {
|
|
2968
|
-
// mlx_lm.upload counterpart (same flag names: --path, --upload-repo) over
|
|
2969
|
-
// our native push-to-hub (src/hf-push.ts — the exact engine the web
|
|
2970
|
-
// /api/{quantize,finetune,dataset}/push routes call). Model-free until
|
|
2971
|
-
// the network: everything before uploadFolder is arg + token validation.
|
|
2972
|
-
const dir = opt("path", "mlx_model")!; // mlx_lm.upload's --path default
|
|
2973
|
-
const repo = ((): string | null => {
|
|
2974
|
-
const v = opt("upload-repo");
|
|
2975
|
-
return v && !v.startsWith("-") ? v : null;
|
|
2976
|
-
})();
|
|
2977
|
-
if (!repo) {
|
|
2978
|
-
console.error("usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [--private]");
|
|
2979
|
-
process.exit(1);
|
|
2980
|
-
}
|
|
2981
|
-
const { existsSync, statSync } = await import("node:fs");
|
|
2982
|
-
if (!existsSync(dir) || !statSync(dir).isDirectory()) {
|
|
2983
|
-
console.error(`not a directory: ${dir} (pass the model directory via --path)`);
|
|
2984
|
-
process.exit(1);
|
|
2985
|
-
}
|
|
2986
|
-
const { getHfToken, uploadFolder } = await import("./hf-push");
|
|
2987
|
-
const token = getHfToken();
|
|
2988
|
-
if (!token) {
|
|
2989
|
-
console.error("no Hugging Face token found — uploading needs a WRITE token from one of:");
|
|
2990
|
-
console.error(" hf auth login (~/.cache/huggingface/token)");
|
|
2991
|
-
console.error(" export HF_TOKEN=hf_…");
|
|
2992
|
-
console.error(" web UI Settings → Hugging Face (~/.mlx-bun/hf.json)");
|
|
2993
|
-
process.exit(1);
|
|
2994
|
-
}
|
|
2995
|
-
const { banner, step, box, style } = await import("./tui");
|
|
2996
|
-
banner(pkg.version);
|
|
2997
|
-
const s = step(`uploading ${dir} → ${repo}`);
|
|
2998
|
-
try {
|
|
2999
|
-
const r = await uploadFolder(dir, repo, {
|
|
3000
|
-
repoType: "model",
|
|
3001
|
-
private: flag("private"),
|
|
3002
|
-
token,
|
|
3003
|
-
commitMessage: "Upload with mlx-bun",
|
|
3004
|
-
onProgress: (file, sent, total) =>
|
|
3005
|
-
s.update(`${style.bold(repo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
|
|
3006
|
-
});
|
|
3007
|
-
s.done(`uploaded ${style.bold(repo)}`);
|
|
3008
|
-
console.log();
|
|
3009
|
-
box([
|
|
3010
|
-
`${style.green("●")} ${style.bold("upload complete")}`,
|
|
3011
|
-
"",
|
|
3012
|
-
`source ${style.dim(dir)}`,
|
|
3013
|
-
`repo ${style.url(r.url)}${flag("private") ? style.dim(" · private") : ""}`,
|
|
3014
|
-
"",
|
|
3015
|
-
`get it back anywhere: ${style.accent(`mlx-bun get ${repo}`)}`,
|
|
3016
|
-
]);
|
|
3017
|
-
} catch (e) {
|
|
3018
|
-
s.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3019
|
-
process.exit(1);
|
|
3020
|
-
}
|
|
3021
|
-
break;
|
|
3022
|
-
}
|
|
3023
|
-
|
|
3024
|
-
case "perplexity": {
|
|
3025
|
-
// mlx_lm.perplexity counterpart (methodology reproduced exactly — see
|
|
3026
|
-
// src/eval/perplexity.ts) over a LOCAL dataset file; never downloads data.
|
|
3027
|
-
const modelArg = positional(0) ?? opt("model") ?? opt("query");
|
|
3028
|
-
const dataPath = opt("data-path");
|
|
3029
|
-
if (!modelArg || !dataPath) {
|
|
3030
|
-
console.error("usage: mlx-bun perplexity <model-query-or-path> --data-path <file.txt|file.jsonl> [--sequence-length 512] [--num-samples 256] [--batch-size 8] [--seed 123]");
|
|
3031
|
-
process.exit(1);
|
|
3032
|
-
}
|
|
3033
|
-
const { existsSync } = await import("node:fs");
|
|
3034
|
-
if (!existsSync(dataPath)) {
|
|
3035
|
-
console.error(`data file not found: ${dataPath} (a local .txt or .jsonl — HF datasets are not downloaded)`);
|
|
3036
|
-
process.exit(1);
|
|
3037
|
-
}
|
|
3038
|
-
const intFlag = (name: string, dflt: number, lo: number): number => {
|
|
3039
|
-
const v = Number(opt(name, String(dflt)));
|
|
3040
|
-
if (!Number.isInteger(v) || (v < lo && v !== -1)) {
|
|
3041
|
-
console.error(`--${name} expects an integer >= ${lo} (got "${opt(name)}")`);
|
|
3042
|
-
process.exit(1);
|
|
3043
|
-
}
|
|
3044
|
-
return v;
|
|
3045
|
-
};
|
|
3046
|
-
const seqLen = intFlag("sequence-length", 512, 2);
|
|
3047
|
-
const numSamples = intFlag("num-samples", 256, 1); // -1 = all
|
|
3048
|
-
const batchSize = intFlag("batch-size", 8, 1);
|
|
3049
|
-
const seed = intFlag("seed", 123, 0);
|
|
3050
|
-
const { banner, step, box, style } = await import("./tui");
|
|
3051
|
-
banner(pkg.version);
|
|
3052
|
-
const sNative = step("native runtime");
|
|
3053
|
-
await ensureNative(sNative);
|
|
3054
|
-
sNative.done("native runtime ready");
|
|
3055
|
-
let modelDir = modelArg;
|
|
3056
|
-
if (!existsSync(`${modelArg}/config.json`)) {
|
|
3057
|
-
const reg = new Registry();
|
|
3058
|
-
if (reg.list().length === 0) await reg.scan();
|
|
3059
|
-
modelDir = reg.resolve(modelArg).path;
|
|
3060
|
-
}
|
|
3061
|
-
const sLoad = step(`loading ${modelDir}`);
|
|
3062
|
-
const { Weights } = await import("./weights");
|
|
3063
|
-
const { createModel } = await import("./model/factory");
|
|
3064
|
-
const { loadTokenizer } = await import("./tokenizer");
|
|
3065
|
-
const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
|
|
3066
|
-
const config = await loadModelConfig(modelDir);
|
|
3067
|
-
const model = createModel(await Weights.open(modelDir), config);
|
|
3068
|
-
const tok = await loadTokenizer(modelDir);
|
|
3069
|
-
sLoad.done(`model loaded ${style.dim(`· ${modelDir}`)}`);
|
|
3070
|
-
|
|
3071
|
-
const { parseSamples, packRows, evalPpl } = await import("./eval/perplexity");
|
|
3072
|
-
const sData = step(`tokenizing ${dataPath}`);
|
|
3073
|
-
const samples = parseSamples(await Bun.file(dataPath).text(), dataPath);
|
|
3074
|
-
const rows = packRows(
|
|
3075
|
-
samples.map((t) => tok.encode(t)),
|
|
3076
|
-
{ sequenceLength: seqLen, numSamples, seed },
|
|
3077
|
-
);
|
|
3078
|
-
if (rows.length === 0) {
|
|
3079
|
-
sData.fail(`dataset too small: fewer than ${seqLen} tokens (need at least one full row)`);
|
|
3080
|
-
process.exit(1);
|
|
3081
|
-
}
|
|
3082
|
-
sData.done(`${rows.length} row(s) × ${seqLen} tokens ${style.dim(`· ${samples.length} sample(s), seed ${seed}`)}`);
|
|
3083
|
-
|
|
3084
|
-
const sEval = step(`evaluating (batch ${batchSize})`);
|
|
3085
|
-
resetPeakMemory();
|
|
3086
|
-
const t0 = performance.now();
|
|
3087
|
-
const r = evalPpl(model, rows, batchSize, (done, total) =>
|
|
3088
|
-
sEval.update(`batch ${done}/${total}`));
|
|
3089
|
-
const seconds = (performance.now() - t0) / 1000;
|
|
3090
|
-
sEval.done(`evaluated ${r.tokens.toLocaleString()} tokens ${style.dim(`in ${seconds.toFixed(1)} s`)}`);
|
|
3091
|
-
console.log();
|
|
3092
|
-
box([
|
|
3093
|
-
`${style.green("●")} ${style.bold("perplexity")} ${style.dim(`· ${modelDir.split("/snapshots/")[0]?.split("/").at(-1) ?? modelDir}`)}`,
|
|
3094
|
-
"",
|
|
3095
|
-
`ppl ${style.green(style.bold(r.ppl.toFixed(3)))} ${style.dim(`± ${r.standardError.toFixed(3)}`)}`,
|
|
3096
|
-
`mean CE ${style.bold(r.meanLoss.toFixed(4))} ${style.dim("nats/token")}`,
|
|
3097
|
-
`tokens ${style.bold(r.tokens.toLocaleString())} ${style.dim(`· ${r.rows} row(s) × ${seqLen}`)}`,
|
|
3098
|
-
`speed ${style.dim(`${(r.tokens / seconds).toFixed(0)} tok/s · peak ${gb(peakMemory())}`)}`,
|
|
3099
|
-
]);
|
|
3100
|
-
break;
|
|
3101
|
-
}
|
|
3102
|
-
|
|
3103
|
-
// `setup` is a true alias for `memory` — same handler, same subcommands
|
|
3104
|
-
// (`mlx-bun setup init` == `mlx-bun memory init`; the handler reads the
|
|
3105
|
-
// subcommand from argv[1] onward, identical for both spellings).
|
|
3106
|
-
case "setup":
|
|
3107
|
-
case "memory": {
|
|
3108
|
-
const { style, box, step, banner } = await import("./tui");
|
|
3109
|
-
const {
|
|
3110
|
-
vaultRoot, vaultStatus, setupVault, importArticlesFrom, commitVault,
|
|
3111
|
-
listMemoryDocuments, readArticle, resolveArticlePath, searchArticles,
|
|
3112
|
-
parseToc, extractSection, getArticleLinks,
|
|
3113
|
-
} = await import("./memory/vault");
|
|
3114
|
-
const { installSchedule, removeSchedule, scheduleStatus, parseAt } = await import("./memory/schedule");
|
|
3115
|
-
const root = vaultRoot();
|
|
3116
|
-
const fmtAt = (a: { hour: number; minute: number }) => `${String(a.hour).padStart(2, "0")}:${String(a.minute).padStart(2, "0")}`;
|
|
3117
|
-
const openMemoryTarget = async (targetPath = root): Promise<"obsidian" | "fallback"> => {
|
|
3118
|
-
// For articles, Obsidian's URL handler opens the exact file inside the
|
|
3119
|
-
// vault. For the vault root, `open -a Obsidian <folder>` opens the folder
|
|
3120
|
-
// as a vault. Both fall back to macOS `open` if Obsidian is unavailable.
|
|
3121
|
-
if (targetPath !== root) {
|
|
3122
|
-
const uri = `obsidian://open?path=${encodeURIComponent(targetPath)}`;
|
|
3123
|
-
const byUri = Bun.spawn(["open", uri], { stdout: "ignore", stderr: "ignore" });
|
|
3124
|
-
if ((await byUri.exited) === 0) return "obsidian";
|
|
3125
|
-
} else {
|
|
3126
|
-
const byApp = Bun.spawn(["open", "-a", "Obsidian", root], { stdout: "ignore", stderr: "ignore" });
|
|
3127
|
-
if ((await byApp.exited) === 0) return "obsidian";
|
|
3128
|
-
}
|
|
3129
|
-
const fallback = Bun.spawn(["open", targetPath], { stdout: "ignore", stderr: "ignore" });
|
|
3130
|
-
await fallback.exited;
|
|
3131
|
-
return "fallback";
|
|
3132
|
-
};
|
|
3133
|
-
|
|
3134
|
-
// Words after `memory`, minus flags: [subcommand, ...query].
|
|
3135
|
-
const words = argv.slice(1).filter((a) => !a.startsWith("-"));
|
|
3136
|
-
const sub = (words[0] ?? "status").toLowerCase();
|
|
3137
|
-
const rest = words.slice(1).join(" ").trim();
|
|
3138
|
-
|
|
3139
|
-
// TTY-only line prompt; in a non-interactive shell returns the default so
|
|
3140
|
-
// the wizard degrades to a non-destructive no-op rather than hanging.
|
|
3141
|
-
const ask = async (q: string, dflt = ""): Promise<string> => {
|
|
3142
|
-
if (!process.stdin.isTTY) return dflt;
|
|
3143
|
-
const { createInterface } = await import("node:readline/promises");
|
|
3144
|
-
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
3145
|
-
try {
|
|
3146
|
-
return ((await rl.question(q)).trim() || dflt);
|
|
3147
|
-
} finally {
|
|
3148
|
-
rl.close();
|
|
3149
|
-
}
|
|
3150
|
-
};
|
|
3151
|
-
const confirmYN = async (q: string, defaultYes: boolean): Promise<boolean> => {
|
|
3152
|
-
const a = (await ask(`${q} ${defaultYes ? "[Y/n]" : "[y/N]"} `)).toLowerCase();
|
|
3153
|
-
if (!a) return defaultYes;
|
|
3154
|
-
return a === "y" || a === "yes";
|
|
3155
|
-
};
|
|
3156
|
-
|
|
3157
|
-
if (sub === "init" || sub === "setup") {
|
|
3158
|
-
banner(pkg.version);
|
|
3159
|
-
box([
|
|
3160
|
-
`${style.accent("●")} ${style.bold("Personal memory for your local AI")}`,
|
|
3161
|
-
"",
|
|
3162
|
-
"A wiki of Markdown articles your local assistant reads to remember",
|
|
3163
|
-
"your projects, people, and history across sessions. It's yours —",
|
|
3164
|
-
`git-tracked at ${style.dim(root)}, editable in any tool, and it`,
|
|
3165
|
-
"never leaves this machine.",
|
|
3166
|
-
]);
|
|
3167
|
-
const sSetup = step("creating your wiki");
|
|
3168
|
-
const res = await setupVault(root);
|
|
3169
|
-
sSetup.done(res.alreadySetUp ? `already set up ${style.dim(`· ${root}`)}` : `wiki ready ${style.dim(`· ${root}`)}`);
|
|
3170
|
-
|
|
3171
|
-
// Offer to seed from an existing vault (e.g. a lucien ~/Dreaming) so the
|
|
3172
|
-
// assistant has real content to read on day one, before synthesis exists.
|
|
3173
|
-
const { homedir } = await import("node:os");
|
|
3174
|
-
const { join } = await import("node:path");
|
|
3175
|
-
const { readdir } = await import("node:fs/promises");
|
|
3176
|
-
const seedRoot = await ask(
|
|
3177
|
-
`\n Seed from an existing wiki? Enter its path, or blank to skip ${style.dim(`[${join(homedir(), "Dreaming")}]`)}: `,
|
|
3178
|
-
"",
|
|
3179
|
-
);
|
|
3180
|
-
if (seedRoot) {
|
|
3181
|
-
const srcArticles = join(seedRoot.replace(/^~(?=$|[/\\])/, homedir()), "articles");
|
|
3182
|
-
let count = 0;
|
|
3183
|
-
try { count = (await readdir(srcArticles)).filter((n) => n.endsWith(".md")).length; } catch { count = 0; }
|
|
3184
|
-
if (count === 0) {
|
|
3185
|
-
console.log(style.dim(` no articles found under ${srcArticles} — skipping import`));
|
|
3186
|
-
} else if (await confirmYN(` Import ${count} article(s) from ${srcArticles}?`, true)) {
|
|
3187
|
-
const sImp = step(`importing ${count} article(s)`);
|
|
3188
|
-
const imported = await importArticlesFrom(srcArticles, root);
|
|
3189
|
-
await commitVault(root, `Import ${imported.length} article(s) from ${seedRoot}`);
|
|
3190
|
-
sImp.done(`imported ${imported.length} article(s)`);
|
|
3191
|
-
}
|
|
3192
|
-
}
|
|
3193
|
-
|
|
3194
|
-
// Offer to install the nightly synthesis job (a launchd agent). The job
|
|
3195
|
-
// runs the real synthesis DAG (`mlx-bun memory synthesize`) — create new
|
|
3196
|
-
// entity articles, then the editorial wikify sweep — so installing the
|
|
3197
|
-
// schedule keeps your wiki current automatically. Persistent action →
|
|
3198
|
-
// TTY-only, defaults to no.
|
|
3199
|
-
let scheduled: string | null = null;
|
|
3200
|
-
console.log(style.dim(
|
|
3201
|
-
"\n Synthesis (turning your conversations into articles) runs as a nightly job;\n" +
|
|
3202
|
-
" install it now so your wiki stays current automatically.",
|
|
3203
|
-
));
|
|
3204
|
-
if (await confirmYN(" Install the nightly synthesis job (runs in the background)?", false)) {
|
|
3205
|
-
const atRaw = await ask(` At what time? 24h HH:MM ${style.dim("[03:00]")}: `, "03:00");
|
|
3206
|
-
const sSched = step(`installing nightly job at ${fmtAt(parseAt(atRaw))}`);
|
|
3207
|
-
const r = await installSchedule({ at: atRaw });
|
|
3208
|
-
scheduled = fmtAt(r.at);
|
|
3209
|
-
sSched.done(
|
|
3210
|
-
r.loaded
|
|
3211
|
-
? `nightly job installed ${style.dim(`· ${scheduled} · ${r.plistPath}`)}`
|
|
3212
|
-
: `plist written ${style.dim(`· ${r.plistPath} · launchctl load failed (load it manually)`)}`,
|
|
3213
|
-
);
|
|
3214
|
-
}
|
|
3215
|
-
|
|
3216
|
-
const st = await vaultStatus(root);
|
|
3217
|
-
console.log();
|
|
3218
|
-
box([
|
|
3219
|
-
`${style.green("●")} ${style.bold("memory is set up")} ${style.dim(`· ${st.articleCount} article(s) · ${st.referenceCount} reference doc(s)`)}`,
|
|
3220
|
-
"",
|
|
3221
|
-
`chat now ${style.accent("mlx-bun pi")} ${style.dim("— the assistant now reads your memory automatically")}`,
|
|
3222
|
-
`inspect ${style.accent("mlx-bun memory status")} ${style.dim("·")} ${style.accent("mlx-bun memory search <q>")}`,
|
|
3223
|
-
`browse ${style.dim(`open ${root} in Obsidian, or any editor`)}`,
|
|
3224
|
-
scheduled
|
|
3225
|
-
? `nightly ${style.green("scheduled")} ${style.dim(`· ${scheduled} daily · mlx-bun memory unschedule to undo`)}`
|
|
3226
|
-
: `nightly ${style.dim("not scheduled · mlx-bun memory schedule to set it up")}`,
|
|
3227
|
-
"",
|
|
3228
|
-
style.dim("Synthesis (conversations → articles) runs the full DAG: create new entity"),
|
|
3229
|
-
style.dim("articles, then an editorial wikify sweep. Run it now: mlx-bun memory synthesize."),
|
|
3230
|
-
]);
|
|
3231
|
-
break;
|
|
3232
|
-
}
|
|
3233
|
-
|
|
3234
|
-
if (sub === "status") {
|
|
3235
|
-
const st = await vaultStatus(root);
|
|
3236
|
-
if (!st.exists) {
|
|
3237
|
-
console.log(`no memory wiki yet at ${style.dim(root)}`);
|
|
3238
|
-
console.log(`set one up: ${style.accent("mlx-bun memory init")}`);
|
|
3239
|
-
break;
|
|
3240
|
-
}
|
|
3241
|
-
const sched = await scheduleStatus();
|
|
3242
|
-
console.log();
|
|
3243
|
-
box([
|
|
3244
|
-
`${style.bold("memory")} ${style.dim(`· ${root}`)}`,
|
|
3245
|
-
"",
|
|
3246
|
-
`articles ${style.bold(String(st.articleCount))}`,
|
|
3247
|
-
`reference ${style.bold(String(st.referenceCount))} ${style.dim("read-only docs")}`,
|
|
3248
|
-
`git ${st.isGitRepo ? style.green("tracked") : style.dim("not a git repo")}`,
|
|
3249
|
-
`synthesis ${style.green("available")} ${style.dim("· mlx-bun memory synthesize")}`,
|
|
3250
|
-
`nightly ${
|
|
3251
|
-
sched.installed
|
|
3252
|
-
? `${sched.loaded ? style.green("scheduled") : style.accent("installed (not loaded)")} ${style.dim(`· ${sched.plistPath}`)}`
|
|
3253
|
-
: style.dim("not scheduled · mlx-bun memory schedule")
|
|
3254
|
-
}`,
|
|
3255
|
-
st.recentArticles.length
|
|
3256
|
-
? `recent ${st.recentArticles.slice(0, 5).map((r) => r.article).join(", ")}`
|
|
3257
|
-
: `recent ${style.dim("none")}`,
|
|
3258
|
-
]);
|
|
3259
|
-
break;
|
|
3260
|
-
}
|
|
3261
|
-
|
|
3262
|
-
if (sub === "open" || sub === "browse") {
|
|
3263
|
-
const st = await vaultStatus(root);
|
|
3264
|
-
if (!st.exists) {
|
|
3265
|
-
console.log(`no memory wiki yet at ${style.dim(root)}`);
|
|
3266
|
-
console.log(`set one up: ${style.accent("mlx-bun memory init")}`);
|
|
3267
|
-
break;
|
|
3268
|
-
}
|
|
3269
|
-
let targetPath = root;
|
|
3270
|
-
let label = "memory";
|
|
3271
|
-
if (rest) {
|
|
3272
|
-
try {
|
|
3273
|
-
targetPath = await resolveArticlePath(root, rest);
|
|
3274
|
-
label = rest.endsWith(".md") ? rest.slice(0, -3) : rest;
|
|
3275
|
-
} catch (err) {
|
|
3276
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
3277
|
-
console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
|
|
3278
|
-
process.exit(1);
|
|
3279
|
-
}
|
|
3280
|
-
}
|
|
3281
|
-
const where = await openMemoryTarget(targetPath);
|
|
3282
|
-
console.log(
|
|
3283
|
-
where === "obsidian"
|
|
3284
|
-
? `opened ${label} in Obsidian: ${targetPath}`
|
|
3285
|
-
: `opened ${label}: ${targetPath}`,
|
|
3286
|
-
);
|
|
3287
|
-
break;
|
|
3288
|
-
}
|
|
3289
|
-
|
|
3290
|
-
// Independent, chronological, resumable STAGE WORKERS. Each pulls its own
|
|
3291
|
-
// eligible work from the DB by state, processes a bounded batch, persists,
|
|
3292
|
-
// and exits — so a user can run the four as four separate processes on
|
|
3293
|
-
// different slices concurrently (GPU/memory allowing). `synthesize` (below)
|
|
3294
|
-
// remains the FULL DAG; these are its decomposed pieces.
|
|
3295
|
-
if (sub === "segment" || sub === "extract" || sub === "route" || sub === "synthesize-stage" || sub === "stage-synthesize") {
|
|
3296
|
-
const stages = await import("./memory/stages");
|
|
3297
|
-
const { MemoryStore } = await import("./memory/db");
|
|
3298
|
-
const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
|
|
3299
|
-
const convsRaw = opt("convs");
|
|
3300
|
-
const convIds = convsRaw ? convsRaw.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
|
|
3301
|
-
const onEvent = (e: import("./memory/pipeline").SynthesisEvent) => {
|
|
3302
|
-
if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
|
|
3303
|
-
else console.log(` ${e.message}`);
|
|
3304
|
-
};
|
|
3305
|
-
const store = new MemoryStore();
|
|
3306
|
-
console.log();
|
|
3307
|
-
try {
|
|
3308
|
-
if (sub === "segment") {
|
|
3309
|
-
const r = await stages.runSegmentStage(store, { convIds, limit, onEvent });
|
|
3310
|
-
console.log(style.dim(`\n segment: ${r.valid} segmented, ${r.chunks} chunks, ${r.skipped} skipped, ${r.errored} errored`));
|
|
3311
|
-
} else if (sub === "extract") {
|
|
3312
|
-
const r = await stages.runExtractStage(store, { convIds, limit, onEvent });
|
|
3313
|
-
console.log(style.dim(`\n extract: ${r.extracted} chunk(s) extracted, ${r.remaining} still pending`));
|
|
3314
|
-
} else if (sub === "route") {
|
|
3315
|
-
const r = await stages.runRouteStage(store, { convIds, onEvent });
|
|
3316
|
-
console.log(style.dim(`\n route: ${r.decisions.length} entities — ${r.createEligible} create-eligible, ${r.captured.length} captured`));
|
|
3317
|
-
} else {
|
|
3318
|
-
const r = await stages.runSynthesizeStage(store, { root, convIds, limit, onEvent });
|
|
3319
|
-
console.log(style.dim(`\n synthesize: ${r.created.length} created, ${r.patched.length} patched, ${r.skippedByGate.length} gated`));
|
|
3320
|
-
}
|
|
3321
|
-
} finally {
|
|
3322
|
-
store.close();
|
|
3323
|
-
}
|
|
3324
|
-
break;
|
|
3325
|
-
}
|
|
3326
|
-
|
|
3327
|
-
// CROSS-LINK — the dedicated edge-building stage. Deterministic + idempotent:
|
|
3328
|
-
// inline-link first mentions of other articles + rebuild each ## See also from
|
|
3329
|
-
// mentions + co-occurrence. No model required.
|
|
3330
|
-
if (sub === "link") {
|
|
3331
|
-
const { runLinkStage } = await import("./memory/crosslink");
|
|
3332
|
-
const { MemoryStore } = await import("./memory/db");
|
|
3333
|
-
const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
|
|
3334
|
-
const store = new MemoryStore();
|
|
3335
|
-
console.log();
|
|
3336
|
-
try {
|
|
3337
|
-
const r = await runLinkStage(store, {
|
|
3338
|
-
root,
|
|
3339
|
-
limit,
|
|
3340
|
-
onEvent: (e) => console.log(e.type === "stage" ? ` ${style.dim("·")} ${e.message}` : ` ${e.message}`),
|
|
3341
|
-
});
|
|
3342
|
-
console.log(
|
|
3343
|
-
style.dim(
|
|
3344
|
-
`\n link: ${r.linked.length} article(s) linked · ${r.mentionEdges} mention edge(s) · ${r.skippedByGate.length} gated`,
|
|
3345
|
-
),
|
|
3346
|
-
);
|
|
3347
|
-
} finally {
|
|
3348
|
-
store.close();
|
|
3349
|
-
}
|
|
3350
|
-
break;
|
|
3351
|
-
}
|
|
3352
|
-
|
|
3353
|
-
if (sub === "synthesize" || sub === "pipeline" || sub === "all") {
|
|
3354
|
-
const { runSynthesis } = await import("./memory/pipeline");
|
|
3355
|
-
const dryRun = flag("dry-run");
|
|
3356
|
-
console.log();
|
|
3357
|
-
const summary = await runSynthesis(
|
|
3358
|
-
{ since: opt("since") ?? undefined, model: opt("model") ?? undefined, dryRun },
|
|
3359
|
-
(e) => {
|
|
3360
|
-
if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
|
|
3361
|
-
else console.log(` ${e.message}`);
|
|
3362
|
-
},
|
|
3363
|
-
);
|
|
3364
|
-
console.log(style.dim(`\n ${summary.note}`));
|
|
3365
|
-
break;
|
|
3366
|
-
}
|
|
3367
|
-
|
|
3368
|
-
if (sub === "schedule") {
|
|
3369
|
-
const atRaw = opt("at") ?? "03:00";
|
|
3370
|
-
const r = await installSchedule({ at: atRaw });
|
|
3371
|
-
console.log();
|
|
3372
|
-
box([
|
|
3373
|
-
r.loaded
|
|
3374
|
-
? `${style.green("●")} ${style.bold("nightly synthesis scheduled")} ${style.dim(`· ${fmtAt(r.at)} daily`)}`
|
|
3375
|
-
: `${style.accent("●")} ${style.bold("plist written, but launchctl load failed")}`,
|
|
3376
|
-
"",
|
|
3377
|
-
`plist ${style.dim(r.plistPath)}`,
|
|
3378
|
-
`runs ${style.dim("mlx-bun memory synthesize (full DAG: create + wikify sweep)")}`,
|
|
3379
|
-
`undo ${style.accent("mlx-bun memory unschedule")}`,
|
|
3380
|
-
]);
|
|
3381
|
-
break;
|
|
3382
|
-
}
|
|
3383
|
-
|
|
3384
|
-
if (sub === "unschedule") {
|
|
3385
|
-
const removed = await removeSchedule();
|
|
3386
|
-
console.log(removed ? "nightly synthesis job removed" : "no nightly job was installed");
|
|
3387
|
-
break;
|
|
3388
|
-
}
|
|
3389
|
-
|
|
3390
|
-
if (sub === "list") {
|
|
3391
|
-
const docs = await listMemoryDocuments(root);
|
|
3392
|
-
if (docs.length === 0) { console.log("no articles or reference docs yet"); break; }
|
|
3393
|
-
for (const s of docs) console.log(` ${s}`);
|
|
3394
|
-
const refs = docs.filter((s) => s.startsWith("Reference/")).length;
|
|
3395
|
-
console.log(style.dim(`\n ${docs.length - refs} article(s), ${refs} read-only reference doc(s)`));
|
|
3396
|
-
break;
|
|
3397
|
-
}
|
|
3398
|
-
|
|
3399
|
-
if (sub === "search") {
|
|
3400
|
-
if (!rest) { console.error("usage: mlx-bun memory search <query>"); process.exit(1); }
|
|
3401
|
-
const { summaries, hits } = await searchArticles(root, rest, { limit: 12 });
|
|
3402
|
-
if (summaries.length === 0) { console.log(`no articles match "${rest}"`); break; }
|
|
3403
|
-
console.log();
|
|
3404
|
-
for (const s of summaries.slice(0, 15)) {
|
|
3405
|
-
const terms = s.matched_terms?.length ? style.dim(` [${s.matched_terms.join(", ")}]`) : "";
|
|
3406
|
-
console.log(` ${style.bold(s.article)} ${style.dim(`· ${s.occurrences} hit(s)`)}${terms}`);
|
|
3407
|
-
}
|
|
3408
|
-
if (hits.length) {
|
|
3409
|
-
console.log(style.dim("\n sample lines:"));
|
|
3410
|
-
for (const h of hits.slice(0, 6)) {
|
|
3411
|
-
const where = h.anchor ? `${h.article}#${h.anchor}` : h.article;
|
|
3412
|
-
console.log(` ${style.dim(`${where}:${h.line}`)} ${h.excerpt}`);
|
|
3413
|
-
}
|
|
3414
|
-
}
|
|
3415
|
-
console.log(style.dim(`\n read one: mlx-bun memory read ${summaries[0]!.article}`));
|
|
3416
|
-
break;
|
|
3417
|
-
}
|
|
3418
|
-
|
|
3419
|
-
if (sub === "toc") {
|
|
3420
|
-
if (!rest) { console.error("usage: mlx-bun memory toc <article>"); process.exit(1); }
|
|
3421
|
-
try {
|
|
3422
|
-
const { content } = await readArticle(root, rest);
|
|
3423
|
-
const toc = parseToc(content);
|
|
3424
|
-
if (toc.length === 0) { console.log("no headings"); break; }
|
|
3425
|
-
for (const h of toc) console.log(`${" ".repeat(Math.max(0, h.depth - 1))}- ${h.title} ${style.dim(`#${h.anchor}`)}`);
|
|
3426
|
-
} catch (err) {
|
|
3427
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
3428
|
-
console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
|
|
3429
|
-
process.exit(1);
|
|
3430
|
-
}
|
|
3431
|
-
break;
|
|
3432
|
-
}
|
|
3433
|
-
|
|
3434
|
-
if (sub === "section") {
|
|
3435
|
-
const [article, anchorRaw] = words.slice(1);
|
|
3436
|
-
const anchor = anchorRaw?.replace(/^#/, "");
|
|
3437
|
-
if (!article || !anchor) { console.error("usage: mlx-bun memory section <article> <anchor>"); process.exit(1); }
|
|
3438
|
-
try {
|
|
3439
|
-
const { content } = await readArticle(root, article);
|
|
3440
|
-
const section = extractSection(content, anchor);
|
|
3441
|
-
if (!section) { console.error(`no section #${anchor} in ${article}`); console.error(`try: ${style.accent(`mlx-bun memory toc ${article}`)}`); process.exit(1); }
|
|
3442
|
-
console.log(section);
|
|
3443
|
-
} catch (err) {
|
|
3444
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
3445
|
-
console.error(`try: ${style.accent(`mlx-bun memory search ${article}`)}`);
|
|
3446
|
-
process.exit(1);
|
|
3447
|
-
}
|
|
3448
|
-
break;
|
|
3449
|
-
}
|
|
3450
|
-
|
|
3451
|
-
if (sub === "links") {
|
|
3452
|
-
if (!rest) { console.error("usage: mlx-bun memory links <article>"); process.exit(1); }
|
|
3453
|
-
try {
|
|
3454
|
-
const { outbound, inbound } = await getArticleLinks(root, rest);
|
|
3455
|
-
console.log(`${style.bold("outbound")}:`);
|
|
3456
|
-
console.log(outbound.length ? outbound.map((s) => ` ${s}`).join("\n") : " (none)");
|
|
3457
|
-
console.log(`\n${style.bold("inbound")}:`);
|
|
3458
|
-
console.log(inbound.length ? inbound.map((s) => ` ${s}`).join("\n") : " (none)");
|
|
3459
|
-
} catch (err) {
|
|
3460
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
3461
|
-
console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
|
|
3462
|
-
process.exit(1);
|
|
3463
|
-
}
|
|
3464
|
-
break;
|
|
3465
|
-
}
|
|
3466
|
-
|
|
3467
|
-
if (sub === "read") {
|
|
3468
|
-
if (!rest) { console.error("usage: mlx-bun memory read <article>"); process.exit(1); }
|
|
3469
|
-
try {
|
|
3470
|
-
const { content } = await readArticle(root, rest);
|
|
3471
|
-
console.log(content);
|
|
3472
|
-
} catch (err) {
|
|
3473
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
3474
|
-
console.error(`try: ${style.accent(`mlx-bun memory search ${rest}`)}`);
|
|
3475
|
-
process.exit(1);
|
|
3476
|
-
}
|
|
3477
|
-
break;
|
|
3478
|
-
}
|
|
3479
|
-
|
|
3480
|
-
|
|
3481
|
-
// Unknown subcommand: help on stdout, but a FAILURE exit — scripts and
|
|
3482
|
-
// launchd jobs must be able to detect typos (printHelp would exit 0).
|
|
3483
|
-
console.error(`unknown: mlx-bun memory ${sub}`);
|
|
3484
|
-
console.log(renderHelp(HELP.memory!));
|
|
3485
|
-
process.exit(1);
|
|
3486
|
-
}
|
|
3487
|
-
|
|
3488
|
-
default:
|
|
3489
|
-
printHelp(cmd);
|
|
3490
|
-
}
|