mlx-bun 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -20
- package/docs/design/batching-v2-plan.md +515 -0
- package/docs/design/bucketing-stage.md +196 -0
- package/docs/design/compat-cli-surface-design.md +11 -11
- package/docs/design/diffusion-gemma-port.md +602 -0
- package/docs/design/docs-reorg-plan.md +1 -1
- package/docs/design/dreaming-nightly-pipeline.md +291 -0
- package/docs/design/dspark-speculative-decoding.md +106 -0
- package/docs/design/generic-model-support.md +381 -0
- package/docs/design/memory-inference-path.md +422 -0
- package/docs/design/memory-synthesis.md +287 -0
- package/docs/design/minicpm5-decode-megakernel.md +484 -0
- package/docs/design/mlx-lm-tool-parity-plan.md +535 -0
- package/docs/design/orpo-dynamic-lambda.md +41 -0
- package/docs/design/orpo-training.md +35 -5
- package/docs/design/parallel-slots.md +12 -1
- package/docs/design/parity-tier-dag.md +13 -5
- package/docs/design/spec-decode-larger-targets.md +229 -0
- package/docs/design/tauri-desktop-app.md +2 -2
- package/docs/design/the-dreaming-handoff.md +141 -0
- package/docs/design/the-dreaming-master-plan.md +446 -0
- package/docs/design/write-pipeline-entity.md +468 -0
- package/docs/investigations/curve-bisector-routing.md +1 -1
- package/docs/investigations/decode-roofline-lookagain.md +262 -0
- package/docs/investigations/dspark-handoff.md +58 -0
- package/docs/investigations/kernel-perf-review-2026-07.md +92 -0
- package/docs/planning/PRODUCT_ROADMAP.md +2 -2
- package/docs/planning/curve-sampler-research-plan.md +331 -0
- package/docs/planning/memory-docs-and-dag-plan.md +185 -0
- package/docs/planning/pi-integration-review.md +178 -0
- package/docs/planning/release-notes-v0.0.9.md +122 -0
- package/docs/planning/web-ui-pass-plan.md +213 -0
- package/docs/planning/website-readme-pass-plan.md +224 -0
- package/docs/reference/cli.md +255 -0
- package/docs/reference/embedding.md +7 -2
- package/docs/reference/library-api.md +22 -3
- package/docs/reference/memory.md +5 -5
- package/docs/reference/models.md +137 -0
- package/docs/reference/orpo-quickstart.md +6 -0
- package/docs/reference/server-api.md +187 -9
- package/docs/reference/server-config.md +40 -9
- package/docs/reference/training.md +25 -11
- package/package.json +1 -1
- package/src/anthropic.ts +22 -0
- package/src/assets/curve-designer.html +1 -1
- package/src/chat-template.ts +25 -7
- package/src/cli.ts +1029 -86
- package/src/config.ts +54 -3
- package/src/diffusion/diffusion-generate.ts +436 -0
- package/src/download.ts +99 -8
- package/src/embed.ts +77 -0
- package/src/eval/perplexity.ts +164 -0
- package/src/eval/runner.ts +19 -7
- package/src/eval/tasks/bfcl.ts +1 -1
- package/src/generate.ts +196 -19
- package/src/harness-pi.ts +43 -17
- package/src/index.ts +2 -0
- package/src/lora.ts +2 -2
- package/src/memory/article.ts +558 -0
- package/src/memory/chunk-validation.ts +150 -0
- package/src/memory/chunk.ts +305 -12
- package/src/memory/cluster.ts +344 -17
- package/src/memory/coverage.ts +134 -0
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +184 -45
- package/src/memory/entity.ts +203 -0
- package/src/memory/gate.ts +187 -0
- package/src/memory/history.ts +152 -0
- package/src/memory/ingest.ts +324 -11
- package/src/memory/model.ts +321 -0
- package/src/memory/normalize.ts +671 -0
- package/src/memory/parse.ts +22 -0
- package/src/memory/pipeline.ts +242 -30
- package/src/memory/prompts.ts +42 -0
- package/src/memory/query.ts +334 -0
- package/src/memory/reconcile.ts +734 -0
- package/src/memory/redirect.ts +108 -0
- package/src/memory/reindex.ts +325 -0
- package/src/memory/resolve.ts +307 -0
- package/src/memory/route.ts +272 -0
- package/src/memory/schedule.ts +6 -4
- package/src/memory/stages.ts +774 -0
- package/src/memory/synthesize.ts +1629 -20
- package/src/memory/tools.ts +446 -96
- package/src/memory/vault.ts +462 -35
- package/src/memory/wikify.ts +1057 -16
- package/src/mlx/ffi.ts +11 -0
- package/src/mlx/ops.ts +113 -0
- package/src/model/compiled-decode.ts +43 -8
- package/src/model/diffusion-gemma.ts +1026 -0
- package/src/model/factory.ts +35 -8
- package/src/model/flash-attention.ts +4 -2
- package/src/model/fused-decode-kernel.ts +14 -7
- package/src/model/fused-mlp-kernel.ts +169 -0
- package/src/model/fused-swiglu-kernel.ts +240 -0
- package/src/model/gemma4-base.ts +62 -6
- package/src/model/gemma4.ts +33 -2
- package/src/model/minicpm5.ts +106 -8
- package/src/model/qwen3.ts +276 -0
- package/src/model/steel-linear-kernel.ts +96 -0
- package/src/model/support.ts +45 -7
- package/src/model/universal/archs.ts +332 -0
- package/src/model/universal/dense.ts +576 -0
- package/src/model/universal/modules.ts +163 -0
- package/src/model/universal/rope.ts +366 -0
- package/src/pi-provider.ts +1 -1
- package/src/pi-terminal.ts +29 -8
- package/src/pi-web.ts +19 -2
- package/src/quantize/config-writer.ts +1 -1
- package/src/registry.ts +215 -7
- package/src/responses.ts +23 -0
- package/src/sampler.ts +220 -9
- package/src/serve/batch-scheduler.ts +272 -80
- package/src/serve/generation-gateway.ts +90 -18
- package/src/server.ts +643 -50
- package/src/spec/drafter.ts +81 -49
- package/src/spec/dspark/data-dflash.ts +156 -0
- package/src/spec/dspark/data.ts +201 -0
- package/src/spec/dspark/generate-dflash.ts +193 -0
- package/src/spec/dspark/generate.ts +290 -0
- package/src/spec/dspark/loss.ts +213 -0
- package/src/spec/dspark/module-dflash.ts +361 -0
- package/src/spec/dspark/module.ts +540 -0
- package/src/spec/dspark/sample.ts +112 -0
- package/src/spec/generate.ts +40 -19
- package/src/status-page.html +1 -1
- package/src/tokenizer.ts +7 -0
- package/src/train/diffusion-lora.ts +216 -0
- package/src/train/flash-cce.ts +70 -15
- package/src/train/fuse.ts +243 -0
- package/src/train/index.ts +3 -0
- package/src/train/job.ts +8 -0
- package/src/train/loss.ts +289 -30
- package/src/train/prefix-shared.ts +33 -5
- package/src/train/segmented.ts +67 -11
- package/src/train/trainer.ts +92 -28
- package/src/vision/diffusion-vision.ts +620 -0
- package/src/web/skills/memory/SKILL.md +40 -17
package/src/cli.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
// mlx-bun get <org/repo> [--revision main] resumable verified download
|
|
5
5
|
// mlx-bun scan index the HF cache
|
|
6
6
|
// mlx-bun ls [--vision] [--max-size 10GB] [query]
|
|
7
|
-
// mlx-bun fit <query> [--ctx
|
|
8
|
-
// mlx-bun serve [query] [--port
|
|
7
|
+
// mlx-bun fit <query> [--ctx 8192] [--skus]
|
|
8
|
+
// mlx-bun serve [query] [--port 8080] [--memory-budget GB]
|
|
9
9
|
// mlx-bun evals recent benchmark runs
|
|
10
10
|
// mlx-bun harness pi [--base-url <url>] [--remove] connect your own pi to the local server
|
|
11
11
|
|
|
@@ -37,12 +37,20 @@ Commands:
|
|
|
37
37
|
serve Start the OpenAI/Anthropic-compatible server + status page
|
|
38
38
|
get Download a model from Hugging Face (resumable, verified)
|
|
39
39
|
ls List downloaded models
|
|
40
|
+
gc Reclaim superseded model snapshots + dead blobs from the HF cache
|
|
40
41
|
fit Will a model fit this machine? Memory + speed assessment
|
|
41
42
|
scan Re-index the Hugging Face cache
|
|
42
43
|
harness Connect your own pi install to the local mlx-bun server
|
|
43
44
|
train Fine-tune a LoRA adapter (SFT / DPO / ORPO) on your data
|
|
45
|
+
train-watch Live dashboard for a training run (tails metrics.jsonl)
|
|
46
|
+
fuse Merge a LoRA adapter into the base weights (standalone model)
|
|
47
|
+
convert Quantize an HF model into a local MLX snapshot
|
|
48
|
+
upload Push a local model directory to the Hugging Face Hub
|
|
49
|
+
perplexity Perplexity of a model over a local text/JSONL dataset
|
|
44
50
|
memory Set up + manage your local AI's personal wiki memory
|
|
51
|
+
generate One-shot raw text generation from a local model (no server)
|
|
45
52
|
benchmark Measure decode/prefill speed of OUR stack on this machine
|
|
53
|
+
embed Text embeddings from a local embedding model (Qwen3-Embedding)
|
|
46
54
|
evals Show recent benchmark runs (all stacks)
|
|
47
55
|
help Show help for a command (also: mlx-bun <command> --help)
|
|
48
56
|
|
|
@@ -53,27 +61,36 @@ Options:
|
|
|
53
61
|
Examples:
|
|
54
62
|
mlx-bun # download (first run) + serve + open the chat UI
|
|
55
63
|
mlx-bun pi # start (download if needed) and chat with an agent
|
|
56
|
-
mlx-bun serve 12B # serve the 12B; status page at http://localhost:
|
|
64
|
+
mlx-bun serve 12B # serve the 12B; status page at http://localhost:8080/
|
|
57
65
|
mlx-bun train e4b --data ./prefs # ORPO LoRA fine-tune on {prompt,chosen,rejected} data
|
|
58
66
|
mlx-bun get mlx-community/gemma-4-12B-it-OptiQ-4bit`;
|
|
59
67
|
|
|
60
68
|
const SERVER_FLAGS = `Server options:
|
|
61
|
-
--host <addr> Interface to bind [default:
|
|
62
|
-
use
|
|
63
|
-
--port <n> Listen port [default:
|
|
69
|
+
--host <addr> Interface to bind [default: 127.0.0.1, loopback
|
|
70
|
+
only; use 0.0.0.0 to expose on your network]
|
|
71
|
+
--port <n> Listen port [default: 8080]
|
|
64
72
|
--memory-budget <GB> Admission-control memory budget; requests that
|
|
65
73
|
cannot fit are rejected instead of crashing the
|
|
66
74
|
GPU [default: machine RAM × 0.75, check-only]
|
|
67
|
-
--prompt-cache <GB> Prompt (KV) cache byte cap [default: 2 GB
|
|
75
|
+
--prompt-cache <GB> Prompt (KV) cache byte cap [default: 2 GB;
|
|
76
|
+
--prompt-cache 0 disables the cache]
|
|
68
77
|
--batch <n> Max concurrent requests batched through the
|
|
69
78
|
mlx-lm-parity engine [default: 1 = serial].
|
|
70
79
|
>1 opts the whole server into bf16 continuous
|
|
71
80
|
batching (= mlx-lm B=N); see --kv-quant.
|
|
81
|
+
--decode-concurrency is accepted as the
|
|
82
|
+
mlx_lm.server alias (semantics differ; see
|
|
83
|
+
docs/reference/server-config.md)
|
|
72
84
|
--no-open Don't open the chat UI in your browser on start.
|
|
73
85
|
By default an interactive terminal opens
|
|
74
86
|
http://<host>:<port>/#/chat once the server is up.
|
|
75
87
|
|
|
76
88
|
Model & quality:
|
|
89
|
+
--adapter <dir> Mount a LoRA adapter at startup and use it as the
|
|
90
|
+
default for requests that don't select one (a
|
|
91
|
+
request's \`adapter\` field, incl. "none", wins).
|
|
92
|
+
--adapter-path is accepted as the mlx_lm.server
|
|
93
|
+
alias. Hot-swap via POST /v1/adapters unchanged.
|
|
77
94
|
--kv-quant <mode> KV cache quantization: config (per-layer
|
|
78
95
|
kv_config.json when the model ships one), off
|
|
79
96
|
(bf16), or 4 / 8 (uniform bits)
|
|
@@ -86,8 +103,14 @@ Model & quality:
|
|
|
86
103
|
[default: the model's own, false for CPM]
|
|
87
104
|
--temperature <n> Server-wide sampling defaults; a per-request field
|
|
88
105
|
--top-p <n> still overrides them, and the browser chat (which
|
|
89
|
-
--top-k <n> sends none) inherits them
|
|
106
|
+
--top-k <n> sends none) inherits them. --temp is accepted as
|
|
107
|
+
an alias for --temperature (mlx_lm.server compat;
|
|
108
|
+
NOTE mlx_lm.server defaults temperature to 0.0 —
|
|
109
|
+
pass --temp 0 for its unset-request behavior)
|
|
90
110
|
[default: the model's generation_config.json]
|
|
111
|
+
--max-tokens <n> Completion cap when a request omits max_tokens
|
|
112
|
+
(mlx_lm.server flag; its default is 512 there)
|
|
113
|
+
[default: 65536 chat / 512 raw completion]
|
|
91
114
|
--hlg-sampling on|off Piecewise tone-curve sampling (HLG): rolls off the
|
|
92
115
|
top, boosts the mids, gentles the tail. Gain folds
|
|
93
116
|
from --temperature. [default: off]
|
|
@@ -97,10 +120,21 @@ Model & quality:
|
|
|
97
120
|
--hlg-toe <nats> HLG shadow rolloff scale [default: 6]
|
|
98
121
|
--hlg-pivot-offset <nats> HLG pivot: nats below the top token [default: 6]
|
|
99
122
|
|
|
123
|
+
Parity tier (sets the whole decode route; a per-fork flag below overrides):
|
|
124
|
+
--l1 Bit-for-bit IDENTICAL to mlx-lm (bf16 KV, unfused)
|
|
125
|
+
--l2 Bit-for-bit IDENTICAL to mlx-optiq (quantized KV +
|
|
126
|
+
fused prefill SDPA, stock unfused decode — the
|
|
127
|
+
optiq-golden composition; perf-kernel stays OFF)
|
|
128
|
+
--l3 Best performance, no bit-exact oracle (KL + test
|
|
129
|
+
gated; adds the envelope-gated perf kernel).
|
|
130
|
+
No tier given = the per-flag defaults below,
|
|
131
|
+
which equal --l3 today.
|
|
132
|
+
|
|
100
133
|
Performance levers (A/B levers; defaults are the measured winners):
|
|
101
134
|
--compiled-decode on|off Compiled decode graphs [default: on]
|
|
102
|
-
--perf-kernel on|off Fused quantized-KV decode-SDPA Metal kernel
|
|
103
|
-
|
|
135
|
+
--perf-kernel on|off Fused quantized-KV decode-SDPA Metal kernel —
|
|
136
|
+
NOT bit-exact, envelope-gated (an L3 lever; the
|
|
137
|
+
perf side of the compat A/B) [default: on]
|
|
104
138
|
--fused-decode on|off Fused-decode experiment lever [default: off]
|
|
105
139
|
--fused-sdpa on|off Fused SDPA path [default: on]
|
|
106
140
|
--force-wire Wire weights into memory at load
|
|
@@ -141,42 +175,73 @@ Already use pi? Connect your own pi to this local model instead:
|
|
|
141
175
|
Usage: mlx-bun serve [query] [options]
|
|
142
176
|
|
|
143
177
|
[query] Registry query (e.g. "12B", "e4b", a repo substring).
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
178
|
+
--query <q> is accepted for the positional. Omitted:
|
|
179
|
+
serves e4b (the default), else the largest downloaded
|
|
180
|
+
model that fits. Fresh install: downloads a small
|
|
181
|
+
starter model first, then e4b in the background
|
|
147
182
|
(default on next run).
|
|
148
183
|
|
|
149
184
|
${SERVER_FLAGS}
|
|
150
185
|
|
|
151
|
-
Endpoints: /v1/chat/completions, /v1/
|
|
152
|
-
/v1/adapters, /stats, /fit, /library, /downloads —
|
|
153
|
-
browser chat at /chat`,
|
|
186
|
+
Endpoints: /v1/chat/completions, /v1/completions, /v1/messages, /v1/responses,
|
|
187
|
+
/v1/models, /v1/adapters, /health, /stats, /fit, /library, /downloads —
|
|
188
|
+
status page at /, browser chat at /chat`,
|
|
154
189
|
|
|
155
190
|
get: `mlx-bun get — download a model from Hugging Face
|
|
156
191
|
|
|
157
|
-
Usage: mlx-bun get <org/repo> [options]
|
|
192
|
+
Usage: mlx-bun get <org/repo | substring> [options]
|
|
193
|
+
|
|
194
|
+
<org/repo> Full repo id (e.g. mlx-community/gemma-4-12B-it-OptiQ-4bit)
|
|
195
|
+
<substring> No "/" = a registry query over already-downloaded repos
|
|
196
|
+
(e.g. \`mlx-bun get 12B\` re-gets/refreshes the match)
|
|
158
197
|
|
|
159
198
|
Options:
|
|
160
199
|
--revision <rev> Git revision [default: main]
|
|
161
200
|
|
|
162
201
|
Resumable (Range requests against partial blobs) and verified (sha256
|
|
163
202
|
for LFS blobs). Re-running after an interruption continues where it
|
|
164
|
-
stopped. Uses HF_TOKEN / hf auth login credentials when present
|
|
203
|
+
stopped. Uses HF_TOKEN / hf auth login credentials when present.
|
|
204
|
+
When upstream pushed a new revision, the previous snapshot stays on
|
|
205
|
+
disk — \`mlx-bun gc\` reclaims it.`,
|
|
165
206
|
|
|
166
207
|
ls: `mlx-bun ls — list downloaded models
|
|
167
208
|
|
|
168
209
|
Usage: mlx-bun ls [query] [options]
|
|
169
210
|
|
|
211
|
+
One row per repo (the canonical revision: refs/main). The HF cache keeps
|
|
212
|
+
a snapshot dir per downloaded revision — superseded ones are hidden here
|
|
213
|
+
and reclaimed by \`mlx-bun gc\`.
|
|
214
|
+
|
|
215
|
+
Options:
|
|
216
|
+
--vision Only vision-capable models (SigLIP sidecar or the
|
|
217
|
+
unified encoder-free tower)
|
|
218
|
+
--max-size <size> Filter by weight size (e.g. 10GB, 800MB)
|
|
219
|
+
--all-revisions One row per cached snapshot (canonical marked *)`,
|
|
220
|
+
|
|
221
|
+
gc: `mlx-bun gc — reclaim superseded snapshots + dead blobs
|
|
222
|
+
|
|
223
|
+
Usage: mlx-bun gc [options]
|
|
224
|
+
|
|
225
|
+
Every \`get\` that follows an upstream push creates a NEW
|
|
226
|
+
snapshots/<commit> dir; the old one (and any blobs only it references)
|
|
227
|
+
stays on disk forever. gc keeps the snapshots refs/* point at, deletes
|
|
228
|
+
the rest, then deletes blobs no surviving snapshot links to.
|
|
229
|
+
|
|
230
|
+
Without --yes it only prints what would be deleted (per-repo reclaim).
|
|
231
|
+
|
|
170
232
|
Options:
|
|
171
|
-
--
|
|
172
|
-
--
|
|
233
|
+
--yes Actually delete (destructive; default prints only)
|
|
234
|
+
--dry-run Print the plan and never delete, even with --yes
|
|
235
|
+
--force Also prune superseded snapshots that contain files
|
|
236
|
+
the canonical revision lacks (normally skipped with
|
|
237
|
+
a warning — deleting them loses the only copy)`,
|
|
173
238
|
|
|
174
239
|
fit: `mlx-bun fit — will this model fit on this machine?
|
|
175
240
|
|
|
176
241
|
Usage: mlx-bun fit <query> [options]
|
|
177
242
|
|
|
178
243
|
Options:
|
|
179
|
-
--ctx <tokens> Context size to assess [default:
|
|
244
|
+
--ctx <tokens> Context size to assess [default: 8192]
|
|
180
245
|
--skus Also print the Apple silicon SKU matrix
|
|
181
246
|
|
|
182
247
|
Solves weights + KV growth + prefill transient against wired memory and
|
|
@@ -199,7 +264,7 @@ local model. (To just chat now with no setup, use \`mlx-bun pi\` — the
|
|
|
199
264
|
built-in agent.)
|
|
200
265
|
|
|
201
266
|
Options:
|
|
202
|
-
--base-url <url> Server base URL [default: http://
|
|
267
|
+
--base-url <url> Server base URL [default: http://127.0.0.1:8080/v1]
|
|
203
268
|
--remove Disconnect (delete the extension)
|
|
204
269
|
|
|
205
270
|
Installs a small discovery extension into ~/.pi/agent/extensions that
|
|
@@ -216,7 +281,8 @@ result to the eval DB. For quotable, cross-stack numbers use
|
|
|
216
281
|
./benchmark.sh from the repo — it preflight-gates on an idle machine
|
|
217
282
|
and runs the mlx-lm/optiq comparison legs.
|
|
218
283
|
|
|
219
|
-
[query] Model to benchmark
|
|
284
|
+
[query] Model to benchmark (--query <q> is accepted for the
|
|
285
|
+
positional) [default: auto-pick]
|
|
220
286
|
|
|
221
287
|
Options:
|
|
222
288
|
--tokens <n> Tokens to decode per run [default: 256]
|
|
@@ -228,6 +294,61 @@ Options:
|
|
|
228
294
|
Performance levers (--compiled-decode, --perf-kernel, --fused-decode,
|
|
229
295
|
--fused-sdpa) apply to the run — A/B by running twice.`,
|
|
230
296
|
|
|
297
|
+
embed: `mlx-bun embed — text embeddings from a local embedding model
|
|
298
|
+
|
|
299
|
+
Usage: mlx-bun embed [query] --text "…" [options]
|
|
300
|
+
mlx-bun embed [query] "…" (text as a positional)
|
|
301
|
+
echo -e "line one\\nline two" | mlx-bun embed [query]
|
|
302
|
+
|
|
303
|
+
Embeds text with a local Qwen3-Embedding model (last-token pooled,
|
|
304
|
+
L2-normalized — bit-exact vs mlx-lm). One text per --text, or one per
|
|
305
|
+
line on stdin. Prints one JSON array (the vector) per input line.
|
|
306
|
+
|
|
307
|
+
[query] Embedding model (--query <q> is accepted for the
|
|
308
|
+
positional) [default: the first downloaded
|
|
309
|
+
embedding model; errors with an mlx-bun get
|
|
310
|
+
suggestion when none is downloaded]
|
|
311
|
+
|
|
312
|
+
Options:
|
|
313
|
+
--text "…" Text to embed (omit to read lines from stdin)
|
|
314
|
+
--instruct "…" Query instruction (Qwen3-Embedding query format);
|
|
315
|
+
steers WHICH similarity axis the geometry reflects.
|
|
316
|
+
Omit for plain document embeddings.
|
|
317
|
+
--json Emit one OpenAI-style {object:"list",data:[…]} object
|
|
318
|
+
instead of bare vector arrays
|
|
319
|
+
|
|
320
|
+
For a server endpoint use \`mlx-bun serve <embedding-model>\` then
|
|
321
|
+
POST /v1/embeddings.`,
|
|
322
|
+
|
|
323
|
+
generate: `mlx-bun generate — one-shot raw text generation (no server)
|
|
324
|
+
|
|
325
|
+
Usage: mlx-bun generate [query] --prompt "…" [options]
|
|
326
|
+
mlx-bun generate [query] "…" (prompt as a positional)
|
|
327
|
+
|
|
328
|
+
A direct one-shot entry point: load the model, generate, print, exit —
|
|
329
|
+
no server, no chat UI. By default the prompt is rendered through the
|
|
330
|
+
model's chat template; --raw feeds it verbatim.
|
|
331
|
+
|
|
332
|
+
[query] Model (registry query; --query is accepted for the
|
|
333
|
+
positional) [default: auto-pick]
|
|
334
|
+
|
|
335
|
+
Options:
|
|
336
|
+
--prompt "…" The prompt (or pass it as the positional)
|
|
337
|
+
--raw Skip the chat template (verbatim prompt)
|
|
338
|
+
--max-tokens <n> Completion cap [default: 256]
|
|
339
|
+
--temperature <n> Sampling temperature (--temp is accepted as an alias)
|
|
340
|
+
--top-p <n> Nucleus sampling
|
|
341
|
+
--top-k <n> Top-k sampling
|
|
342
|
+
--seed <n> Sampler seed (reproducible runs)
|
|
343
|
+
|
|
344
|
+
Decode-path levers mirror serve: --l1/--l2/--l3 and the per-fork flags
|
|
345
|
+
(--perf-kernel/--fused-sdpa/--compiled-decode/--fused-decode/--kv-quant).
|
|
346
|
+
The tier's KV scheme applies here too (--l2/--l3 = the model's kv_config,
|
|
347
|
+
same tokens as serve); with no tier/--kv-quant the KV cache stays bf16
|
|
348
|
+
(the bit-exact mlx-lm greedy path). mlx-lm compat = --l1.
|
|
349
|
+
|
|
350
|
+
Aliases: mlx-bun gen`,
|
|
351
|
+
|
|
231
352
|
evals: `mlx-bun evals — recent benchmark runs
|
|
232
353
|
|
|
233
354
|
Usage: mlx-bun evals [options]
|
|
@@ -261,7 +382,15 @@ Options:
|
|
|
261
382
|
--scale <f> LoRA scale [default: orpo 2.0 · else 1.0]
|
|
262
383
|
--seq <n> Max sequence length [default: gemma 8192 · else 4096]
|
|
263
384
|
--batch <n> Batch size [default: 1]
|
|
385
|
+
--grad-accum <n> Gradient accumulation steps (effective batch =
|
|
386
|
+
batch × grad-accum at batch-size-1 memory) [default: 1]
|
|
387
|
+
--grad-clip <f> Gradient-norm clip (0 = off) [default: 1.0]
|
|
388
|
+
--seed <n> Data-shuffle / init seed [default: 0]
|
|
389
|
+
--val-size <n> Max validation examples per eval [default: 256]
|
|
264
390
|
--lambda <f> ORPO odds-ratio weight [default: 0.1]
|
|
391
|
+
--sft-scope <s> ORPO chosen-NLL scope: full (paper/TRL-faithful,
|
|
392
|
+
prompt+response) | response (pre-2026-07 runs,
|
|
393
|
+
bit-exact) [default: full]
|
|
265
394
|
--seg <n> Layers per segment (segmented backward; orpo default 2)
|
|
266
395
|
--save-every <n> Crash-safe mountable checkpoint every n steps
|
|
267
396
|
--resume <dir> Warm-start LoRA weights from a checkpoint/adapter dir
|
|
@@ -274,7 +403,125 @@ ORPO defaults run the full stack: the [M,vocab]-free flash-CCE head +
|
|
|
274
403
|
prefix-sharing + segmented backward (each falls back + logs if a row's
|
|
275
404
|
preconditions aren't met). Gemma/e4b sets its required training env flags
|
|
276
405
|
automatically. Adapters are mountable directly (mlx-bun serve --adapter).
|
|
277
|
-
Run long jobs detached from your own shell: nohup mlx-bun train …
|
|
406
|
+
Run long jobs detached from your own shell: nohup mlx-bun train … &
|
|
407
|
+
Watch a run live from another tab: mlx-bun train-watch <adapter-dir>`,
|
|
408
|
+
|
|
409
|
+
"train-watch": `mlx-bun train-watch — live dashboard for a training run
|
|
410
|
+
|
|
411
|
+
Usage: mlx-bun train-watch [adapter-dir]
|
|
412
|
+
|
|
413
|
+
Tails <adapter-dir>/metrics.jsonl (written by mlx-bun train and the web
|
|
414
|
+
fine-tune tile) and renders a live loss/speed/memory dashboard. Point it
|
|
415
|
+
at the same --adapter directory the training run uses.
|
|
416
|
+
|
|
417
|
+
[adapter-dir] Adapter directory to watch; --adapter <dir> is
|
|
418
|
+
accepted for the positional
|
|
419
|
+
[default: ~/.cache/mlx-bun/mlx-bun-finetunes/orpo-cpm5]`,
|
|
420
|
+
fuse: `mlx-bun fuse — merge a LoRA adapter into the base weights
|
|
421
|
+
|
|
422
|
+
Usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [options]
|
|
423
|
+
|
|
424
|
+
Folds the adapter's low-rank deltas into the base weights and writes a
|
|
425
|
+
standalone, loadable model snapshot (weights + config + tokenizer). A
|
|
426
|
+
quantized base keeps its exact quantization layout: fused modules are
|
|
427
|
+
dequantized, updated, and re-quantized with their own per-module spec.
|
|
428
|
+
|
|
429
|
+
<model> Base model (registry query or a snapshot path);
|
|
430
|
+
--model is accepted as the mlx_lm.fuse spelling
|
|
431
|
+
|
|
432
|
+
Options:
|
|
433
|
+
--adapter <dir> Adapter directory (adapters.safetensors +
|
|
434
|
+
adapter_config.json). --adapter-path is accepted
|
|
435
|
+
as the mlx_lm.fuse alias [default: adapters]
|
|
436
|
+
--save-path <dir> Output model directory [default: fused_model]
|
|
437
|
+
|
|
438
|
+
Not supported (mlx_lm.fuse flags we don't implement — the command exits
|
|
439
|
+
with an error rather than guessing): --de-quantize, --export-gguf,
|
|
440
|
+
--gguf-path, --upload-repo.
|
|
441
|
+
|
|
442
|
+
Serve the result: mlx-bun serve <save-path>`,
|
|
443
|
+
|
|
444
|
+
convert: `mlx-bun convert — quantize an HF model into a local MLX snapshot
|
|
445
|
+
|
|
446
|
+
Usage: mlx-bun convert --hf-path <repo-or-path> -q [options]
|
|
447
|
+
mlx-bun convert <repo-or-path> -q [options] (positional source)
|
|
448
|
+
|
|
449
|
+
Wraps mlx-bun's native quantize pipeline (the same engine as the web
|
|
450
|
+
/api/quantize). The source may be a local model directory, a downloaded
|
|
451
|
+
model (registry query), or a Hugging Face repo id — an un-downloaded
|
|
452
|
+
repo is fetched first (resumable, verified; can be many GB).
|
|
453
|
+
|
|
454
|
+
Options:
|
|
455
|
+
--hf-path <src> Source model: local path, downloaded model, or HF
|
|
456
|
+
repo id (--model and the bare positional are
|
|
457
|
+
accepted as aliases)
|
|
458
|
+
--mlx-path <dir> Output directory [default: mlx_model; must not
|
|
459
|
+
already exist, matching mlx_lm.convert]
|
|
460
|
+
-q, --quantize Quantize the model (uniform affine)
|
|
461
|
+
--q-bits <n> Bits per weight: 4 or 8 [default: 4]
|
|
462
|
+
--q-group-size <n> Quantization group size: 32 or 64 [default: 64]
|
|
463
|
+
--upload-repo <id> Push the converted model to this Hugging Face repo
|
|
464
|
+
after converting (mlx_lm.convert parity; same engine
|
|
465
|
+
as \`mlx-bun upload\`, needs a write token)
|
|
466
|
+
|
|
467
|
+
Mixed precision (the mlx-bun differentiator — OptiQ sensitivity sweep +
|
|
468
|
+
knapsack per-layer bit allocation; implies quantization, no -q needed):
|
|
469
|
+
--target-bpw <f> Target bits-per-weight (e.g. 4.5). Runs
|
|
470
|
+
calibration → per-layer KL sensitivity → greedy
|
|
471
|
+
knapsack; writes per-module bits into config.json
|
|
472
|
+
--candidate-bits <l> Comma list the knapsack may pick from [default: 4,8]
|
|
473
|
+
--calibration-mix <m> "optiq" or a JSONL path [default: optiq]
|
|
474
|
+
--n-calibration <n> Calibration samples [default: 2]
|
|
475
|
+
|
|
476
|
+
Not supported (mlx_lm.convert flags we don't implement — the command
|
|
477
|
+
exits with an error rather than guessing): --dtype, -d/--dequantize,
|
|
478
|
+
--q-mode other than affine, and the mlx-lm --quant-predicate recipes
|
|
479
|
+
(use --target-bpw instead). Plain non-quantizing conversion is also not
|
|
480
|
+
supported: pass -q or --target-bpw.`,
|
|
481
|
+
|
|
482
|
+
upload: `mlx-bun upload — push a local model directory to the Hugging Face Hub
|
|
483
|
+
|
|
484
|
+
Usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [options]
|
|
485
|
+
|
|
486
|
+
Native push-to-hub, no Python (src/hf-push.ts — the same engine as the
|
|
487
|
+
web UI's push buttons): creates the repo if needed (idempotent), uploads
|
|
488
|
+
weights via the git-LFS batch protocol, and commits. Flag names match
|
|
489
|
+
mlx_lm.upload.
|
|
490
|
+
|
|
491
|
+
Options:
|
|
492
|
+
--path <dir> Local model directory to upload [default: mlx_model]
|
|
493
|
+
--upload-repo <id> Hub repo id, org/name or bare name (required)
|
|
494
|
+
--private Create the repo as private (mlx-bun extension)
|
|
495
|
+
|
|
496
|
+
Auth: needs a Hugging Face WRITE token, resolved from ~/.mlx-bun/hf.json
|
|
497
|
+
(web UI Settings → Hugging Face), $HF_TOKEN, or ~/.cache/huggingface/token
|
|
498
|
+
(hf auth login). Errors before uploading anything if none is found.
|
|
499
|
+
|
|
500
|
+
Convert + push in one step: mlx-bun convert … --upload-repo <org/repo>`,
|
|
501
|
+
|
|
502
|
+
perplexity: `mlx-bun perplexity — perplexity over a local text/JSONL dataset
|
|
503
|
+
|
|
504
|
+
Usage: mlx-bun perplexity <model-query-or-path> --data-path <file> [options]
|
|
505
|
+
|
|
506
|
+
mlx_lm.perplexity methodology, exactly: samples are visited in a seeded
|
|
507
|
+
random order, tokenized, concatenated, and cut into NON-OVERLAPPING rows
|
|
508
|
+
of --sequence-length tokens; per batch the model scores rows[:, :-1]
|
|
509
|
+
against rows[:, 1:] in f32 (every position counts); reported as
|
|
510
|
+
ppl = exp(mean CE) ± the delta-method standard error. The one deliberate
|
|
511
|
+
difference: the data source is a LOCAL file (never a Hugging Face
|
|
512
|
+
dataset download).
|
|
513
|
+
|
|
514
|
+
<model> Model (registry query or snapshot path);
|
|
515
|
+
--model (mlx_lm spelling) and --query are accepted
|
|
516
|
+
for the positional
|
|
517
|
+
|
|
518
|
+
Options:
|
|
519
|
+
--data-path <file> .jsonl ({"text": …} rows) or plain .txt (required)
|
|
520
|
+
--sequence-length <n> Tokens per row [default: 512]
|
|
521
|
+
--num-samples <n> Rows to score (-1 = all available) [default: 256]
|
|
522
|
+
--batch-size <n> Rows per forward [default: 8]
|
|
523
|
+
--seed <n> Sample-shuffle seed [default: 123]`,
|
|
524
|
+
|
|
278
525
|
memory: `mlx-bun memory — your local AI's personal wiki
|
|
279
526
|
|
|
280
527
|
Usage: mlx-bun memory <subcommand>
|
|
@@ -301,8 +548,16 @@ Subcommands:
|
|
|
301
548
|
Print one article section
|
|
302
549
|
links <article> Show resolved outbound + inbound wikilinks
|
|
303
550
|
read <article> Print an article (stem, e.g. Archie_Project)
|
|
304
|
-
synthesize Run the synthesis
|
|
305
|
-
--dry-run)
|
|
551
|
+
synthesize Run the FULL synthesis DAG now (--since, --model,
|
|
552
|
+
--dry-run); also: pipeline, all
|
|
553
|
+
segment | extract | route | synthesize-stage
|
|
554
|
+
Run ONE decomposed stage worker (--limit N —
|
|
555
|
+
segment/extract/synthesize-stage only; --convs a,b).
|
|
556
|
+
Each pulls its eligible work from the DB by state, walks
|
|
557
|
+
oldest-conversation-first, persists, and exits — resumable,
|
|
558
|
+
and runnable as separate concurrent processes on slices.
|
|
559
|
+
link Deterministic cross-linking stage: inline-link first
|
|
560
|
+
mentions + rebuild ## See also (--limit N; no model)
|
|
306
561
|
schedule Install the nightly launchd job (--at HH:MM [03:00])
|
|
307
562
|
unschedule Remove the nightly launchd job
|
|
308
563
|
|
|
@@ -310,12 +565,13 @@ The read path is live: the assistant reads a wiki you set up by hand or
|
|
|
310
565
|
import during \`memory init\`. Ask the assistant to "open my memory" or run
|
|
311
566
|
\`mlx-bun memory open\` to browse it in Obsidian/Finder, or
|
|
312
567
|
\`mlx-bun memory open <article>\` to jump to a specific page. Synthesis
|
|
313
|
-
(conversations → articles)
|
|
314
|
-
|
|
315
|
-
|
|
568
|
+
(conversations → articles) runs the full local pipeline via
|
|
569
|
+
\`mlx-bun memory synthesize\` (or per-stage: segment/extract/route/
|
|
570
|
+
synthesize-stage/link); the nightly job runs it on a schedule.`,
|
|
316
571
|
};
|
|
317
572
|
|
|
318
573
|
HELP.bench = HELP.benchmark!;
|
|
574
|
+
HELP.gen = HELP.generate!;
|
|
319
575
|
HELP.setup = HELP.memory!;
|
|
320
576
|
|
|
321
577
|
function printHelp(topic?: string): never {
|
|
@@ -454,11 +710,16 @@ function openChatUi(url: string, hostPort: string): void {
|
|
|
454
710
|
* Each tier is a GUARANTEE about which reference you reproduce bit-for-bit:
|
|
455
711
|
* --l1 bit-for-bit IDENTICAL to mlx-lm — drop-in replacement for mlx-lm
|
|
456
712
|
* --l2 bit-for-bit IDENTICAL to mlx-optiq — drop-in replacement for mlx-optiq.
|
|
457
|
-
* optiq
|
|
458
|
-
*
|
|
713
|
+
* The optiq-golden decode composition (scripts/regen-kvq-goldens.ts) is
|
|
714
|
+
* fused N-tiled SDPA for L>1 + STOCK unfused L=1 decode, so L2 = fused-sdpa
|
|
715
|
+
* ON, perf-kernel OFF. The perf kernel is an mlx-bun ORIGINAL Metal kernel,
|
|
716
|
+
* envelope-gated (≥56/64 teacher-forced argmax vs OUR OWN frozen compat
|
|
717
|
+
* trajectory, tests/perf-kernel-oracle.test.ts) — an L3 node, never the
|
|
718
|
+
* bare-L2 default. (Commit f1bf5cc put it in L2 claiming the goldens track
|
|
719
|
+
* it; they don't — reverted 2026-07-01 with the evidence above.)
|
|
459
720
|
* --l3 best performance, NO bit-exact guarantee — correct, but gated by KL +
|
|
460
721
|
* tests, because mlx-optiq has no analogy for what L3 does.
|
|
461
|
-
* L2 ⊂ L3 (L3 = L2 +
|
|
722
|
+
* L2 ⊂ L3 (L3 = L2 + perf kernel). L1 is a separate target (bf16, unfused). */
|
|
462
723
|
function applyDecodeRoute(): { kvQuant?: "off" | "config" | number } {
|
|
463
724
|
const onOff = (name: string): boolean | null => {
|
|
464
725
|
const v = opt(name); if (v == null) return null;
|
|
@@ -471,15 +732,18 @@ function applyDecodeRoute(): { kvQuant?: "off" | "config" | number } {
|
|
|
471
732
|
// A tier is the GUARANTEE; within it we default to the FAST kernel that still
|
|
472
733
|
// holds it, and expose the slow one as an opt-in. compiled-decode is BIT-EXACT
|
|
473
734
|
// with uncompiled (compiled-decode.test: on==off) → free speed, so it's ON in
|
|
474
|
-
// EVERY tier (--compiled-decode off is the slow, same-tier opt-in).
|
|
475
|
-
//
|
|
476
|
-
// quantized-KV only →
|
|
477
|
-
//
|
|
478
|
-
//
|
|
479
|
-
//
|
|
735
|
+
// EVERY tier (--compiled-decode off is the slow, same-tier opt-in). fused-sdpa
|
|
736
|
+
// matches OPTIQ bit-for-bit (tier-a goldens, fused-sdpa.test) and is
|
|
737
|
+
// quantized-KV only → it's the L2 bridge, a no-op on L1's bf16. perf-kernel is
|
|
738
|
+
// NOT: it's an original flash-decoding kernel, envelope-gated against our own
|
|
739
|
+
// frozen trajectory (perf-kernel-oracle.test) — bare --l2 must ship the
|
|
740
|
+
// optiq-golden decode composition (stock unfused L=1), so perf-kernel is
|
|
741
|
+
// L3/explicit-only (--perf-kernel on opts a tier into it by choice). This is
|
|
742
|
+
// the decode-axis difference between L2 and L3; L3 also owns the no-oracle
|
|
743
|
+
// FEATURES (HLG sampler, expert offload, batched mixed-precision).
|
|
480
744
|
l1: { kv: "off", perf: false, fusedSdpa: false, compiled: true, fusedDecode: false }, // = mlx-lm bit-for-bit (bf16)
|
|
481
|
-
l2: { kv: "config", perf:
|
|
482
|
-
l3: { kv: "config", perf: true, fusedSdpa: true, compiled: true, fusedDecode: false }, // best perf (decode
|
|
745
|
+
l2: { kv: "config", perf: false, fusedSdpa: true, compiled: true, fusedDecode: false }, // = mlx-optiq bit-for-bit
|
|
746
|
+
l3: { kv: "config", perf: true, fusedSdpa: true, compiled: true, fusedDecode: false }, // best perf (envelope-gated decode)
|
|
483
747
|
};
|
|
484
748
|
const tier = flag("l1") ? "l1" : flag("l2") ? "l2" : flag("l3") ? "l3" : null;
|
|
485
749
|
const p = tier ? TIERS[tier]! : null;
|
|
@@ -522,8 +786,11 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
|
|
|
522
786
|
const pcRaw = opt("prompt-cache"); // null if absent; an explicit value (incl. 0) wins → `--prompt-cache 0` DISABLES the cache
|
|
523
787
|
if (pcRaw !== null) serverOptions.promptCacheBytes = Math.max(0, Number(pcRaw)) * 2 ** 30;
|
|
524
788
|
// --batch N: max concurrent requests batched through the mlx-lm-parity
|
|
525
|
-
// engine (N=1 = today's serial path). --decode-concurrency is accepted
|
|
526
|
-
//
|
|
789
|
+
// engine (N=1 = today's serial path). --decode-concurrency is accepted for
|
|
790
|
+
// drop-in compatibility with mlx_lm.server, but the semantics differ: there
|
|
791
|
+
// it caps per-BatchGenerator decode parallelism (default 32); in mlx-bun it
|
|
792
|
+
// enables continuous batching with this cap (the default is the optimized
|
|
793
|
+
// serial path).
|
|
527
794
|
const batchRaw = opt("batch") ?? opt("decode-concurrency");
|
|
528
795
|
if (batchRaw !== null) {
|
|
529
796
|
const n = Number(batchRaw);
|
|
@@ -534,8 +801,11 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
|
|
|
534
801
|
serverOptions.batch = n;
|
|
535
802
|
}
|
|
536
803
|
if (route.kvQuant !== undefined) serverOptions.kvQuant = route.kvQuant;
|
|
537
|
-
|
|
538
|
-
|
|
804
|
+
// Bind loopback unless asked otherwise (mlx_lm.server parity); --host
|
|
805
|
+
// 0.0.0.0 is the explicit opt-in for LAN exposure. The chat-UI open and
|
|
806
|
+
// `mlx-bun pi` attach both go through localhost, so loopback-only is
|
|
807
|
+
// transparent to them.
|
|
808
|
+
serverOptions.hostname = opt("host", "127.0.0.1")!;
|
|
539
809
|
// Server-wide default for the chat template's enable_thinking variable
|
|
540
810
|
// (CPM/MiniCPM5 and other hybrid-reasoning models). Unset ⇒ the model's
|
|
541
811
|
// own default (false for MiniCPM5). onOff accepts true|false|on|off|1|0.
|
|
@@ -554,12 +824,18 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
|
|
|
554
824
|
}
|
|
555
825
|
return n;
|
|
556
826
|
};
|
|
557
|
-
|
|
827
|
+
// --temp is the mlx_lm.server spelling; explicit --temperature wins.
|
|
828
|
+
const temp = numFlag("temperature", 0, 5) ?? numFlag("temp", 0, 5);
|
|
558
829
|
if (temp !== null) serverOptions.defaultTemperature = temp;
|
|
559
830
|
const topP = numFlag("top-p", 0, 1);
|
|
560
831
|
if (topP !== null) serverOptions.defaultTopP = topP;
|
|
561
832
|
const topK = numFlag("top-k", 0, 1_000_000);
|
|
562
833
|
if (topK !== null) serverOptions.defaultTopK = topK;
|
|
834
|
+
// mlx_lm.server's --max-tokens: the completion cap when a request omits
|
|
835
|
+
// max_tokens. Our built-in default stays 65,536 (chat) / 512 (raw text
|
|
836
|
+
// completion); `--max-tokens 512` reproduces mlx-lm's behavior exactly.
|
|
837
|
+
const maxTok = numFlag("max-tokens", 1, 10_000_000);
|
|
838
|
+
if (maxTok !== null) serverOptions.defaultMaxTokens = Math.floor(maxTok);
|
|
563
839
|
// HLG sampling — piecewise tone curve on the logprobs (default off). Knobs in
|
|
564
840
|
// nats; the mid gain folds from --temperature. docs/design/hlg-sampling.md.
|
|
565
841
|
if (onOff("hlg-sampling") === true) {
|
|
@@ -572,7 +848,33 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
|
|
|
572
848
|
pivot: "top",
|
|
573
849
|
};
|
|
574
850
|
}
|
|
575
|
-
return { port: Number(opt("port", "
|
|
851
|
+
return { port: Number(opt("port", "8080")), serverOptions };
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/** `--adapter <dir>` (alias `--adapter-path`, mlx_lm.server's spelling):
|
|
855
|
+
* mount a LoRA adapter at startup — the same machinery as POST /v1/adapters —
|
|
856
|
+
* and make it the default for requests that don't select one (a request's
|
|
857
|
+
* explicit `adapter`, including "none", still wins; hot-swap unchanged).
|
|
858
|
+
* This is the flag `mlx-bun train`'s completion message points at. Shared by
|
|
859
|
+
* `serve` and the `pi` session server. Exits on a bad adapter — better a
|
|
860
|
+
* loud startup failure than silently serving the base model. */
|
|
861
|
+
async function mountStartupAdapter(
|
|
862
|
+
ctx: import("./server").ServerContext,
|
|
863
|
+
serverOptions: import("./server").ServerOptions,
|
|
864
|
+
): Promise<void> {
|
|
865
|
+
const adapterDir = opt("adapter") ?? opt("adapter-path");
|
|
866
|
+
if (!adapterDir) return;
|
|
867
|
+
const { step, style } = await import("./tui");
|
|
868
|
+
const sAdp = step("mounting adapter");
|
|
869
|
+
try {
|
|
870
|
+
const adapterId = adapterDir.replace(/\/+$/, "").split("/").pop()!;
|
|
871
|
+
const info = await ctx.adapters.mount(adapterId, adapterDir);
|
|
872
|
+
serverOptions.defaultAdapter = info.id;
|
|
873
|
+
sAdp.done(`adapter ${style.bold(info.id)} ${style.dim(`· ${info.mountedLayers} layers · default for requests (select others via \`adapter\`)`)}`);
|
|
874
|
+
} catch (e) {
|
|
875
|
+
sAdp.fail(`adapter mount failed: ${(e as Error).message}`);
|
|
876
|
+
process.exit(1);
|
|
877
|
+
}
|
|
576
878
|
}
|
|
577
879
|
|
|
578
880
|
/** One-line summary of the active runtime levers for the ready card. */
|
|
@@ -664,13 +966,33 @@ function parseSize(s: string): number {
|
|
|
664
966
|
|
|
665
967
|
switch (cmd) {
|
|
666
968
|
case "get": {
|
|
667
|
-
|
|
668
|
-
if (!repoId
|
|
669
|
-
console.error("usage: mlx-bun get <org/repo> [--revision main]");
|
|
969
|
+
let repoId = positional(0);
|
|
970
|
+
if (!repoId) {
|
|
971
|
+
console.error("usage: mlx-bun get <org/repo | substring> [--revision main]");
|
|
670
972
|
process.exit(1);
|
|
671
973
|
}
|
|
974
|
+
if (!repoId.includes("/")) {
|
|
975
|
+
// No org/name — treat it as a registry query over already-downloaded
|
|
976
|
+
// repos (re-get/refresh, e.g. `mlx-bun get 12B` after an upstream push).
|
|
977
|
+
try {
|
|
978
|
+
const regQ = new Registry();
|
|
979
|
+
if (regQ.list().length === 0) await regQ.scan();
|
|
980
|
+
repoId = regQ.resolve(repoId).repoId;
|
|
981
|
+
} catch (e) {
|
|
982
|
+
console.error((e as Error).message);
|
|
983
|
+
console.error(`usage: mlx-bun get <org/repo> [--revision main] — or a substring of a downloaded repo (try \`mlx-bun ls ${repoId}\`)`);
|
|
984
|
+
process.exit(1);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
672
987
|
const { downloadModel } = await import("./download");
|
|
673
988
|
const { step, style } = await import("./tui");
|
|
989
|
+
const { join: joinPath, basename } = await import("node:path");
|
|
990
|
+
const { readdirSync, existsSync } = await import("node:fs");
|
|
991
|
+
const { DEFAULT_HUB, planRepoGc } = await import("./registry");
|
|
992
|
+
const repoDir = joinPath(DEFAULT_HUB, `models--${repoId.replaceAll("/", "--")}`);
|
|
993
|
+
const priorSnapshots = existsSync(joinPath(repoDir, "snapshots"))
|
|
994
|
+
? readdirSync(joinPath(repoDir, "snapshots"))
|
|
995
|
+
: [];
|
|
674
996
|
const s = step(`downloading ${repoId}`);
|
|
675
997
|
const snap = await downloadModel(repoId, {
|
|
676
998
|
revision: opt("revision", "main")!,
|
|
@@ -684,6 +1006,18 @@ switch (cmd) {
|
|
|
684
1006
|
const reg = new Registry();
|
|
685
1007
|
await reg.scan();
|
|
686
1008
|
sScan.done(`registry updated ${style.dim(`· ${snap}`)}`);
|
|
1009
|
+
// Upstream pushed a new revision → this get created a NEW snapshots/<sha>
|
|
1010
|
+
// dir; the old one stays on disk (that's how ~25 GB of dead blobs pile
|
|
1011
|
+
// up). Say so, with what `gc` would get back.
|
|
1012
|
+
if (priorSnapshots.length > 0 && !priorSnapshots.includes(basename(snap))) {
|
|
1013
|
+
const plan = planRepoGc(repoDir);
|
|
1014
|
+
const note = plan.skippedSnapshots.length > 0
|
|
1015
|
+
? " (a previous snapshot has files this revision lacks — gc will warn; --force to prune it)"
|
|
1016
|
+
: "";
|
|
1017
|
+
console.log(style.dim(
|
|
1018
|
+
` previous revision kept on disk — \`mlx-bun gc\` reclaims ~${gb(plan.reclaimBytes)}${note}`,
|
|
1019
|
+
));
|
|
1020
|
+
}
|
|
687
1021
|
break;
|
|
688
1022
|
}
|
|
689
1023
|
|
|
@@ -700,43 +1034,125 @@ switch (cmd) {
|
|
|
700
1034
|
const reg = new Registry();
|
|
701
1035
|
if ((reg.list().length ?? 0) === 0) await reg.scan();
|
|
702
1036
|
const maxSize = opt("max-size");
|
|
703
|
-
const
|
|
1037
|
+
const allRevisions = flag("all-revisions");
|
|
1038
|
+
const lsFilter = {
|
|
704
1039
|
vision: flag("vision") ? true : undefined,
|
|
705
1040
|
maxBytes: maxSize ? parseSize(maxSize) : undefined,
|
|
706
1041
|
query: positional(0),
|
|
707
|
-
}
|
|
1042
|
+
};
|
|
1043
|
+
// One row per repo by default: the HF cache keeps a snapshots/<commit>
|
|
1044
|
+
// dir per downloaded revision, and re-getting after an upstream push
|
|
1045
|
+
// strands the old one — those are duplicates, not separate models.
|
|
1046
|
+
// --all-revisions shows the per-snapshot truth (canonical marked *).
|
|
1047
|
+
const models = allRevisions ? reg.list(lsFilter) : reg.listCanonical(lsFilter);
|
|
708
1048
|
if (models.length === 0) {
|
|
709
1049
|
console.log("no models match (try `mlx-bun scan`)");
|
|
710
1050
|
break;
|
|
711
1051
|
}
|
|
1052
|
+
const { visionCapable } = await import("./registry");
|
|
1053
|
+
const { supportTier } = await import("./model/support");
|
|
1054
|
+
const canonicalPaths = new Set(reg.listCanonical(lsFilter).map((m) => m.path));
|
|
1055
|
+
const capabilities = (m: (typeof models)[number]) => {
|
|
1056
|
+
const tier = supportTier(m.modelType, m.repoId);
|
|
1057
|
+
return [
|
|
1058
|
+
tier ? `supported (${tier})` : `unsupported (${m.modelType})`,
|
|
1059
|
+
visionCapable(m) ? "vision" : null,
|
|
1060
|
+
m.hasToolTemplate ? "tools" : null,
|
|
1061
|
+
m.hasKvConfig ? "kv-quant" : null,
|
|
1062
|
+
].filter(Boolean).join(" · ");
|
|
1063
|
+
};
|
|
712
1064
|
const { table, style, h1 } = await import("./tui");
|
|
713
1065
|
h1("library");
|
|
714
1066
|
console.log();
|
|
715
1067
|
table(
|
|
716
1068
|
[
|
|
717
1069
|
{ header: "model", paint: (c) => style.bold(c) },
|
|
718
|
-
{ header: "
|
|
719
|
-
{ header: "
|
|
1070
|
+
...(allRevisions ? [{ header: "revision" }] : []),
|
|
1071
|
+
{ header: "size", align: "right" as const },
|
|
1072
|
+
{ header: "params", align: "right" as const },
|
|
720
1073
|
{ header: "quant" },
|
|
721
|
-
{ header: "license", paint: (c) => style.dim(c) },
|
|
722
|
-
{ header: "capabilities", paint: (c) => style.dim(c) },
|
|
1074
|
+
{ header: "license", paint: (c: string) => style.dim(c) },
|
|
1075
|
+
{ header: "capabilities", paint: (c: string) => style.dim(c) },
|
|
723
1076
|
],
|
|
724
1077
|
models.map((m) => [
|
|
725
1078
|
m.repoId,
|
|
1079
|
+
...(allRevisions
|
|
1080
|
+
? [`${(m.path.split("/snapshots/")[1] ?? "").slice(0, 12)}${canonicalPaths.has(m.path) ? " *" : ""}`]
|
|
1081
|
+
: []),
|
|
726
1082
|
gb(m.sizeBytes),
|
|
727
1083
|
m.paramCount ? `${(m.paramCount / 1e9).toFixed(1)}B` : "?",
|
|
728
1084
|
m.quantBits ? `${m.quantBits}-bit g${m.quantGroupSize}` : "full",
|
|
729
1085
|
m.license ?? "?",
|
|
730
|
-
|
|
731
|
-
isSupportedModelRecord(m.modelType, m.repoId) ? "supported" : `unsupported (${m.modelType})`,
|
|
732
|
-
m.hasVisionSidecar ? "vision" : null,
|
|
733
|
-
m.hasToolTemplate ? "tools" : null,
|
|
734
|
-
m.hasKvConfig ? "kv-quant" : null,
|
|
735
|
-
].filter(Boolean).join(" · "),
|
|
1086
|
+
capabilities(m),
|
|
736
1087
|
]),
|
|
737
1088
|
);
|
|
738
1089
|
console.log();
|
|
739
|
-
|
|
1090
|
+
if (allRevisions) {
|
|
1091
|
+
console.log(style.dim(
|
|
1092
|
+
` ${models.length} snapshot(s) · * = canonical (refs/main) · superseded snapshots: \`mlx-bun gc\``,
|
|
1093
|
+
));
|
|
1094
|
+
} else {
|
|
1095
|
+
console.log(style.dim(` ${models.length} model(s) · mlx-bun fit <query> for a memory assessment`));
|
|
1096
|
+
}
|
|
1097
|
+
break;
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
case "gc": {
|
|
1101
|
+
const { planGc, executeGc, DEFAULT_HUB } = await import("./registry");
|
|
1102
|
+
const { table, style, h1 } = await import("./tui");
|
|
1103
|
+
const force = flag("force");
|
|
1104
|
+
const plans = planGc(DEFAULT_HUB, { force }).filter(
|
|
1105
|
+
(p) => p.pruneSnapshots.length || p.skippedSnapshots.length || p.deadBlobs.length,
|
|
1106
|
+
);
|
|
1107
|
+
h1("gc — superseded snapshots + dead blobs");
|
|
1108
|
+
console.log();
|
|
1109
|
+
if (plans.length === 0) {
|
|
1110
|
+
console.log(" nothing to reclaim — every snapshot is referenced by refs/*");
|
|
1111
|
+
break;
|
|
1112
|
+
}
|
|
1113
|
+
table(
|
|
1114
|
+
[
|
|
1115
|
+
{ header: "repo", paint: (c) => style.bold(c) },
|
|
1116
|
+
{ header: "keep", align: "right" },
|
|
1117
|
+
{ header: "prune", align: "right" },
|
|
1118
|
+
{ header: "skip", align: "right" },
|
|
1119
|
+
{ header: "reclaims", align: "right" },
|
|
1120
|
+
],
|
|
1121
|
+
plans.map((p) => [
|
|
1122
|
+
p.repoId,
|
|
1123
|
+
String(p.keepSnapshots.length),
|
|
1124
|
+
String(p.pruneSnapshots.length),
|
|
1125
|
+
String(p.skippedSnapshots.length),
|
|
1126
|
+
gb(p.reclaimBytes),
|
|
1127
|
+
]),
|
|
1128
|
+
);
|
|
1129
|
+
console.log();
|
|
1130
|
+
for (const p of plans) {
|
|
1131
|
+
for (const s of p.skippedSnapshots) {
|
|
1132
|
+
const rev = (s.path.split("/snapshots/")[1] ?? s.path).slice(0, 12);
|
|
1133
|
+
console.log(style.accent(
|
|
1134
|
+
` WARNING ${p.repoId}@${rev}: superseded snapshot has files the canonical revision lacks — skipped.`,
|
|
1135
|
+
));
|
|
1136
|
+
console.log(style.dim(` only copy of: ${s.extraFiles.join(", ")}`));
|
|
1137
|
+
console.log(style.dim(" deleting it loses those files — rerun with --force if that's intended."));
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
const totalReclaim = plans.reduce((a, p) => a + p.reclaimBytes, 0);
|
|
1141
|
+
const totalSnaps = plans.reduce((a, p) => a + p.pruneSnapshots.length, 0);
|
|
1142
|
+
const totalBlobs = plans.reduce((a, p) => a + p.deadBlobs.length, 0);
|
|
1143
|
+
console.log(
|
|
1144
|
+
` total: ${totalSnaps} snapshot(s), ${totalBlobs} blob(s), ${style.bold(gb(totalReclaim))}`,
|
|
1145
|
+
);
|
|
1146
|
+
if (flag("dry-run") || !flag("yes")) {
|
|
1147
|
+
console.log(style.dim(" nothing deleted — rerun with --yes to delete (destructive)."));
|
|
1148
|
+
break;
|
|
1149
|
+
}
|
|
1150
|
+
const res = executeGc(plans);
|
|
1151
|
+
console.log(
|
|
1152
|
+
` deleted ${res.snapshots} snapshot(s) + ${res.blobs} blob(s) — reclaimed ${style.bold(gb(res.reclaimedBytes))}`,
|
|
1153
|
+
);
|
|
1154
|
+
const reg = new Registry();
|
|
1155
|
+
await reg.scan(); // reap deleted snapshots from the registry
|
|
740
1156
|
break;
|
|
741
1157
|
}
|
|
742
1158
|
|
|
@@ -829,8 +1245,9 @@ switch (cmd) {
|
|
|
829
1245
|
const sLoad = step("loading weights");
|
|
830
1246
|
const t0 = performance.now();
|
|
831
1247
|
const ctx = await loadContext(m.path, m.repoId, { memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes });
|
|
832
|
-
const server = createServer(ctx, rt.port, { ...rt.serverOptions, owner: "serve" });
|
|
833
1248
|
sLoad.done(`weights loaded ${style.dim(`in ${(performance.now() - t0).toFixed(0)} ms`)}`);
|
|
1249
|
+
await mountStartupAdapter(ctx, rt.serverOptions);
|
|
1250
|
+
const server = createServer(ctx, rt.port, { ...rt.serverOptions, owner: "serve" });
|
|
834
1251
|
const shownHost = rt.serverOptions.hostname ?? "localhost";
|
|
835
1252
|
console.log();
|
|
836
1253
|
box([
|
|
@@ -860,29 +1277,109 @@ switch (cmd) {
|
|
|
860
1277
|
// RAW generate — a direct one-shot entry point (no server, no chat UI). The
|
|
861
1278
|
// three entry points (raw / OpenAI API / chat UI) differ only in how params
|
|
862
1279
|
// are POPULATED; here every param is explicit. Decode-path levers mirror
|
|
863
|
-
// serve so you can pin the route (mlx-lm compat = --
|
|
864
|
-
// --
|
|
865
|
-
|
|
1280
|
+
// serve so you can pin the route (mlx-lm compat = --l1).
|
|
1281
|
+
// --l1/--l2/--l3 + per-fork overrides set the decode env levers; the
|
|
1282
|
+
// returned KV scheme is applied below so `generate --l2` runs the same
|
|
1283
|
+
// quantized-KV route (and produces the same tokens) as `serve --l2`.
|
|
1284
|
+
const route = applyDecodeRoute();
|
|
866
1285
|
|
|
867
1286
|
const prompt = opt("prompt") ?? positional(1);
|
|
868
1287
|
if (!prompt) {
|
|
869
1288
|
console.error('usage: mlx-bun generate [query] --prompt "…" [--raw] [--max-tokens N]');
|
|
870
|
-
console.error(' sampling: --temperature N --top-p N --top-k N --seed N');
|
|
871
|
-
console.error(' decode path: --
|
|
1289
|
+
console.error(' sampling: --temperature N (alias --temp) --top-p N --top-k N --seed N');
|
|
1290
|
+
console.error(' decode path: --l1 (= mlx-lm compat) / --l2 / --l3, or per-fork flags');
|
|
872
1291
|
process.exit(1);
|
|
873
1292
|
}
|
|
874
1293
|
const { loadTaskModel, generateText } = await import("./eval/runner");
|
|
875
1294
|
const tm = await loadTaskModel(positional(0) ?? opt("query") ?? "");
|
|
1295
|
+
// Resolve the tier/--kv-quant KV scheme exactly the way serve does
|
|
1296
|
+
// (server.ts kvScheme): "config" = the model's kv_config.json (absent →
|
|
1297
|
+
// bf16, same silent fallback as serve), N = uniform bits from decode
|
|
1298
|
+
// start. Unset (no tier, no --kv-quant) keeps generate's historical
|
|
1299
|
+
// default: bf16 bit-exact greedy (generateText's parity path).
|
|
1300
|
+
const kvScheme =
|
|
1301
|
+
route.kvQuant === "off" ? {}
|
|
1302
|
+
: route.kvQuant === "config"
|
|
1303
|
+
? (tm.config.kvQuant?.length ? { kvConfig: tm.config.kvQuant } : {})
|
|
1304
|
+
: typeof route.kvQuant === "number"
|
|
1305
|
+
? { kvBits: route.kvQuant, quantizedKvStart: 0 }
|
|
1306
|
+
: undefined;
|
|
876
1307
|
const num = (n: string): number | undefined => { const v = opt(n); return v == null ? undefined : Number(v); };
|
|
877
1308
|
const text = await generateText(tm, prompt, {
|
|
878
1309
|
maxTokens: Number(opt("max-tokens", "256")),
|
|
879
1310
|
useChat: !flag("raw"),
|
|
880
|
-
|
|
1311
|
+
...(kvScheme !== undefined ? { kvScheme } : {}),
|
|
1312
|
+
sampler: { temperature: num("temperature") ?? num("temp"), topP: num("top-p"), topK: num("top-k"), seed: num("seed") },
|
|
881
1313
|
});
|
|
882
1314
|
process.stdout.write(text.endsWith("\n") ? text : text + "\n");
|
|
883
1315
|
break;
|
|
884
1316
|
}
|
|
885
1317
|
|
|
1318
|
+
case "embed": {
|
|
1319
|
+
// Raw text embeddings — one-shot, no server. Text from --text, or one per
|
|
1320
|
+
// line on stdin. Uses the same embedPooled path the parity test verifies.
|
|
1321
|
+
const { Weights } = await import("./weights");
|
|
1322
|
+
const { createModel } = await import("./model/factory");
|
|
1323
|
+
const { loadTokenizer } = await import("./tokenizer");
|
|
1324
|
+
const { embedMany, isEmbeddingModel } = await import("./embed");
|
|
1325
|
+
|
|
1326
|
+
let texts: string[];
|
|
1327
|
+
const oneText = opt("text") ?? positional(1);
|
|
1328
|
+
if (oneText != null) {
|
|
1329
|
+
texts = [oneText];
|
|
1330
|
+
} else if (!process.stdin.isTTY) {
|
|
1331
|
+
texts = (await Bun.stdin.text()).split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
|
|
1332
|
+
} else {
|
|
1333
|
+
texts = [];
|
|
1334
|
+
}
|
|
1335
|
+
if (texts.length === 0) {
|
|
1336
|
+
console.error('usage: mlx-bun embed [query] --text "…" (or pipe text, one per line)');
|
|
1337
|
+
console.error(' --instruct "…" query instruction · --json OpenAI-style output');
|
|
1338
|
+
process.exit(1);
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
// Model resolution: an explicit query resolves normally; the no-query
|
|
1342
|
+
// path auto-picks the first DOWNLOADED embedding model (plain Qwen3 —
|
|
1343
|
+
// the record-level equivalent of isEmbeddingModel, which the factory
|
|
1344
|
+
// maps to Qwen3Model). Never the chat default, and never the fresh-
|
|
1345
|
+
// install starter download — a chat model can't embed, so downloading
|
|
1346
|
+
// one here only to fail the isEmbeddingModel check would be a trap.
|
|
1347
|
+
const embedQuery = positional(0) ?? opt("query");
|
|
1348
|
+
let m: import("./registry").ModelRecord;
|
|
1349
|
+
if (embedQuery) {
|
|
1350
|
+
({ m } = await resolveModelAuto(embedQuery));
|
|
1351
|
+
} else {
|
|
1352
|
+
const reg = new Registry();
|
|
1353
|
+
if (reg.list().length === 0) await reg.scan();
|
|
1354
|
+
const embedder = reg.list().find((r) => r.modelType === "qwen3");
|
|
1355
|
+
if (!embedder) {
|
|
1356
|
+
console.error("no embedding model downloaded — try: mlx-bun get mlx-community/Qwen3-Embedding-4B-4bit-DWQ");
|
|
1357
|
+
process.exit(1);
|
|
1358
|
+
}
|
|
1359
|
+
m = embedder;
|
|
1360
|
+
}
|
|
1361
|
+
const config = await loadModelConfig(m.path);
|
|
1362
|
+
const model = createModel(await Weights.open(m.path), config);
|
|
1363
|
+
if (!isEmbeddingModel(model)) {
|
|
1364
|
+
console.error(`"${m.repoId}" is not an embedding model (need plain Qwen3, e.g. Qwen3-Embedding).`);
|
|
1365
|
+
process.exit(1);
|
|
1366
|
+
}
|
|
1367
|
+
const tok = await loadTokenizer(m.path);
|
|
1368
|
+
const results = embedMany(model, tok, texts, opt("instruct") ?? undefined);
|
|
1369
|
+
|
|
1370
|
+
if (flag("json")) {
|
|
1371
|
+
let total = 0;
|
|
1372
|
+
const data = results.map((r, index) => { total += r.tokens; return { object: "embedding", index, embedding: Array.from(r.vector) }; });
|
|
1373
|
+
process.stdout.write(JSON.stringify({
|
|
1374
|
+
object: "list", data, model: m.repoId,
|
|
1375
|
+
usage: { prompt_tokens: total, total_tokens: total },
|
|
1376
|
+
}) + "\n");
|
|
1377
|
+
} else {
|
|
1378
|
+
for (const r of results) process.stdout.write(JSON.stringify(Array.from(r.vector)) + "\n");
|
|
1379
|
+
}
|
|
1380
|
+
break;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
886
1383
|
case "bench":
|
|
887
1384
|
case "benchmark": {
|
|
888
1385
|
const { banner, step, box, style } = await import("./tui");
|
|
@@ -1077,10 +1574,12 @@ switch (cmd) {
|
|
|
1077
1574
|
// already run their own pi connect it with `mlx-bun harness pi`.
|
|
1078
1575
|
const OURS_VAL = new Set([
|
|
1079
1576
|
"--query", "-q", "--port", "--host", "--memory-budget", "--prompt-cache", "--kv-quant",
|
|
1577
|
+
"--batch", "--decode-concurrency", "--adapter", "--adapter-path",
|
|
1080
1578
|
"--compiled-decode", "--perf-kernel", "--fused-decode", "--fused-sdpa", "--thinking",
|
|
1081
|
-
"--temperature", "--top-p", "--top-k",
|
|
1579
|
+
"--temperature", "--temp", "--top-p", "--top-k", "--max-tokens",
|
|
1580
|
+
"--hlg-sampling", "--hlg-width", "--hlg-shoulder", "--hlg-toe", "--hlg-pivot-offset",
|
|
1082
1581
|
]);
|
|
1083
|
-
const OURS_BOOL = new Set(["--force-wire", "--expert-offload"]);
|
|
1582
|
+
const OURS_BOOL = new Set(["--force-wire", "--expert-offload", "--no-open", "--l1", "--l2", "--l3"]);
|
|
1084
1583
|
const passthrough: string[] = [];
|
|
1085
1584
|
for (let i = 1; i < argv.length; i++) {
|
|
1086
1585
|
const a = argv[i]!;
|
|
@@ -1108,7 +1607,7 @@ switch (cmd) {
|
|
|
1108
1607
|
|
|
1109
1608
|
const rt = serverRuntimeFlags();
|
|
1110
1609
|
const port = rt.port;
|
|
1111
|
-
const baseUrl = `http://
|
|
1610
|
+
const baseUrl = `http://127.0.0.1:${port}/v1`;
|
|
1112
1611
|
const { probeServer } = await import("./harness-pi");
|
|
1113
1612
|
let models = await probeServer(baseUrl);
|
|
1114
1613
|
let startedServer = false;
|
|
@@ -1123,7 +1622,7 @@ switch (cmd) {
|
|
|
1123
1622
|
console.log(style.dim(" (stop it or use --port <other> to start a fresh server with these flags)"));
|
|
1124
1623
|
}
|
|
1125
1624
|
try {
|
|
1126
|
-
const stats = await (await fetch(`http://
|
|
1625
|
+
const stats = await (await fetch(`http://127.0.0.1:${port}/stats`)).json() as
|
|
1127
1626
|
{ server?: { owner?: string } };
|
|
1128
1627
|
if (stats.server?.owner === "pi-session") {
|
|
1129
1628
|
console.log("note: that server belongs to another `mlx-bun pi` session and ends when it does —");
|
|
@@ -1143,6 +1642,7 @@ switch (cmd) {
|
|
|
1143
1642
|
const sLoad = step(`loading ${m.repoId}`);
|
|
1144
1643
|
const t0 = performance.now();
|
|
1145
1644
|
const ctx = await loadContext(m.path, m.repoId, { memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes });
|
|
1645
|
+
await mountStartupAdapter(ctx, rt.serverOptions);
|
|
1146
1646
|
createServer(ctx, port, { ...rt.serverOptions, owner: "pi-session" });
|
|
1147
1647
|
startedServer = true;
|
|
1148
1648
|
sLoad.done(`serving ${style.bold(m.repoId)} ${style.dim(`at ${baseUrl} · ready in ${(performance.now() - t0).toFixed(0)} ms`)}`);
|
|
@@ -1167,11 +1667,17 @@ switch (cmd) {
|
|
|
1167
1667
|
if (parsed.mode === "interactive")
|
|
1168
1668
|
console.log(`launching mlx-bun pi (model ${models[0]!.id}) — /help for commands, double-Ctrl+C to exit`);
|
|
1169
1669
|
restoreStdout();
|
|
1670
|
+
if (parsed.ignored.length) {
|
|
1671
|
+
const { style } = await import("./tui");
|
|
1672
|
+
console.log(style.dim(` ignoring pi flags: ${parsed.ignored.join(", ")}`));
|
|
1673
|
+
}
|
|
1170
1674
|
const code = await runEmbeddedPi({
|
|
1171
1675
|
baseUrl,
|
|
1172
1676
|
modelLabel: models[0]!.id,
|
|
1173
1677
|
contextWindow: models[0]!.contextWindow,
|
|
1174
1678
|
reasoning: models[0]!.reasoning,
|
|
1679
|
+
vision: models[0]!.vision,
|
|
1680
|
+
verbose: parsed.verbose,
|
|
1175
1681
|
mode: parsed.mode,
|
|
1176
1682
|
printFormat: parsed.printFormat,
|
|
1177
1683
|
initialMessage: message,
|
|
@@ -1238,6 +1744,15 @@ switch (cmd) {
|
|
|
1238
1744
|
process.exit(1);
|
|
1239
1745
|
}
|
|
1240
1746
|
const isOrpo = method === "orpo";
|
|
1747
|
+
// ORPO chosen-NLL scope (trainer sftScope; job.ts sft_scope). "full" =
|
|
1748
|
+
// paper/TRL-faithful prompt+response CE (the default); "response" =
|
|
1749
|
+
// response-only, bit-exact to pre-2026-07 runs. Validated here so junk
|
|
1750
|
+
// fails fast, before any model resolution or weight loading.
|
|
1751
|
+
const sftScope = opt("sft-scope");
|
|
1752
|
+
if (sftScope !== null && sftScope !== "full" && sftScope !== "response") {
|
|
1753
|
+
console.error(`--sft-scope must be full | response (got "${sftScope}")`);
|
|
1754
|
+
process.exit(1);
|
|
1755
|
+
}
|
|
1241
1756
|
// Validated numeric flag: default if absent, hard-exit on a non-number.
|
|
1242
1757
|
const numFlag = (name: string, dflt: number): number => {
|
|
1243
1758
|
const v = opt(name);
|
|
@@ -1297,6 +1812,7 @@ switch (cmd) {
|
|
|
1297
1812
|
grad_clip_norm: numFlag("grad-clip", 1.0),
|
|
1298
1813
|
val_max_examples: numFlag("val-size", 256),
|
|
1299
1814
|
warm_start_adapter: resume,
|
|
1815
|
+
...(sftScope ? { sft_scope: sftScope } : {}),
|
|
1300
1816
|
...(isOrpo ? {
|
|
1301
1817
|
orpo_lambda: numFlag("lambda", 0.1),
|
|
1302
1818
|
orpo_lr_schedule: "cosine",
|
|
@@ -1387,6 +1903,368 @@ switch (cmd) {
|
|
|
1387
1903
|
break;
|
|
1388
1904
|
}
|
|
1389
1905
|
|
|
1906
|
+
case "fuse": {
|
|
1907
|
+
// mlx_lm.fuse counterpart over our native machinery (src/train/fuse.ts):
|
|
1908
|
+
// fold a LoRA adapter into the base weights → standalone snapshot.
|
|
1909
|
+
const unsupported = ["--de-quantize", "--dequantize", "--export-gguf", "--gguf-path", "--upload-repo"]
|
|
1910
|
+
.filter((f) => argv.includes(f));
|
|
1911
|
+
if (unsupported.length > 0) {
|
|
1912
|
+
console.error(`${unsupported.join(", ")}: not supported (see: mlx-bun help fuse)`);
|
|
1913
|
+
process.exit(1);
|
|
1914
|
+
}
|
|
1915
|
+
const modelArg = positional(0) ?? opt("model");
|
|
1916
|
+
if (!modelArg) {
|
|
1917
|
+
console.error("usage: mlx-bun fuse <model-query-or-path> --adapter <dir> [--save-path <dir>]");
|
|
1918
|
+
process.exit(1);
|
|
1919
|
+
}
|
|
1920
|
+
const adapterDir = opt("adapter") ?? opt("adapter-path") ?? "adapters";
|
|
1921
|
+
const savePath = opt("save-path") ?? "fused_model";
|
|
1922
|
+
const { existsSync } = await import("node:fs");
|
|
1923
|
+
if (!existsSync(adapterDir)) {
|
|
1924
|
+
console.error(`adapter dir not found: ${adapterDir}`);
|
|
1925
|
+
process.exit(1);
|
|
1926
|
+
}
|
|
1927
|
+
let modelDir = modelArg;
|
|
1928
|
+
if (!existsSync(`${modelArg}/config.json`)) {
|
|
1929
|
+
const reg = new Registry();
|
|
1930
|
+
if (reg.list().length === 0) await reg.scan();
|
|
1931
|
+
modelDir = reg.resolve(modelArg).path;
|
|
1932
|
+
}
|
|
1933
|
+
const { banner, step, box, style } = await import("./tui");
|
|
1934
|
+
banner(pkg.version);
|
|
1935
|
+
const sNative = step("native runtime");
|
|
1936
|
+
await ensureNative(sNative);
|
|
1937
|
+
sNative.done("native runtime ready");
|
|
1938
|
+
const s = step(`fusing ${adapterDir} into ${modelDir}`);
|
|
1939
|
+
const { fuseAdapter } = await import("./train");
|
|
1940
|
+
try {
|
|
1941
|
+
const stats = await fuseAdapter(modelDir, adapterDir, savePath, (e) => s.update(e.message));
|
|
1942
|
+
s.done(`fused ${stats.fusedModules} module(s) ${style.dim(`· ${stats.totalTensors} tensors written`)}`);
|
|
1943
|
+
console.log();
|
|
1944
|
+
box([
|
|
1945
|
+
`${style.green("●")} ${style.bold("fuse complete")}`,
|
|
1946
|
+
"",
|
|
1947
|
+
`base ${style.dim(modelDir)}`,
|
|
1948
|
+
`adapter ${style.dim(adapterDir)}`,
|
|
1949
|
+
`model ${style.bold(stats.outDir)}`,
|
|
1950
|
+
...(stats.skippedAdapterTensors > 0
|
|
1951
|
+
? [`skipped ${style.dim(`${stats.skippedAdapterTensors} adapter tensor(s) with no matching base weight`)}`] : []),
|
|
1952
|
+
"",
|
|
1953
|
+
`serve it ${style.accent(`mlx-bun serve ${stats.outDir}`)}`,
|
|
1954
|
+
]);
|
|
1955
|
+
} catch (e) {
|
|
1956
|
+
s.fail(`fuse failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
1957
|
+
process.exit(1);
|
|
1958
|
+
}
|
|
1959
|
+
break;
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
case "convert": {
|
|
1963
|
+
// mlx_lm.convert counterpart over our native quantize pipeline (the same
|
|
1964
|
+
// engine as the web /api/quantize): uniform affine 4/8-bit, or the OptiQ
|
|
1965
|
+
// mixed-precision path via --target-bpw.
|
|
1966
|
+
// --upload-repo (mlx_lm.convert parity): push the converted model to the
|
|
1967
|
+
// Hub afterwards, via the same engine as `mlx-bun upload`. The token is
|
|
1968
|
+
// resolved BEFORE any conversion work — failing after minutes of
|
|
1969
|
+
// quantizing because auth is missing would be cruel.
|
|
1970
|
+
const uploadRepo = ((): string | null => {
|
|
1971
|
+
if (!argv.includes("--upload-repo")) return null;
|
|
1972
|
+
const v = opt("upload-repo");
|
|
1973
|
+
if (!v || v.startsWith("-")) {
|
|
1974
|
+
console.error("--upload-repo expects a repo id (org/name)");
|
|
1975
|
+
process.exit(1);
|
|
1976
|
+
}
|
|
1977
|
+
return v;
|
|
1978
|
+
})();
|
|
1979
|
+
let uploadToken: string | null = null;
|
|
1980
|
+
if (uploadRepo) {
|
|
1981
|
+
const { getHfToken } = await import("./hf-push");
|
|
1982
|
+
uploadToken = getHfToken();
|
|
1983
|
+
if (!uploadToken) {
|
|
1984
|
+
console.error("--upload-repo needs a Hugging Face WRITE token and none was found —");
|
|
1985
|
+
console.error("run `hf auth login`, export HF_TOKEN, or save one in the web UI (Settings → Hugging Face).");
|
|
1986
|
+
process.exit(1);
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
const unsupported = ["--dtype", "-d", "--dequantize", "--quant-predicate"]
|
|
1990
|
+
.filter((f) => argv.includes(f));
|
|
1991
|
+
if (unsupported.length > 0) {
|
|
1992
|
+
console.error(`${unsupported.join(", ")}: not supported (mixed precision: --target-bpw; see: mlx-bun help convert)`);
|
|
1993
|
+
process.exit(1);
|
|
1994
|
+
}
|
|
1995
|
+
const qMode = opt("q-mode", "affine")!;
|
|
1996
|
+
if (qMode !== "affine") {
|
|
1997
|
+
console.error(`--q-mode ${qMode}: only "affine" is supported`);
|
|
1998
|
+
process.exit(1);
|
|
1999
|
+
}
|
|
2000
|
+
const hfPath = opt("hf-path") ?? opt("model") ?? positional(0);
|
|
2001
|
+
if (!hfPath) {
|
|
2002
|
+
console.error("usage: mlx-bun convert --hf-path <repo-or-path> -q [--q-bits N] [--q-group-size N] [--mlx-path <dir>] [--target-bpw F]");
|
|
2003
|
+
process.exit(1);
|
|
2004
|
+
}
|
|
2005
|
+
const targetBpwRaw = opt("target-bpw");
|
|
2006
|
+
if (!argv.includes("-q") && !argv.includes("--quantize") && targetBpwRaw === null) {
|
|
2007
|
+
console.error("plain (non-quantizing) conversion is not supported yet — pass -q or --target-bpw");
|
|
2008
|
+
process.exit(1);
|
|
2009
|
+
}
|
|
2010
|
+
const targetBpw = targetBpwRaw !== null ? Number(targetBpwRaw) : undefined;
|
|
2011
|
+
if (targetBpwRaw !== null && (!Number.isFinite(targetBpw!) || targetBpw! <= 0)) {
|
|
2012
|
+
console.error(`--target-bpw expects a positive number (got "${targetBpwRaw}")`);
|
|
2013
|
+
process.exit(1);
|
|
2014
|
+
}
|
|
2015
|
+
const qBits = Number(opt("q-bits", "4"));
|
|
2016
|
+
if (qBits !== 4 && qBits !== 8) {
|
|
2017
|
+
console.error(`--q-bits must be 4 or 8 (got "${opt("q-bits")}")`);
|
|
2018
|
+
process.exit(1);
|
|
2019
|
+
}
|
|
2020
|
+
const qGroup = Number(opt("q-group-size", "64"));
|
|
2021
|
+
if (qGroup !== 32 && qGroup !== 64) {
|
|
2022
|
+
console.error(`--q-group-size must be 32 or 64 (got "${opt("q-group-size")}")`);
|
|
2023
|
+
process.exit(1);
|
|
2024
|
+
}
|
|
2025
|
+
const candidateBits = opt("candidate-bits")?.split(",").map((s) => Number(s.trim()));
|
|
2026
|
+
if (candidateBits && candidateBits.some((b) => !Number.isInteger(b) || b < 2 || b > 8)) {
|
|
2027
|
+
console.error(`--candidate-bits expects a comma list of integers in [2, 8] (got "${opt("candidate-bits")}")`);
|
|
2028
|
+
process.exit(1);
|
|
2029
|
+
}
|
|
2030
|
+
const mlxPath = opt("mlx-path", "mlx_model")!;
|
|
2031
|
+
const { existsSync } = await import("node:fs");
|
|
2032
|
+
if (existsSync(mlxPath)) {
|
|
2033
|
+
console.error(`Cannot save to the path ${mlxPath} as it already exists — delete it or pass a fresh --mlx-path.`);
|
|
2034
|
+
process.exit(1);
|
|
2035
|
+
}
|
|
2036
|
+
const { banner, step, box, style } = await import("./tui");
|
|
2037
|
+
banner(pkg.version);
|
|
2038
|
+
const sNative = step("native runtime");
|
|
2039
|
+
await ensureNative(sNative);
|
|
2040
|
+
sNative.done("native runtime ready");
|
|
2041
|
+
// Resolve the source: local dir → itself; downloaded model → registry;
|
|
2042
|
+
// otherwise an HF repo id, downloaded first (the point of the verb).
|
|
2043
|
+
let srcDir = hfPath;
|
|
2044
|
+
if (!existsSync(`${hfPath}/config.json`)) {
|
|
2045
|
+
const reg = new Registry();
|
|
2046
|
+
if (reg.list().length === 0) await reg.scan();
|
|
2047
|
+
try {
|
|
2048
|
+
srcDir = reg.resolve(hfPath).path;
|
|
2049
|
+
} catch (e) {
|
|
2050
|
+
if (!/^[\w.-]+\/[\w.-]+$/.test(hfPath)) {
|
|
2051
|
+
console.error(e instanceof Error ? e.message : String(e));
|
|
2052
|
+
process.exit(1);
|
|
2053
|
+
}
|
|
2054
|
+
const { downloadModel } = await import("./download");
|
|
2055
|
+
const sDl = step(`downloading ${hfPath}`);
|
|
2056
|
+
srcDir = await downloadModel(hfPath, {
|
|
2057
|
+
onProgress: (file, received, total) => {
|
|
2058
|
+
const pct = total ? Math.floor((received / total) * 100) : 0;
|
|
2059
|
+
sDl.update(`${style.bold(hfPath)} ${style.dim(`· ${file} · ${gb(received)} / ${gb(total)} (${pct}%)`)}`);
|
|
2060
|
+
},
|
|
2061
|
+
});
|
|
2062
|
+
sDl.done(`${style.bold(hfPath)} ${style.dim("downloaded · verified")}`);
|
|
2063
|
+
await reg.scan();
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
const sQ = step(
|
|
2067
|
+
targetBpw !== undefined
|
|
2068
|
+
? `quantizing (mixed, target ${targetBpw} bpw — sensitivity sweep, ~minutes)`
|
|
2069
|
+
: `quantizing (${qBits}-bit, group ${qGroup})`,
|
|
2070
|
+
);
|
|
2071
|
+
const { quantizeModelDir } = await import("./quantize");
|
|
2072
|
+
try {
|
|
2073
|
+
const r = await quantizeModelDir(
|
|
2074
|
+
srcDir,
|
|
2075
|
+
mlxPath,
|
|
2076
|
+
{
|
|
2077
|
+
bits: qBits as 4 | 8,
|
|
2078
|
+
groupSize: qGroup as 32 | 64,
|
|
2079
|
+
mode: "affine",
|
|
2080
|
+
...(targetBpw !== undefined ? { targetBpw } : {}),
|
|
2081
|
+
...(candidateBits ? { candidateBits } : {}),
|
|
2082
|
+
...(opt("calibration-mix") ? { calibrationMix: opt("calibration-mix")! } : {}),
|
|
2083
|
+
...(opt("n-calibration") ? { nCalibration: Number(opt("n-calibration")) } : {}),
|
|
2084
|
+
},
|
|
2085
|
+
(e) => sQ.update(e.message),
|
|
2086
|
+
);
|
|
2087
|
+
sQ.done(`quantized ${r.nQuantized} module(s) ${style.dim(`· ${r.achievedBpw.toFixed(2)} bpw achieved`)}`);
|
|
2088
|
+
console.log();
|
|
2089
|
+
box([
|
|
2090
|
+
`${style.green("●")} ${style.bold("convert complete")}`,
|
|
2091
|
+
"",
|
|
2092
|
+
`source ${style.dim(srcDir)}`,
|
|
2093
|
+
`model ${style.bold(r.outDir)} ${style.dim(`· ${gb(r.write.totalSize)}`)}`,
|
|
2094
|
+
`quant ${style.dim(targetBpw !== undefined ? `mixed ${r.achievedBpw.toFixed(2)} bpw (target ${targetBpw})` : `${qBits}-bit g${qGroup} affine`)}`,
|
|
2095
|
+
"",
|
|
2096
|
+
`serve it ${style.accent(`mlx-bun serve ${r.outDir}`)}`,
|
|
2097
|
+
]);
|
|
2098
|
+
if (uploadRepo) {
|
|
2099
|
+
const sUp = step(`uploading ${r.outDir} → ${uploadRepo}`);
|
|
2100
|
+
try {
|
|
2101
|
+
const { uploadFolder } = await import("./hf-push");
|
|
2102
|
+
const up = await uploadFolder(r.outDir, uploadRepo, {
|
|
2103
|
+
repoType: "model",
|
|
2104
|
+
token: uploadToken,
|
|
2105
|
+
commitMessage: "Upload with mlx-bun convert",
|
|
2106
|
+
onProgress: (file, sent, total) =>
|
|
2107
|
+
sUp.update(`${style.bold(uploadRepo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
|
|
2108
|
+
});
|
|
2109
|
+
sUp.done(`uploaded ${style.bold(up.url)}`);
|
|
2110
|
+
} catch (e) {
|
|
2111
|
+
sUp.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2112
|
+
console.error(`the converted model is intact at ${r.outDir} — retry with: mlx-bun upload --path ${r.outDir} --upload-repo ${uploadRepo}`);
|
|
2113
|
+
process.exit(1);
|
|
2114
|
+
}
|
|
2115
|
+
}
|
|
2116
|
+
} catch (e) {
|
|
2117
|
+
sQ.fail(`convert failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2118
|
+
process.exit(1);
|
|
2119
|
+
}
|
|
2120
|
+
break;
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
case "upload": {
|
|
2124
|
+
// mlx_lm.upload counterpart (same flag names: --path, --upload-repo) over
|
|
2125
|
+
// our native push-to-hub (src/hf-push.ts — the exact engine the web
|
|
2126
|
+
// /api/{quantize,finetune,dataset}/push routes call). Model-free until
|
|
2127
|
+
// the network: everything before uploadFolder is arg + token validation.
|
|
2128
|
+
const dir = opt("path", "mlx_model")!; // mlx_lm.upload's --path default
|
|
2129
|
+
const repo = ((): string | null => {
|
|
2130
|
+
const v = opt("upload-repo");
|
|
2131
|
+
return v && !v.startsWith("-") ? v : null;
|
|
2132
|
+
})();
|
|
2133
|
+
if (!repo) {
|
|
2134
|
+
console.error("usage: mlx-bun upload --path <model-dir> --upload-repo <org/repo> [--private]");
|
|
2135
|
+
process.exit(1);
|
|
2136
|
+
}
|
|
2137
|
+
const { existsSync, statSync } = await import("node:fs");
|
|
2138
|
+
if (!existsSync(dir) || !statSync(dir).isDirectory()) {
|
|
2139
|
+
console.error(`not a directory: ${dir} (pass the model directory via --path)`);
|
|
2140
|
+
process.exit(1);
|
|
2141
|
+
}
|
|
2142
|
+
const { getHfToken, uploadFolder } = await import("./hf-push");
|
|
2143
|
+
const token = getHfToken();
|
|
2144
|
+
if (!token) {
|
|
2145
|
+
console.error("no Hugging Face token found — uploading needs a WRITE token from one of:");
|
|
2146
|
+
console.error(" hf auth login (~/.cache/huggingface/token)");
|
|
2147
|
+
console.error(" export HF_TOKEN=hf_…");
|
|
2148
|
+
console.error(" web UI Settings → Hugging Face (~/.mlx-bun/hf.json)");
|
|
2149
|
+
process.exit(1);
|
|
2150
|
+
}
|
|
2151
|
+
const { banner, step, box, style } = await import("./tui");
|
|
2152
|
+
banner(pkg.version);
|
|
2153
|
+
const s = step(`uploading ${dir} → ${repo}`);
|
|
2154
|
+
try {
|
|
2155
|
+
const r = await uploadFolder(dir, repo, {
|
|
2156
|
+
repoType: "model",
|
|
2157
|
+
private: flag("private"),
|
|
2158
|
+
token,
|
|
2159
|
+
commitMessage: "Upload with mlx-bun",
|
|
2160
|
+
onProgress: (file, sent, total) =>
|
|
2161
|
+
s.update(`${style.bold(repo)} ${style.dim(`· ${file} · ${gb(sent)} / ${gb(total)}`)}`),
|
|
2162
|
+
});
|
|
2163
|
+
s.done(`uploaded ${style.bold(repo)}`);
|
|
2164
|
+
console.log();
|
|
2165
|
+
box([
|
|
2166
|
+
`${style.green("●")} ${style.bold("upload complete")}`,
|
|
2167
|
+
"",
|
|
2168
|
+
`source ${style.dim(dir)}`,
|
|
2169
|
+
`repo ${style.url(r.url)}${flag("private") ? style.dim(" · private") : ""}`,
|
|
2170
|
+
"",
|
|
2171
|
+
`get it back anywhere: ${style.accent(`mlx-bun get ${repo}`)}`,
|
|
2172
|
+
]);
|
|
2173
|
+
} catch (e) {
|
|
2174
|
+
s.fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2175
|
+
process.exit(1);
|
|
2176
|
+
}
|
|
2177
|
+
break;
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
case "perplexity": {
|
|
2181
|
+
// mlx_lm.perplexity counterpart (methodology reproduced exactly — see
|
|
2182
|
+
// src/eval/perplexity.ts) over a LOCAL dataset file; never downloads data.
|
|
2183
|
+
const modelArg = positional(0) ?? opt("model") ?? opt("query");
|
|
2184
|
+
const dataPath = opt("data-path");
|
|
2185
|
+
if (!modelArg || !dataPath) {
|
|
2186
|
+
console.error("usage: mlx-bun perplexity <model-query-or-path> --data-path <file.txt|file.jsonl> [--sequence-length 512] [--num-samples 256] [--batch-size 8] [--seed 123]");
|
|
2187
|
+
process.exit(1);
|
|
2188
|
+
}
|
|
2189
|
+
const { existsSync } = await import("node:fs");
|
|
2190
|
+
if (!existsSync(dataPath)) {
|
|
2191
|
+
console.error(`data file not found: ${dataPath} (a local .txt or .jsonl — HF datasets are not downloaded)`);
|
|
2192
|
+
process.exit(1);
|
|
2193
|
+
}
|
|
2194
|
+
const intFlag = (name: string, dflt: number, lo: number): number => {
|
|
2195
|
+
const v = Number(opt(name, String(dflt)));
|
|
2196
|
+
if (!Number.isInteger(v) || (v < lo && v !== -1)) {
|
|
2197
|
+
console.error(`--${name} expects an integer >= ${lo} (got "${opt(name)}")`);
|
|
2198
|
+
process.exit(1);
|
|
2199
|
+
}
|
|
2200
|
+
return v;
|
|
2201
|
+
};
|
|
2202
|
+
const seqLen = intFlag("sequence-length", 512, 2);
|
|
2203
|
+
const numSamples = intFlag("num-samples", 256, 1); // -1 = all
|
|
2204
|
+
const batchSize = intFlag("batch-size", 8, 1);
|
|
2205
|
+
const seed = intFlag("seed", 123, 0);
|
|
2206
|
+
const { banner, step, box, style } = await import("./tui");
|
|
2207
|
+
banner(pkg.version);
|
|
2208
|
+
const sNative = step("native runtime");
|
|
2209
|
+
await ensureNative(sNative);
|
|
2210
|
+
sNative.done("native runtime ready");
|
|
2211
|
+
let modelDir = modelArg;
|
|
2212
|
+
if (!existsSync(`${modelArg}/config.json`)) {
|
|
2213
|
+
const reg = new Registry();
|
|
2214
|
+
if (reg.list().length === 0) await reg.scan();
|
|
2215
|
+
modelDir = reg.resolve(modelArg).path;
|
|
2216
|
+
}
|
|
2217
|
+
// Gemma/e4b: same forward env flags the trainer sets (full-sequence pass).
|
|
2218
|
+
if ((await Bun.file(`${modelDir}/config.json`).text()).toLowerCase().includes("gemma")) {
|
|
2219
|
+
process.env.MLX_BUN_PERF_KERNEL ??= "0";
|
|
2220
|
+
process.env.MLX_BUN_FUSED_GELU ??= "0";
|
|
2221
|
+
}
|
|
2222
|
+
const sLoad = step(`loading ${modelDir}`);
|
|
2223
|
+
const { Weights } = await import("./weights");
|
|
2224
|
+
const { createModel } = await import("./model/factory");
|
|
2225
|
+
const { loadTokenizer } = await import("./tokenizer");
|
|
2226
|
+
const { peakMemory, resetPeakMemory } = await import("./mlx/ffi");
|
|
2227
|
+
const config = await loadModelConfig(modelDir);
|
|
2228
|
+
const model = createModel(await Weights.open(modelDir), config);
|
|
2229
|
+
const tok = await loadTokenizer(modelDir);
|
|
2230
|
+
sLoad.done(`model loaded ${style.dim(`· ${modelDir}`)}`);
|
|
2231
|
+
|
|
2232
|
+
const { parseSamples, packRows, evalPpl } = await import("./eval/perplexity");
|
|
2233
|
+
const sData = step(`tokenizing ${dataPath}`);
|
|
2234
|
+
const samples = parseSamples(await Bun.file(dataPath).text(), dataPath);
|
|
2235
|
+
const rows = packRows(
|
|
2236
|
+
samples.map((t) => tok.encode(t)),
|
|
2237
|
+
{ sequenceLength: seqLen, numSamples, seed },
|
|
2238
|
+
);
|
|
2239
|
+
if (rows.length === 0) {
|
|
2240
|
+
sData.fail(`dataset too small: fewer than ${seqLen} tokens (need at least one full row)`);
|
|
2241
|
+
process.exit(1);
|
|
2242
|
+
}
|
|
2243
|
+
sData.done(`${rows.length} row(s) × ${seqLen} tokens ${style.dim(`· ${samples.length} sample(s), seed ${seed}`)}`);
|
|
2244
|
+
|
|
2245
|
+
const sEval = step(`evaluating (batch ${batchSize})`);
|
|
2246
|
+
resetPeakMemory();
|
|
2247
|
+
const t0 = performance.now();
|
|
2248
|
+
const r = evalPpl(model, rows, batchSize, (done, total) =>
|
|
2249
|
+
sEval.update(`batch ${done}/${total}`));
|
|
2250
|
+
const seconds = (performance.now() - t0) / 1000;
|
|
2251
|
+
sEval.done(`evaluated ${r.tokens.toLocaleString()} tokens ${style.dim(`in ${seconds.toFixed(1)} s`)}`);
|
|
2252
|
+
console.log();
|
|
2253
|
+
box([
|
|
2254
|
+
`${style.green("●")} ${style.bold("perplexity")} ${style.dim(`· ${modelDir.split("/snapshots/")[0]?.split("/").at(-1) ?? modelDir}`)}`,
|
|
2255
|
+
"",
|
|
2256
|
+
`ppl ${style.green(style.bold(r.ppl.toFixed(3)))} ${style.dim(`± ${r.standardError.toFixed(3)}`)}`,
|
|
2257
|
+
`mean CE ${style.bold(r.meanLoss.toFixed(4))} ${style.dim("nats/token")}`,
|
|
2258
|
+
`tokens ${style.bold(r.tokens.toLocaleString())} ${style.dim(`· ${r.rows} row(s) × ${seqLen}`)}`,
|
|
2259
|
+
`speed ${style.dim(`${(r.tokens / seconds).toFixed(0)} tok/s · peak ${gb(peakMemory())}`)}`,
|
|
2260
|
+
]);
|
|
2261
|
+
break;
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2264
|
+
// `setup` is a true alias for `memory` — same handler, same subcommands
|
|
2265
|
+
// (`mlx-bun setup init` == `mlx-bun memory init`; the handler reads the
|
|
2266
|
+
// subcommand from argv[1] onward, identical for both spellings).
|
|
2267
|
+
case "setup":
|
|
1390
2268
|
case "memory": {
|
|
1391
2269
|
const { style, box, step, banner } = await import("./tui");
|
|
1392
2270
|
const {
|
|
@@ -1475,13 +2353,14 @@ switch (cmd) {
|
|
|
1475
2353
|
}
|
|
1476
2354
|
|
|
1477
2355
|
// Offer to install the nightly synthesis job (a launchd agent). The job
|
|
1478
|
-
//
|
|
1479
|
-
//
|
|
1480
|
-
//
|
|
2356
|
+
// runs the real synthesis DAG (`mlx-bun memory synthesize`) — create new
|
|
2357
|
+
// entity articles, then the editorial wikify sweep — so installing the
|
|
2358
|
+
// schedule keeps your wiki current automatically. Persistent action →
|
|
2359
|
+
// TTY-only, defaults to no.
|
|
1481
2360
|
let scheduled: string | null = null;
|
|
1482
2361
|
console.log(style.dim(
|
|
1483
|
-
"\n Synthesis (turning your conversations into articles)
|
|
1484
|
-
"
|
|
2362
|
+
"\n Synthesis (turning your conversations into articles) runs as a nightly job;\n" +
|
|
2363
|
+
" install it now so your wiki stays current automatically.",
|
|
1485
2364
|
));
|
|
1486
2365
|
if (await confirmYN(" Install the nightly synthesis job (runs in the background)?", false)) {
|
|
1487
2366
|
const atRaw = await ask(` At what time? 24h HH:MM ${style.dim("[03:00]")}: `, "03:00");
|
|
@@ -1507,8 +2386,8 @@ switch (cmd) {
|
|
|
1507
2386
|
? `nightly ${style.green("scheduled")} ${style.dim(`· ${scheduled} daily · mlx-bun memory unschedule to undo`)}`
|
|
1508
2387
|
: `nightly ${style.dim("not scheduled · mlx-bun memory schedule to set it up")}`,
|
|
1509
2388
|
"",
|
|
1510
|
-
style.dim("Synthesis (conversations → articles)
|
|
1511
|
-
style.dim("
|
|
2389
|
+
style.dim("Synthesis (conversations → articles) runs the full DAG: create new entity"),
|
|
2390
|
+
style.dim("articles, then an editorial wikify sweep. Run it now: mlx-bun memory synthesize."),
|
|
1512
2391
|
]);
|
|
1513
2392
|
break;
|
|
1514
2393
|
}
|
|
@@ -1528,8 +2407,7 @@ switch (cmd) {
|
|
|
1528
2407
|
`articles ${style.bold(String(st.articleCount))}`,
|
|
1529
2408
|
`reference ${style.bold(String(st.referenceCount))} ${style.dim("read-only docs")}`,
|
|
1530
2409
|
`git ${st.isGitRepo ? style.green("tracked") : style.dim("not a git repo")}`,
|
|
1531
|
-
`synthesis ${style.
|
|
1532
|
-
`last run ${style.dim("not available yet")}`,
|
|
2410
|
+
`synthesis ${style.green("available")} ${style.dim("· mlx-bun memory synthesize")}`,
|
|
1533
2411
|
`nightly ${
|
|
1534
2412
|
sched.installed
|
|
1535
2413
|
? `${sched.loaded ? style.green("scheduled") : style.accent("installed (not loaded)")} ${style.dim(`· ${sched.plistPath}`)}`
|
|
@@ -1570,7 +2448,70 @@ switch (cmd) {
|
|
|
1570
2448
|
break;
|
|
1571
2449
|
}
|
|
1572
2450
|
|
|
1573
|
-
|
|
2451
|
+
// Independent, chronological, resumable STAGE WORKERS. Each pulls its own
|
|
2452
|
+
// eligible work from the DB by state, processes a bounded batch, persists,
|
|
2453
|
+
// and exits — so a user can run the four as four separate processes on
|
|
2454
|
+
// different slices concurrently (GPU/memory allowing). `synthesize` (below)
|
|
2455
|
+
// remains the FULL DAG; these are its decomposed pieces.
|
|
2456
|
+
if (sub === "segment" || sub === "extract" || sub === "route" || sub === "synthesize-stage" || sub === "stage-synthesize") {
|
|
2457
|
+
const stages = await import("./memory/stages");
|
|
2458
|
+
const { MemoryStore } = await import("./memory/db");
|
|
2459
|
+
const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
|
|
2460
|
+
const convsRaw = opt("convs");
|
|
2461
|
+
const convIds = convsRaw ? convsRaw.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
|
|
2462
|
+
const onEvent = (e: import("./memory/pipeline").SynthesisEvent) => {
|
|
2463
|
+
if (e.type === "stage") console.log(` ${style.dim("·")} ${e.message}`);
|
|
2464
|
+
else console.log(` ${e.message}`);
|
|
2465
|
+
};
|
|
2466
|
+
const store = new MemoryStore();
|
|
2467
|
+
console.log();
|
|
2468
|
+
try {
|
|
2469
|
+
if (sub === "segment") {
|
|
2470
|
+
const r = await stages.runSegmentStage(store, { convIds, limit, onEvent });
|
|
2471
|
+
console.log(style.dim(`\n segment: ${r.valid} segmented, ${r.chunks} chunks, ${r.skipped} skipped, ${r.errored} errored`));
|
|
2472
|
+
} else if (sub === "extract") {
|
|
2473
|
+
const r = await stages.runExtractStage(store, { convIds, limit, onEvent });
|
|
2474
|
+
console.log(style.dim(`\n extract: ${r.extracted} chunk(s) extracted, ${r.remaining} still pending`));
|
|
2475
|
+
} else if (sub === "route") {
|
|
2476
|
+
const r = await stages.runRouteStage(store, { convIds, onEvent });
|
|
2477
|
+
console.log(style.dim(`\n route: ${r.decisions.length} entities — ${r.createEligible} create-eligible, ${r.captured.length} captured`));
|
|
2478
|
+
} else {
|
|
2479
|
+
const r = await stages.runSynthesizeStage(store, { root, convIds, limit, onEvent });
|
|
2480
|
+
console.log(style.dim(`\n synthesize: ${r.created.length} created, ${r.patched.length} patched, ${r.skippedByGate.length} gated`));
|
|
2481
|
+
}
|
|
2482
|
+
} finally {
|
|
2483
|
+
store.close();
|
|
2484
|
+
}
|
|
2485
|
+
break;
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
// CROSS-LINK — the dedicated edge-building stage. Deterministic + idempotent:
|
|
2489
|
+
// inline-link first mentions of other articles + rebuild each ## See also from
|
|
2490
|
+
// mentions + co-occurrence. No model required.
|
|
2491
|
+
if (sub === "link") {
|
|
2492
|
+
const { runLinkStage } = await import("./memory/crosslink");
|
|
2493
|
+
const { MemoryStore } = await import("./memory/db");
|
|
2494
|
+
const limit = opt("limit") ? parseInt(opt("limit")!, 10) : undefined;
|
|
2495
|
+
const store = new MemoryStore();
|
|
2496
|
+
console.log();
|
|
2497
|
+
try {
|
|
2498
|
+
const r = await runLinkStage(store, {
|
|
2499
|
+
root,
|
|
2500
|
+
limit,
|
|
2501
|
+
onEvent: (e) => console.log(e.type === "stage" ? ` ${style.dim("·")} ${e.message}` : ` ${e.message}`),
|
|
2502
|
+
});
|
|
2503
|
+
console.log(
|
|
2504
|
+
style.dim(
|
|
2505
|
+
`\n link: ${r.linked.length} article(s) linked · ${r.mentionEdges} mention edge(s) · ${r.skippedByGate.length} gated`,
|
|
2506
|
+
),
|
|
2507
|
+
);
|
|
2508
|
+
} finally {
|
|
2509
|
+
store.close();
|
|
2510
|
+
}
|
|
2511
|
+
break;
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
if (sub === "synthesize" || sub === "pipeline" || sub === "all") {
|
|
1574
2515
|
const { runSynthesis } = await import("./memory/pipeline");
|
|
1575
2516
|
const dryRun = flag("dry-run");
|
|
1576
2517
|
console.log();
|
|
@@ -1595,7 +2536,7 @@ switch (cmd) {
|
|
|
1595
2536
|
: `${style.accent("●")} ${style.bold("plist written, but launchctl load failed")}`,
|
|
1596
2537
|
"",
|
|
1597
2538
|
`plist ${style.dim(r.plistPath)}`,
|
|
1598
|
-
`runs ${style.dim("mlx-bun memory synthesize (
|
|
2539
|
+
`runs ${style.dim("mlx-bun memory synthesize (full DAG: create + wikify sweep)")}`,
|
|
1599
2540
|
`undo ${style.accent("mlx-bun memory unschedule")}`,
|
|
1600
2541
|
]);
|
|
1601
2542
|
break;
|
|
@@ -1698,9 +2639,11 @@ switch (cmd) {
|
|
|
1698
2639
|
}
|
|
1699
2640
|
|
|
1700
2641
|
|
|
2642
|
+
// Unknown subcommand: help on stdout, but a FAILURE exit — scripts and
|
|
2643
|
+
// launchd jobs must be able to detect typos (printHelp would exit 0).
|
|
1701
2644
|
console.error(`unknown: mlx-bun memory ${sub}`);
|
|
1702
|
-
|
|
1703
|
-
|
|
2645
|
+
console.log(renderHelp(HELP.memory!));
|
|
2646
|
+
process.exit(1);
|
|
1704
2647
|
}
|
|
1705
2648
|
|
|
1706
2649
|
default:
|