mlx-bun 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -32
- package/docs/README.md +58 -42
- package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +3617 -0
- package/docs/archive/investigations/openjarvis-fit.md +198 -0
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +1092 -0
- package/docs/archive/planning/release-notes-v0.4.0.md +45 -0
- package/docs/archive/planning/release-notes-v0.5.0.md +37 -0
- package/docs/design/batching.md +1535 -50
- package/docs/design/decode-speed-program.md +394 -1171
- package/docs/design/generic-model-support.md +195 -3
- package/docs/design/kv-cache.md +538 -225
- package/docs/design/speculative-decoding.md +552 -50
- package/docs/design/turboquant.md +140 -17
- package/docs/design/unified-engine-frontier-plan.md +259 -150
- package/docs/planning/PRODUCT_ROADMAP.md +21 -6
- package/docs/planning/public-readiness-handoff.md +244 -0
- package/docs/planning/public-readiness.md +149 -0
- package/docs/reference/benchmarks.md +3537 -1112
- package/docs/reference/cli.md +102 -2
- package/docs/reference/distribution.md +45 -13
- package/docs/reference/environment.md +159 -13
- package/docs/reference/library-api.md +18 -4
- package/docs/reference/memory.md +1 -1
- package/docs/reference/models.md +28 -7
- package/docs/reference/server-api.md +257 -33
- package/docs/reference/server-config.md +372 -151
- package/docs/reference/troubleshooting.md +13 -21
- package/package.json +2 -2
- package/src/audio/audiotoolbox.ts +112 -0
- package/src/audio/conformer.ts +8 -0
- package/src/audio/mic-capture.ts +98 -0
- package/src/audio/silero-vad.ts +379 -0
- package/src/audio/whisper-decode.ts +774 -0
- package/src/audio/whisper-fast.ts +732 -0
- package/src/audio/whisper-format.ts +73 -0
- package/src/audio/whisper-mel.ts +185 -0
- package/src/audio/whisper-timing.ts +274 -0
- package/src/audio/whisper-tokenizer.ts +239 -0
- package/src/audio/whisper-transcribe.ts +410 -0
- package/src/backends/mlx/assistant-target.ts +46 -0
- package/src/backends/mlx/autoregressive-draft-rows.ts +120 -0
- package/src/backends/mlx/autoregressive.ts +22 -1
- package/src/backends/mlx/batch-group.ts +464 -236
- package/src/backends/mlx/cache-attachment.ts +67 -0
- package/src/backends/mlx/cache-layout.ts +31 -0
- package/src/backends/mlx/checkpoint-state.ts +53 -0
- package/src/backends/mlx/continuation-persistence.ts +107 -0
- package/src/backends/mlx/continuation-request.ts +43 -0
- package/src/backends/mlx/continuation.ts +77 -0
- package/src/backends/mlx/draft-target.ts +7 -22
- package/src/backends/mlx/encoder-cache.ts +25 -0
- package/src/backends/mlx/fill-append.ts +21 -8
- package/src/backends/mlx/fill-group.ts +497 -0
- package/src/backends/mlx/full-kv-row-append.ts +32 -0
- package/src/backends/mlx/full-kv-row-donor.ts +54 -0
- package/src/backends/mlx/full-transitioning-kv-rows.ts +65 -0
- package/src/backends/mlx/gateway-binding.ts +122 -19
- package/src/backends/mlx/kv-codec.ts +9 -0
- package/src/backends/mlx/kv-maintenance.ts +161 -0
- package/src/backends/mlx/kv-tensor-rows.ts +212 -0
- package/src/backends/mlx/mixed-iteration.ts +62 -0
- package/src/backends/mlx/model-prompt.ts +46 -32
- package/src/backends/mlx/model-serving.ts +5 -4
- package/src/backends/mlx/paged-cache-codec.ts +27 -0
- package/src/backends/mlx/prefill-cohort.ts +100 -0
- package/src/backends/mlx/prefill-policy.ts +26 -0
- package/src/backends/mlx/prefill-rows.ts +244 -0
- package/src/backends/mlx/prefill.ts +1 -1
- package/src/backends/mlx/projected-context-rows.ts +72 -0
- package/src/backends/mlx/prompt-input.ts +29 -0
- package/src/backends/mlx/qwen-prompt-input.ts +40 -0
- package/src/backends/mlx/request-state-policy.ts +54 -0
- package/src/backends/mlx/rollback.ts +45 -4
- package/src/backends/mlx/row-sampling.ts +49 -0
- package/src/backends/mlx/serial-executor.ts +51 -73
- package/src/backends/mlx/speculative-group.ts +438 -0
- package/src/backends/mlx/speculative-round.ts +161 -0
- package/src/backends/mlx/speculative.ts +24 -7
- package/src/backends/mlx/state-rows.ts +104 -0
- package/src/backends/mlx/state-views.ts +21 -1
- package/src/backends/mlx/target-layout-capability.ts +35 -0
- package/src/backends/mlx/transitioning-kv-rows.ts +173 -0
- package/src/cli.ts +333 -36
- package/src/contracts/execution.ts +21 -0
- package/src/contracts/object-cache.ts +8 -0
- package/src/contracts/pi-provider.ts +1 -1
- package/src/contracts/pi.ts +3 -1
- package/src/contracts/prefix-cache.ts +33 -0
- package/src/contracts/resources.ts +7 -0
- package/src/contracts/sampling.ts +23 -0
- package/src/contracts/scheduling.ts +27 -0
- package/src/engine/execution-plan.ts +38 -25
- package/src/engine/execution-tasks.ts +44 -0
- package/src/engine/namespaced-cache.ts +13 -0
- package/src/engine/resources.ts +12 -1
- package/src/engine/scheduler.ts +32 -7
- package/src/fill/fill-session.ts +14 -3
- package/src/fit.ts +29 -15
- package/src/generate.ts +105 -195
- package/src/grammar.ts +13 -0
- package/src/inference/draft-acceptance.ts +41 -0
- package/src/inference/draft-output.ts +49 -0
- package/src/inference/prefill.ts +6 -0
- package/src/inference/rollback.ts +2 -2
- package/src/kv-scheme.ts +92 -12
- package/src/kv-store.ts +183 -93
- package/src/lab/README.md +5 -1
- package/src/lab/paged-kv/paged-attention.ts +117 -0
- package/src/lab/paged-kv/paged-kv-rows.ts +111 -0
- package/src/lab/paged-kv/paged-kv.ts +126 -46
- package/src/md-modules.d.ts +5 -0
- package/src/mlx/array.ts +39 -1
- package/src/mlx/ffi.ts +19 -0
- package/src/mlx/normalized-argmax.ts +116 -0
- package/src/mlx/ops.ts +97 -10
- package/src/model/artifact-identity.ts +73 -0
- package/src/model/batched-kv.ts +235 -0
- package/src/model/batched-mask.ts +12 -4
- package/src/model/batched-quant.ts +8 -2
- package/src/model/batched-quantized-kv.ts +144 -0
- package/src/model/batched-rotating-quant.ts +76 -15
- package/src/model/batched-rotating-state.ts +76 -0
- package/src/model/batched-rotating.ts +64 -22
- package/src/model/batched-row-storage.ts +52 -0
- package/src/model/batched-ssm.ts +52 -0
- package/src/model/batched-turboquant-kv.ts +94 -0
- package/src/model/compiled-decode.ts +7 -1
- package/src/model/decoded-kv-donor.ts +12 -0
- package/src/model/delayed-quantized-kv.ts +72 -0
- package/src/model/delayed-rotating-quantized-kv.ts +93 -0
- package/src/model/delayed-turboquant-kv.ts +57 -0
- package/src/model/factory.ts +26 -0
- package/src/model/fingerprint.ts +12 -0
- package/src/model/full-prefill-padding.ts +56 -0
- package/src/model/full-prefill-row.ts +105 -0
- package/src/model/gemma4-base.ts +183 -82
- package/src/model/gemma4.ts +116 -28
- package/src/model/generated/gemma4-12b.ts +4 -1
- package/src/model/generated/gemma4-26b.ts +4 -1
- package/src/model/generated/gemma4-e4b.ts +4 -1
- package/src/model/glm52-cache.ts +80 -4
- package/src/model/glm52-mla.ts +12 -6
- package/src/model/glm52.ts +19 -40
- package/src/model/kv-attention-view.ts +70 -0
- package/src/model/minicpm5.ts +11 -7
- package/src/model/profile.ts +29 -4
- package/src/model/quantized-append-attention.ts +31 -0
- package/src/model/qwen3-delta-state.ts +74 -0
- package/src/model/qwen3-delta.ts +59 -36
- package/src/model/qwen3-mrope.ts +7 -9
- package/src/model/qwen3_5.ts +177 -75
- package/src/model/rotating-kv-donor.ts +35 -0
- package/src/model/rotating-kv-layout.ts +262 -0
- package/src/model/rotating-row-transaction.ts +35 -0
- package/src/model/speculative-rotating-kv.ts +101 -0
- package/src/model/ssm-prefill-padding.ts +39 -0
- package/src/model/support.ts +8 -0
- package/src/model/token-groups.ts +66 -0
- package/src/model/trellis-linear.ts +4 -4
- package/src/model/turboquant-codec.ts +89 -0
- package/src/model/universal/dense.ts +14 -13
- package/src/model/whisper.ts +435 -0
- package/src/native/mic_capture.swift +92 -0
- package/src/native-pack.ts +6 -5
- package/src/pi-web.ts +9 -1
- package/src/preflight.ts +31 -15
- package/src/prompt-cache.ts +306 -86
- package/src/runtime-config.ts +1 -5
- package/src/safetensors.ts +9 -2
- package/src/sampler.ts +37 -25
- package/src/serve/audio-routes.ts +323 -0
- package/src/serve/chat-request.ts +2 -0
- package/src/serve/chat-stage.ts +19 -14
- package/src/serve/completion-executor.ts +9 -1
- package/src/serve/discovery-routes.ts +17 -1
- package/src/serve/generated-token-history.ts +57 -0
- package/src/serve/generation-gateway.ts +88 -38
- package/src/serve/model-binding.ts +7 -1
- package/src/serve/model-host.ts +3 -4
- package/src/serve/preparation.ts +3 -3
- package/src/serve/prompt-response-trace.ts +14 -0
- package/src/serve/request-plan.ts +6 -3
- package/src/serve/request-prep.ts +40 -25
- package/src/serve/text-completion-stage.ts +2 -2
- package/src/serve/transcription-server.ts +64 -0
- package/src/serve/transcription-service.ts +502 -0
- package/src/server.ts +185 -261
- package/src/spec/assistant-rows.ts +93 -0
- package/src/spec/assistant-source.ts +28 -18
- package/src/spec/cache-identity.ts +22 -0
- package/src/spec/deepspec-rows.ts +29 -0
- package/src/spec/deepspec-source.ts +12 -10
- package/src/spec/dflash-source.ts +28 -4
- package/src/spec/drafter.ts +153 -142
- package/src/spec/dspark/deepspec-module.ts +95 -120
- package/src/spec/dspark/module-dflash.ts +126 -170
- package/src/spec/glm52-mtp-graph.ts +29 -0
- package/src/spec/glm52-mtp-rows.ts +132 -0
- package/src/spec/glm52-mtp-source.ts +54 -136
- package/src/spec/glm52-mtp-state.ts +33 -0
- package/src/spec/ngram-source.ts +115 -21
- package/src/spec/projected-draft-rows.ts +73 -0
- package/src/spec/qwen-mtp-module.ts +171 -0
- package/src/spec/qwen-mtp-rows.ts +230 -0
- package/src/spec/qwen-mtp-source.ts +84 -226
- package/src/spec/qwen-mtp-state.ts +30 -0
- package/src/spec/serve-loop.ts +73 -69
- package/src/spec/source.ts +93 -15
- package/src/spec/two-model-rows.ts +62 -0
- package/src/spec/two-model.ts +45 -103
- package/src/ssd-cache.ts +107 -36
- package/src/ssd-durability.ts +29 -27
- package/src/storage/host-buffer.ts +24 -0
- package/src/storage/kv-writer.ts +83 -0
- package/src/storage/kv-writer.worker.js +264 -0
- package/src/storage/retention-policy.ts +37 -0
- package/src/tiered-prompt-cache.ts +45 -0
- package/src/vision/embedder.ts +8 -0
- package/src/vision/prompt.ts +101 -65
- package/src/vision/qwen3vl-prompt.ts +101 -64
- package/src/vision/qwen3vl-tower.ts +11 -0
- package/src/vision/siglip.ts +8 -0
- package/src/web/app.html +12 -0
- package/src/web/app.js +217 -57
- package/src/web/src/chat.ts +4 -1
- package/src/web/src/composer.ts +3 -0
- package/src/web/src/voice.ts +141 -0
- package/src/web/sw.js +1 -1
- package/src/backends/mlx/memory-guard.ts +0 -40
- package/src/spec/prefix-state.ts +0 -45
- /package/docs/{planning → archive/planning}/release-notes-v0.3.0.md +0 -0
package/src/preflight.ts
CHANGED
|
@@ -32,6 +32,9 @@ export interface MachineState {
|
|
|
32
32
|
/** Foreign processes holding > bigRssMB resident (excludes kernel,
|
|
33
33
|
* WindowServer, and our own bun/python benchmark processes). */
|
|
34
34
|
bigProcesses: { rssMB: number; command: string }[];
|
|
35
|
+
/** Explicitly permitted CPU-only background commands; RSS limits still apply. */
|
|
36
|
+
allowedCpuProcesses?: string[];
|
|
37
|
+
backgroundCpuProcesses?: { rssMB: number; command: string }[];
|
|
35
38
|
ok: boolean;
|
|
36
39
|
problems: string[];
|
|
37
40
|
at: string;
|
|
@@ -41,6 +44,29 @@ export interface PreflightLimits {
|
|
|
41
44
|
maxSwapUsedMB?: number; // default 512
|
|
42
45
|
minFreePercent?: number; // default 35
|
|
43
46
|
bigRssMB?: number; // default 2048
|
|
47
|
+
allowCpuProcesses?: string[];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function foreignProcessFindings(output: string, limits: PreflightLimits = {}) {
|
|
51
|
+
const bigProcesses: { rssMB: number; command: string }[] = [];
|
|
52
|
+
const backgroundCpuProcesses: { rssMB: number; command: string }[] = [];
|
|
53
|
+
for (const line of output.split("\n")) {
|
|
54
|
+
const m = line.match(/^\s*([\d.]+)\s+(\d+)\s+(.*)$/);
|
|
55
|
+
if (!m) continue;
|
|
56
|
+
const cpuPct = Number(m[1]);
|
|
57
|
+
const rssMB = Number(m[2]) / 1024;
|
|
58
|
+
const command = m[3]!;
|
|
59
|
+
if (rssMB < (limits.bigRssMB ?? 2048) && cpuPct < 50) continue;
|
|
60
|
+
if (/(bun|\.venv\/bin\/python|mlx_lm|optiq)/.test(command) ||
|
|
61
|
+
/(kernel_task|WindowServer|launchd|mds|spotlight)/i.test(command)) continue;
|
|
62
|
+
const finding = { rssMB: Math.round(rssMB),
|
|
63
|
+
command: command.slice(0, 80) + (cpuPct >= 50 ? ` (${cpuPct.toFixed(0)}% cpu)` : "") };
|
|
64
|
+
// Exact command match; permission to tolerate CPU activity never permits
|
|
65
|
+
// that process to consume the benchmark's memory headroom.
|
|
66
|
+
const allowed = rssMB < (limits.bigRssMB ?? 2048) && limits.allowCpuProcesses?.includes(command);
|
|
67
|
+
(allowed ? backgroundCpuProcesses : bigProcesses).push(finding);
|
|
68
|
+
}
|
|
69
|
+
return { bigProcesses, backgroundCpuProcesses };
|
|
44
70
|
}
|
|
45
71
|
|
|
46
72
|
function sh(cmd: string[]): string {
|
|
@@ -51,7 +77,6 @@ function sh(cmd: string[]): string {
|
|
|
51
77
|
export function checkMachine(limits: PreflightLimits = {}): MachineState {
|
|
52
78
|
const maxSwap = limits.maxSwapUsedMB ?? 512;
|
|
53
79
|
const minFree = limits.minFreePercent ?? 35;
|
|
54
|
-
const bigRss = limits.bigRssMB ?? 2048;
|
|
55
80
|
const problems: string[] = [];
|
|
56
81
|
|
|
57
82
|
// swap: "vm.swapusage: total = 7168.00M used = 6413.06M free = ..."
|
|
@@ -95,26 +120,15 @@ export function checkMachine(limits: PreflightLimits = {}): MachineState {
|
|
|
95
120
|
// one clean sample doesn't prove absence (the run-spread stability
|
|
96
121
|
// policy in bench-h2h.ts is the direct signal); a hot sample here is
|
|
97
122
|
// still a cheap early warning.
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
const BIG_CPU_PCT = 50;
|
|
101
|
-
const bigProcesses: { rssMB: number; command: string }[] = [];
|
|
102
|
-
for (const line of sh(["ps", "axo", "pcpu=,rss=,command="]).split("\n")) {
|
|
103
|
-
const m = line.match(/^\s*([\d.]+)\s+(\d+)\s+(.*)$/);
|
|
104
|
-
if (!m) continue;
|
|
105
|
-
const cpuPct = Number(m[1]);
|
|
106
|
-
const rssMB = Number(m[2]) / 1024;
|
|
107
|
-
const command = m[3]!.slice(0, 80);
|
|
108
|
-
if (rssMB < bigRss && cpuPct < BIG_CPU_PCT) continue;
|
|
109
|
-
if (SELF_PATTERN.test(command) || SYSTEM_PATTERN.test(command)) continue;
|
|
110
|
-
bigProcesses.push({ rssMB: Math.round(rssMB), command: cpuPct >= BIG_CPU_PCT ? `${command} (${cpuPct.toFixed(0)}% cpu)` : command });
|
|
111
|
-
}
|
|
123
|
+
const { bigProcesses, backgroundCpuProcesses } = foreignProcessFindings(
|
|
124
|
+
sh(["ps", "axo", "pcpu=,rss=,command="]), limits);
|
|
112
125
|
for (const p of bigProcesses)
|
|
113
126
|
problems.push(`big process: ${p.rssMB} MB — ${p.command}`);
|
|
114
127
|
|
|
115
128
|
return {
|
|
116
129
|
swapUsedMB, freePercent, cpuSpeedLimit, gpuWiredLimitMB, loadAvg1m,
|
|
117
130
|
bigProcesses,
|
|
131
|
+
allowedCpuProcesses: limits.allowCpuProcesses ?? [], backgroundCpuProcesses,
|
|
118
132
|
ok: problems.length === 0,
|
|
119
133
|
problems,
|
|
120
134
|
at: new Date().toISOString(),
|
|
@@ -136,6 +150,8 @@ export function machineStateJson(s: MachineState): string {
|
|
|
136
150
|
gpu_wired_mb: s.gpuWiredLimitMB,
|
|
137
151
|
load1m: s.loadAvg1m,
|
|
138
152
|
big_procs: s.bigProcesses.length,
|
|
153
|
+
allowed_cpu_processes: s.allowedCpuProcesses ?? [],
|
|
154
|
+
background_cpu_processes: s.backgroundCpuProcesses ?? [],
|
|
139
155
|
ok: s.ok,
|
|
140
156
|
at: s.at,
|
|
141
157
|
});
|