mlx-bun 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +57 -32
  2. package/docs/README.md +58 -42
  3. package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +3617 -0
  4. package/docs/archive/investigations/openjarvis-fit.md +198 -0
  5. package/docs/archive/investigations/qwen38-27b-campaign-log.md +1092 -0
  6. package/docs/archive/planning/release-notes-v0.4.0.md +45 -0
  7. package/docs/archive/planning/release-notes-v0.5.0.md +37 -0
  8. package/docs/design/batching.md +1535 -50
  9. package/docs/design/decode-speed-program.md +394 -1171
  10. package/docs/design/generic-model-support.md +195 -3
  11. package/docs/design/kv-cache.md +538 -225
  12. package/docs/design/speculative-decoding.md +552 -50
  13. package/docs/design/turboquant.md +140 -17
  14. package/docs/design/unified-engine-frontier-plan.md +259 -150
  15. package/docs/planning/PRODUCT_ROADMAP.md +21 -6
  16. package/docs/planning/public-readiness-handoff.md +244 -0
  17. package/docs/planning/public-readiness.md +149 -0
  18. package/docs/reference/benchmarks.md +3537 -1112
  19. package/docs/reference/cli.md +102 -2
  20. package/docs/reference/distribution.md +45 -13
  21. package/docs/reference/environment.md +159 -13
  22. package/docs/reference/library-api.md +18 -4
  23. package/docs/reference/memory.md +1 -1
  24. package/docs/reference/models.md +28 -7
  25. package/docs/reference/server-api.md +257 -33
  26. package/docs/reference/server-config.md +372 -151
  27. package/docs/reference/troubleshooting.md +13 -21
  28. package/package.json +2 -2
  29. package/src/audio/audiotoolbox.ts +112 -0
  30. package/src/audio/conformer.ts +8 -0
  31. package/src/audio/mic-capture.ts +98 -0
  32. package/src/audio/silero-vad.ts +379 -0
  33. package/src/audio/whisper-decode.ts +774 -0
  34. package/src/audio/whisper-fast.ts +732 -0
  35. package/src/audio/whisper-format.ts +73 -0
  36. package/src/audio/whisper-mel.ts +185 -0
  37. package/src/audio/whisper-timing.ts +274 -0
  38. package/src/audio/whisper-tokenizer.ts +239 -0
  39. package/src/audio/whisper-transcribe.ts +410 -0
  40. package/src/backends/mlx/assistant-target.ts +46 -0
  41. package/src/backends/mlx/autoregressive-draft-rows.ts +120 -0
  42. package/src/backends/mlx/autoregressive.ts +22 -1
  43. package/src/backends/mlx/batch-group.ts +464 -236
  44. package/src/backends/mlx/cache-attachment.ts +67 -0
  45. package/src/backends/mlx/cache-layout.ts +31 -0
  46. package/src/backends/mlx/checkpoint-state.ts +53 -0
  47. package/src/backends/mlx/continuation-persistence.ts +107 -0
  48. package/src/backends/mlx/continuation-request.ts +43 -0
  49. package/src/backends/mlx/continuation.ts +77 -0
  50. package/src/backends/mlx/draft-target.ts +7 -22
  51. package/src/backends/mlx/encoder-cache.ts +25 -0
  52. package/src/backends/mlx/fill-append.ts +21 -8
  53. package/src/backends/mlx/fill-group.ts +497 -0
  54. package/src/backends/mlx/full-kv-row-append.ts +32 -0
  55. package/src/backends/mlx/full-kv-row-donor.ts +54 -0
  56. package/src/backends/mlx/full-transitioning-kv-rows.ts +65 -0
  57. package/src/backends/mlx/gateway-binding.ts +122 -19
  58. package/src/backends/mlx/kv-codec.ts +9 -0
  59. package/src/backends/mlx/kv-maintenance.ts +161 -0
  60. package/src/backends/mlx/kv-tensor-rows.ts +212 -0
  61. package/src/backends/mlx/mixed-iteration.ts +62 -0
  62. package/src/backends/mlx/model-prompt.ts +46 -32
  63. package/src/backends/mlx/model-serving.ts +5 -4
  64. package/src/backends/mlx/paged-cache-codec.ts +27 -0
  65. package/src/backends/mlx/prefill-cohort.ts +100 -0
  66. package/src/backends/mlx/prefill-policy.ts +26 -0
  67. package/src/backends/mlx/prefill-rows.ts +244 -0
  68. package/src/backends/mlx/prefill.ts +1 -1
  69. package/src/backends/mlx/projected-context-rows.ts +72 -0
  70. package/src/backends/mlx/prompt-input.ts +29 -0
  71. package/src/backends/mlx/qwen-prompt-input.ts +40 -0
  72. package/src/backends/mlx/request-state-policy.ts +54 -0
  73. package/src/backends/mlx/rollback.ts +45 -4
  74. package/src/backends/mlx/row-sampling.ts +49 -0
  75. package/src/backends/mlx/serial-executor.ts +51 -73
  76. package/src/backends/mlx/speculative-group.ts +438 -0
  77. package/src/backends/mlx/speculative-round.ts +161 -0
  78. package/src/backends/mlx/speculative.ts +24 -7
  79. package/src/backends/mlx/state-rows.ts +104 -0
  80. package/src/backends/mlx/state-views.ts +21 -1
  81. package/src/backends/mlx/target-layout-capability.ts +35 -0
  82. package/src/backends/mlx/transitioning-kv-rows.ts +173 -0
  83. package/src/cli.ts +333 -36
  84. package/src/contracts/execution.ts +21 -0
  85. package/src/contracts/object-cache.ts +8 -0
  86. package/src/contracts/pi-provider.ts +1 -1
  87. package/src/contracts/pi.ts +3 -1
  88. package/src/contracts/prefix-cache.ts +33 -0
  89. package/src/contracts/resources.ts +7 -0
  90. package/src/contracts/sampling.ts +23 -0
  91. package/src/contracts/scheduling.ts +27 -0
  92. package/src/engine/execution-plan.ts +38 -25
  93. package/src/engine/execution-tasks.ts +44 -0
  94. package/src/engine/namespaced-cache.ts +13 -0
  95. package/src/engine/resources.ts +12 -1
  96. package/src/engine/scheduler.ts +32 -7
  97. package/src/fill/fill-session.ts +14 -3
  98. package/src/fit.ts +29 -15
  99. package/src/generate.ts +105 -195
  100. package/src/grammar.ts +13 -0
  101. package/src/inference/draft-acceptance.ts +41 -0
  102. package/src/inference/draft-output.ts +49 -0
  103. package/src/inference/prefill.ts +6 -0
  104. package/src/inference/rollback.ts +2 -2
  105. package/src/kv-scheme.ts +92 -12
  106. package/src/kv-store.ts +183 -93
  107. package/src/lab/README.md +5 -1
  108. package/src/lab/paged-kv/paged-attention.ts +117 -0
  109. package/src/lab/paged-kv/paged-kv-rows.ts +111 -0
  110. package/src/lab/paged-kv/paged-kv.ts +126 -46
  111. package/src/md-modules.d.ts +5 -0
  112. package/src/mlx/array.ts +39 -1
  113. package/src/mlx/ffi.ts +19 -0
  114. package/src/mlx/normalized-argmax.ts +116 -0
  115. package/src/mlx/ops.ts +97 -10
  116. package/src/model/artifact-identity.ts +73 -0
  117. package/src/model/batched-kv.ts +235 -0
  118. package/src/model/batched-mask.ts +12 -4
  119. package/src/model/batched-quant.ts +8 -2
  120. package/src/model/batched-quantized-kv.ts +144 -0
  121. package/src/model/batched-rotating-quant.ts +76 -15
  122. package/src/model/batched-rotating-state.ts +76 -0
  123. package/src/model/batched-rotating.ts +64 -22
  124. package/src/model/batched-row-storage.ts +52 -0
  125. package/src/model/batched-ssm.ts +52 -0
  126. package/src/model/batched-turboquant-kv.ts +94 -0
  127. package/src/model/compiled-decode.ts +7 -1
  128. package/src/model/decoded-kv-donor.ts +12 -0
  129. package/src/model/delayed-quantized-kv.ts +72 -0
  130. package/src/model/delayed-rotating-quantized-kv.ts +93 -0
  131. package/src/model/delayed-turboquant-kv.ts +57 -0
  132. package/src/model/factory.ts +26 -0
  133. package/src/model/fingerprint.ts +12 -0
  134. package/src/model/full-prefill-padding.ts +56 -0
  135. package/src/model/full-prefill-row.ts +105 -0
  136. package/src/model/gemma4-base.ts +183 -82
  137. package/src/model/gemma4.ts +116 -28
  138. package/src/model/generated/gemma4-12b.ts +4 -1
  139. package/src/model/generated/gemma4-26b.ts +4 -1
  140. package/src/model/generated/gemma4-e4b.ts +4 -1
  141. package/src/model/glm52-cache.ts +80 -4
  142. package/src/model/glm52-mla.ts +12 -6
  143. package/src/model/glm52.ts +19 -40
  144. package/src/model/kv-attention-view.ts +70 -0
  145. package/src/model/minicpm5.ts +11 -7
  146. package/src/model/profile.ts +29 -4
  147. package/src/model/quantized-append-attention.ts +31 -0
  148. package/src/model/qwen3-delta-state.ts +74 -0
  149. package/src/model/qwen3-delta.ts +59 -36
  150. package/src/model/qwen3-mrope.ts +7 -9
  151. package/src/model/qwen3_5.ts +177 -75
  152. package/src/model/rotating-kv-donor.ts +35 -0
  153. package/src/model/rotating-kv-layout.ts +262 -0
  154. package/src/model/rotating-row-transaction.ts +35 -0
  155. package/src/model/speculative-rotating-kv.ts +101 -0
  156. package/src/model/ssm-prefill-padding.ts +39 -0
  157. package/src/model/support.ts +8 -0
  158. package/src/model/token-groups.ts +66 -0
  159. package/src/model/trellis-linear.ts +4 -4
  160. package/src/model/turboquant-codec.ts +89 -0
  161. package/src/model/universal/dense.ts +14 -13
  162. package/src/model/whisper.ts +435 -0
  163. package/src/native/mic_capture.swift +92 -0
  164. package/src/native-pack.ts +6 -5
  165. package/src/pi-web.ts +9 -1
  166. package/src/preflight.ts +31 -15
  167. package/src/prompt-cache.ts +306 -86
  168. package/src/runtime-config.ts +1 -5
  169. package/src/safetensors.ts +9 -2
  170. package/src/sampler.ts +37 -25
  171. package/src/serve/audio-routes.ts +323 -0
  172. package/src/serve/chat-request.ts +2 -0
  173. package/src/serve/chat-stage.ts +19 -14
  174. package/src/serve/completion-executor.ts +9 -1
  175. package/src/serve/discovery-routes.ts +17 -1
  176. package/src/serve/generated-token-history.ts +57 -0
  177. package/src/serve/generation-gateway.ts +88 -38
  178. package/src/serve/model-binding.ts +7 -1
  179. package/src/serve/model-host.ts +3 -4
  180. package/src/serve/preparation.ts +3 -3
  181. package/src/serve/prompt-response-trace.ts +14 -0
  182. package/src/serve/request-plan.ts +6 -3
  183. package/src/serve/request-prep.ts +40 -25
  184. package/src/serve/text-completion-stage.ts +2 -2
  185. package/src/serve/transcription-server.ts +64 -0
  186. package/src/serve/transcription-service.ts +502 -0
  187. package/src/server.ts +185 -261
  188. package/src/spec/assistant-rows.ts +93 -0
  189. package/src/spec/assistant-source.ts +28 -18
  190. package/src/spec/cache-identity.ts +22 -0
  191. package/src/spec/deepspec-rows.ts +29 -0
  192. package/src/spec/deepspec-source.ts +12 -10
  193. package/src/spec/dflash-source.ts +28 -4
  194. package/src/spec/drafter.ts +153 -142
  195. package/src/spec/dspark/deepspec-module.ts +95 -120
  196. package/src/spec/dspark/module-dflash.ts +126 -170
  197. package/src/spec/glm52-mtp-graph.ts +29 -0
  198. package/src/spec/glm52-mtp-rows.ts +132 -0
  199. package/src/spec/glm52-mtp-source.ts +54 -136
  200. package/src/spec/glm52-mtp-state.ts +33 -0
  201. package/src/spec/ngram-source.ts +115 -21
  202. package/src/spec/projected-draft-rows.ts +73 -0
  203. package/src/spec/qwen-mtp-module.ts +171 -0
  204. package/src/spec/qwen-mtp-rows.ts +230 -0
  205. package/src/spec/qwen-mtp-source.ts +84 -226
  206. package/src/spec/qwen-mtp-state.ts +30 -0
  207. package/src/spec/serve-loop.ts +73 -69
  208. package/src/spec/source.ts +93 -15
  209. package/src/spec/two-model-rows.ts +62 -0
  210. package/src/spec/two-model.ts +45 -103
  211. package/src/ssd-cache.ts +107 -36
  212. package/src/ssd-durability.ts +29 -27
  213. package/src/storage/host-buffer.ts +24 -0
  214. package/src/storage/kv-writer.ts +83 -0
  215. package/src/storage/kv-writer.worker.js +264 -0
  216. package/src/storage/retention-policy.ts +37 -0
  217. package/src/tiered-prompt-cache.ts +45 -0
  218. package/src/vision/embedder.ts +8 -0
  219. package/src/vision/prompt.ts +101 -65
  220. package/src/vision/qwen3vl-prompt.ts +101 -64
  221. package/src/vision/qwen3vl-tower.ts +11 -0
  222. package/src/vision/siglip.ts +8 -0
  223. package/src/web/app.html +12 -0
  224. package/src/web/app.js +217 -57
  225. package/src/web/src/chat.ts +4 -1
  226. package/src/web/src/composer.ts +3 -0
  227. package/src/web/src/voice.ts +141 -0
  228. package/src/web/sw.js +1 -1
  229. package/src/backends/mlx/memory-guard.ts +0 -40
  230. package/src/spec/prefix-state.ts +0 -45
  231. /package/docs/{planning → archive/planning}/release-notes-v0.3.0.md +0 -0
package/src/preflight.ts CHANGED
@@ -32,6 +32,9 @@ export interface MachineState {
32
32
  /** Foreign processes holding > bigRssMB resident (excludes kernel,
33
33
  * WindowServer, and our own bun/python benchmark processes). */
34
34
  bigProcesses: { rssMB: number; command: string }[];
35
+ /** Explicitly permitted CPU-only background commands; RSS limits still apply. */
36
+ allowedCpuProcesses?: string[];
37
+ backgroundCpuProcesses?: { rssMB: number; command: string }[];
35
38
  ok: boolean;
36
39
  problems: string[];
37
40
  at: string;
@@ -41,6 +44,29 @@ export interface PreflightLimits {
41
44
  maxSwapUsedMB?: number; // default 512
42
45
  minFreePercent?: number; // default 35
43
46
  bigRssMB?: number; // default 2048
47
+ allowCpuProcesses?: string[];
48
+ }
49
+
50
+ export function foreignProcessFindings(output: string, limits: PreflightLimits = {}) {
51
+ const bigProcesses: { rssMB: number; command: string }[] = [];
52
+ const backgroundCpuProcesses: { rssMB: number; command: string }[] = [];
53
+ for (const line of output.split("\n")) {
54
+ const m = line.match(/^\s*([\d.]+)\s+(\d+)\s+(.*)$/);
55
+ if (!m) continue;
56
+ const cpuPct = Number(m[1]);
57
+ const rssMB = Number(m[2]) / 1024;
58
+ const command = m[3]!;
59
+ if (rssMB < (limits.bigRssMB ?? 2048) && cpuPct < 50) continue;
60
+ if (/(bun|\.venv\/bin\/python|mlx_lm|optiq)/.test(command) ||
61
+ /(kernel_task|WindowServer|launchd|mds|spotlight)/i.test(command)) continue;
62
+ const finding = { rssMB: Math.round(rssMB),
63
+ command: command.slice(0, 80) + (cpuPct >= 50 ? ` (${cpuPct.toFixed(0)}% cpu)` : "") };
64
+ // Exact command match; permission to tolerate CPU activity never permits
65
+ // that process to consume the benchmark's memory headroom.
66
+ const allowed = rssMB < (limits.bigRssMB ?? 2048) && limits.allowCpuProcesses?.includes(command);
67
+ (allowed ? backgroundCpuProcesses : bigProcesses).push(finding);
68
+ }
69
+ return { bigProcesses, backgroundCpuProcesses };
44
70
  }
45
71
 
46
72
  function sh(cmd: string[]): string {
@@ -51,7 +77,6 @@ function sh(cmd: string[]): string {
51
77
  export function checkMachine(limits: PreflightLimits = {}): MachineState {
52
78
  const maxSwap = limits.maxSwapUsedMB ?? 512;
53
79
  const minFree = limits.minFreePercent ?? 35;
54
- const bigRss = limits.bigRssMB ?? 2048;
55
80
  const problems: string[] = [];
56
81
 
57
82
  // swap: "vm.swapusage: total = 7168.00M used = 6413.06M free = ..."
@@ -95,26 +120,15 @@ export function checkMachine(limits: PreflightLimits = {}): MachineState {
95
120
  // one clean sample doesn't prove absence (the run-spread stability
96
121
  // policy in bench-h2h.ts is the direct signal); a hot sample here is
97
122
  // still a cheap early warning.
98
- const SELF_PATTERN = /(bun|\.venv\/bin\/python|mlx_lm|optiq)/;
99
- const SYSTEM_PATTERN = /(kernel_task|WindowServer|launchd|mds|spotlight)/i;
100
- const BIG_CPU_PCT = 50;
101
- const bigProcesses: { rssMB: number; command: string }[] = [];
102
- for (const line of sh(["ps", "axo", "pcpu=,rss=,command="]).split("\n")) {
103
- const m = line.match(/^\s*([\d.]+)\s+(\d+)\s+(.*)$/);
104
- if (!m) continue;
105
- const cpuPct = Number(m[1]);
106
- const rssMB = Number(m[2]) / 1024;
107
- const command = m[3]!.slice(0, 80);
108
- if (rssMB < bigRss && cpuPct < BIG_CPU_PCT) continue;
109
- if (SELF_PATTERN.test(command) || SYSTEM_PATTERN.test(command)) continue;
110
- bigProcesses.push({ rssMB: Math.round(rssMB), command: cpuPct >= BIG_CPU_PCT ? `${command} (${cpuPct.toFixed(0)}% cpu)` : command });
111
- }
123
+ const { bigProcesses, backgroundCpuProcesses } = foreignProcessFindings(
124
+ sh(["ps", "axo", "pcpu=,rss=,command="]), limits);
112
125
  for (const p of bigProcesses)
113
126
  problems.push(`big process: ${p.rssMB} MB — ${p.command}`);
114
127
 
115
128
  return {
116
129
  swapUsedMB, freePercent, cpuSpeedLimit, gpuWiredLimitMB, loadAvg1m,
117
130
  bigProcesses,
131
+ allowedCpuProcesses: limits.allowCpuProcesses ?? [], backgroundCpuProcesses,
118
132
  ok: problems.length === 0,
119
133
  problems,
120
134
  at: new Date().toISOString(),
@@ -136,6 +150,8 @@ export function machineStateJson(s: MachineState): string {
136
150
  gpu_wired_mb: s.gpuWiredLimitMB,
137
151
  load1m: s.loadAvg1m,
138
152
  big_procs: s.bigProcesses.length,
153
+ allowed_cpu_processes: s.allowedCpuProcesses ?? [],
154
+ background_cpu_processes: s.backgroundCpuProcesses ?? [],
139
155
  ok: s.ok,
140
156
  at: s.at,
141
157
  });