mlx-bun 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +57 -32
  2. package/docs/README.md +58 -42
  3. package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +3617 -0
  4. package/docs/archive/investigations/openjarvis-fit.md +198 -0
  5. package/docs/archive/investigations/qwen38-27b-campaign-log.md +1092 -0
  6. package/docs/archive/planning/release-notes-v0.4.0.md +45 -0
  7. package/docs/archive/planning/release-notes-v0.5.0.md +37 -0
  8. package/docs/design/batching.md +1535 -50
  9. package/docs/design/decode-speed-program.md +394 -1171
  10. package/docs/design/generic-model-support.md +195 -3
  11. package/docs/design/kv-cache.md +538 -225
  12. package/docs/design/speculative-decoding.md +552 -50
  13. package/docs/design/turboquant.md +140 -17
  14. package/docs/design/unified-engine-frontier-plan.md +259 -150
  15. package/docs/planning/PRODUCT_ROADMAP.md +21 -6
  16. package/docs/planning/public-readiness-handoff.md +244 -0
  17. package/docs/planning/public-readiness.md +149 -0
  18. package/docs/reference/benchmarks.md +3537 -1112
  19. package/docs/reference/cli.md +102 -2
  20. package/docs/reference/distribution.md +45 -13
  21. package/docs/reference/environment.md +159 -13
  22. package/docs/reference/library-api.md +18 -4
  23. package/docs/reference/memory.md +1 -1
  24. package/docs/reference/models.md +28 -7
  25. package/docs/reference/server-api.md +257 -33
  26. package/docs/reference/server-config.md +372 -151
  27. package/docs/reference/troubleshooting.md +13 -21
  28. package/package.json +2 -2
  29. package/src/audio/audiotoolbox.ts +112 -0
  30. package/src/audio/conformer.ts +8 -0
  31. package/src/audio/mic-capture.ts +98 -0
  32. package/src/audio/silero-vad.ts +379 -0
  33. package/src/audio/whisper-decode.ts +774 -0
  34. package/src/audio/whisper-fast.ts +732 -0
  35. package/src/audio/whisper-format.ts +73 -0
  36. package/src/audio/whisper-mel.ts +185 -0
  37. package/src/audio/whisper-timing.ts +274 -0
  38. package/src/audio/whisper-tokenizer.ts +239 -0
  39. package/src/audio/whisper-transcribe.ts +410 -0
  40. package/src/backends/mlx/assistant-target.ts +46 -0
  41. package/src/backends/mlx/autoregressive-draft-rows.ts +120 -0
  42. package/src/backends/mlx/autoregressive.ts +22 -1
  43. package/src/backends/mlx/batch-group.ts +464 -236
  44. package/src/backends/mlx/cache-attachment.ts +67 -0
  45. package/src/backends/mlx/cache-layout.ts +31 -0
  46. package/src/backends/mlx/checkpoint-state.ts +53 -0
  47. package/src/backends/mlx/continuation-persistence.ts +107 -0
  48. package/src/backends/mlx/continuation-request.ts +43 -0
  49. package/src/backends/mlx/continuation.ts +77 -0
  50. package/src/backends/mlx/draft-target.ts +7 -22
  51. package/src/backends/mlx/encoder-cache.ts +25 -0
  52. package/src/backends/mlx/fill-append.ts +21 -8
  53. package/src/backends/mlx/fill-group.ts +497 -0
  54. package/src/backends/mlx/full-kv-row-append.ts +32 -0
  55. package/src/backends/mlx/full-kv-row-donor.ts +54 -0
  56. package/src/backends/mlx/full-transitioning-kv-rows.ts +65 -0
  57. package/src/backends/mlx/gateway-binding.ts +122 -19
  58. package/src/backends/mlx/kv-codec.ts +9 -0
  59. package/src/backends/mlx/kv-maintenance.ts +161 -0
  60. package/src/backends/mlx/kv-tensor-rows.ts +212 -0
  61. package/src/backends/mlx/mixed-iteration.ts +62 -0
  62. package/src/backends/mlx/model-prompt.ts +46 -32
  63. package/src/backends/mlx/model-serving.ts +5 -4
  64. package/src/backends/mlx/paged-cache-codec.ts +27 -0
  65. package/src/backends/mlx/prefill-cohort.ts +100 -0
  66. package/src/backends/mlx/prefill-policy.ts +26 -0
  67. package/src/backends/mlx/prefill-rows.ts +244 -0
  68. package/src/backends/mlx/prefill.ts +1 -1
  69. package/src/backends/mlx/projected-context-rows.ts +72 -0
  70. package/src/backends/mlx/prompt-input.ts +29 -0
  71. package/src/backends/mlx/qwen-prompt-input.ts +40 -0
  72. package/src/backends/mlx/request-state-policy.ts +54 -0
  73. package/src/backends/mlx/rollback.ts +45 -4
  74. package/src/backends/mlx/row-sampling.ts +49 -0
  75. package/src/backends/mlx/serial-executor.ts +51 -73
  76. package/src/backends/mlx/speculative-group.ts +438 -0
  77. package/src/backends/mlx/speculative-round.ts +161 -0
  78. package/src/backends/mlx/speculative.ts +24 -7
  79. package/src/backends/mlx/state-rows.ts +104 -0
  80. package/src/backends/mlx/state-views.ts +21 -1
  81. package/src/backends/mlx/target-layout-capability.ts +35 -0
  82. package/src/backends/mlx/transitioning-kv-rows.ts +173 -0
  83. package/src/cli.ts +333 -36
  84. package/src/contracts/execution.ts +21 -0
  85. package/src/contracts/object-cache.ts +8 -0
  86. package/src/contracts/pi-provider.ts +1 -1
  87. package/src/contracts/pi.ts +3 -1
  88. package/src/contracts/prefix-cache.ts +33 -0
  89. package/src/contracts/resources.ts +7 -0
  90. package/src/contracts/sampling.ts +23 -0
  91. package/src/contracts/scheduling.ts +27 -0
  92. package/src/engine/execution-plan.ts +38 -25
  93. package/src/engine/execution-tasks.ts +44 -0
  94. package/src/engine/namespaced-cache.ts +13 -0
  95. package/src/engine/resources.ts +12 -1
  96. package/src/engine/scheduler.ts +32 -7
  97. package/src/fill/fill-session.ts +14 -3
  98. package/src/fit.ts +29 -15
  99. package/src/generate.ts +105 -195
  100. package/src/grammar.ts +13 -0
  101. package/src/inference/draft-acceptance.ts +41 -0
  102. package/src/inference/draft-output.ts +49 -0
  103. package/src/inference/prefill.ts +6 -0
  104. package/src/inference/rollback.ts +2 -2
  105. package/src/kv-scheme.ts +92 -12
  106. package/src/kv-store.ts +183 -93
  107. package/src/lab/README.md +5 -1
  108. package/src/lab/paged-kv/paged-attention.ts +117 -0
  109. package/src/lab/paged-kv/paged-kv-rows.ts +111 -0
  110. package/src/lab/paged-kv/paged-kv.ts +126 -46
  111. package/src/md-modules.d.ts +5 -0
  112. package/src/mlx/array.ts +39 -1
  113. package/src/mlx/ffi.ts +19 -0
  114. package/src/mlx/normalized-argmax.ts +116 -0
  115. package/src/mlx/ops.ts +97 -10
  116. package/src/model/artifact-identity.ts +73 -0
  117. package/src/model/batched-kv.ts +235 -0
  118. package/src/model/batched-mask.ts +12 -4
  119. package/src/model/batched-quant.ts +8 -2
  120. package/src/model/batched-quantized-kv.ts +144 -0
  121. package/src/model/batched-rotating-quant.ts +76 -15
  122. package/src/model/batched-rotating-state.ts +76 -0
  123. package/src/model/batched-rotating.ts +64 -22
  124. package/src/model/batched-row-storage.ts +52 -0
  125. package/src/model/batched-ssm.ts +52 -0
  126. package/src/model/batched-turboquant-kv.ts +94 -0
  127. package/src/model/compiled-decode.ts +7 -1
  128. package/src/model/decoded-kv-donor.ts +12 -0
  129. package/src/model/delayed-quantized-kv.ts +72 -0
  130. package/src/model/delayed-rotating-quantized-kv.ts +93 -0
  131. package/src/model/delayed-turboquant-kv.ts +57 -0
  132. package/src/model/factory.ts +26 -0
  133. package/src/model/fingerprint.ts +12 -0
  134. package/src/model/full-prefill-padding.ts +56 -0
  135. package/src/model/full-prefill-row.ts +105 -0
  136. package/src/model/gemma4-base.ts +183 -82
  137. package/src/model/gemma4.ts +116 -28
  138. package/src/model/generated/gemma4-12b.ts +4 -1
  139. package/src/model/generated/gemma4-26b.ts +4 -1
  140. package/src/model/generated/gemma4-e4b.ts +4 -1
  141. package/src/model/glm52-cache.ts +80 -4
  142. package/src/model/glm52-mla.ts +12 -6
  143. package/src/model/glm52.ts +19 -40
  144. package/src/model/kv-attention-view.ts +70 -0
  145. package/src/model/minicpm5.ts +11 -7
  146. package/src/model/profile.ts +29 -4
  147. package/src/model/quantized-append-attention.ts +31 -0
  148. package/src/model/qwen3-delta-state.ts +74 -0
  149. package/src/model/qwen3-delta.ts +59 -36
  150. package/src/model/qwen3-mrope.ts +7 -9
  151. package/src/model/qwen3_5.ts +177 -75
  152. package/src/model/rotating-kv-donor.ts +35 -0
  153. package/src/model/rotating-kv-layout.ts +262 -0
  154. package/src/model/rotating-row-transaction.ts +35 -0
  155. package/src/model/speculative-rotating-kv.ts +101 -0
  156. package/src/model/ssm-prefill-padding.ts +39 -0
  157. package/src/model/support.ts +8 -0
  158. package/src/model/token-groups.ts +66 -0
  159. package/src/model/trellis-linear.ts +4 -4
  160. package/src/model/turboquant-codec.ts +89 -0
  161. package/src/model/universal/dense.ts +14 -13
  162. package/src/model/whisper.ts +435 -0
  163. package/src/native/mic_capture.swift +92 -0
  164. package/src/native-pack.ts +6 -5
  165. package/src/pi-web.ts +9 -1
  166. package/src/preflight.ts +31 -15
  167. package/src/prompt-cache.ts +306 -86
  168. package/src/runtime-config.ts +1 -5
  169. package/src/safetensors.ts +9 -2
  170. package/src/sampler.ts +37 -25
  171. package/src/serve/audio-routes.ts +323 -0
  172. package/src/serve/chat-request.ts +2 -0
  173. package/src/serve/chat-stage.ts +19 -14
  174. package/src/serve/completion-executor.ts +9 -1
  175. package/src/serve/discovery-routes.ts +17 -1
  176. package/src/serve/generated-token-history.ts +57 -0
  177. package/src/serve/generation-gateway.ts +88 -38
  178. package/src/serve/model-binding.ts +7 -1
  179. package/src/serve/model-host.ts +3 -4
  180. package/src/serve/preparation.ts +3 -3
  181. package/src/serve/prompt-response-trace.ts +14 -0
  182. package/src/serve/request-plan.ts +6 -3
  183. package/src/serve/request-prep.ts +40 -25
  184. package/src/serve/text-completion-stage.ts +2 -2
  185. package/src/serve/transcription-server.ts +64 -0
  186. package/src/serve/transcription-service.ts +502 -0
  187. package/src/server.ts +185 -261
  188. package/src/spec/assistant-rows.ts +93 -0
  189. package/src/spec/assistant-source.ts +28 -18
  190. package/src/spec/cache-identity.ts +22 -0
  191. package/src/spec/deepspec-rows.ts +29 -0
  192. package/src/spec/deepspec-source.ts +12 -10
  193. package/src/spec/dflash-source.ts +28 -4
  194. package/src/spec/drafter.ts +153 -142
  195. package/src/spec/dspark/deepspec-module.ts +95 -120
  196. package/src/spec/dspark/module-dflash.ts +126 -170
  197. package/src/spec/glm52-mtp-graph.ts +29 -0
  198. package/src/spec/glm52-mtp-rows.ts +132 -0
  199. package/src/spec/glm52-mtp-source.ts +54 -136
  200. package/src/spec/glm52-mtp-state.ts +33 -0
  201. package/src/spec/ngram-source.ts +115 -21
  202. package/src/spec/projected-draft-rows.ts +73 -0
  203. package/src/spec/qwen-mtp-module.ts +171 -0
  204. package/src/spec/qwen-mtp-rows.ts +230 -0
  205. package/src/spec/qwen-mtp-source.ts +84 -226
  206. package/src/spec/qwen-mtp-state.ts +30 -0
  207. package/src/spec/serve-loop.ts +73 -69
  208. package/src/spec/source.ts +93 -15
  209. package/src/spec/two-model-rows.ts +62 -0
  210. package/src/spec/two-model.ts +45 -103
  211. package/src/ssd-cache.ts +107 -36
  212. package/src/ssd-durability.ts +29 -27
  213. package/src/storage/host-buffer.ts +24 -0
  214. package/src/storage/kv-writer.ts +83 -0
  215. package/src/storage/kv-writer.worker.js +264 -0
  216. package/src/storage/retention-policy.ts +37 -0
  217. package/src/tiered-prompt-cache.ts +45 -0
  218. package/src/vision/embedder.ts +8 -0
  219. package/src/vision/prompt.ts +101 -65
  220. package/src/vision/qwen3vl-prompt.ts +101 -64
  221. package/src/vision/qwen3vl-tower.ts +11 -0
  222. package/src/vision/siglip.ts +8 -0
  223. package/src/web/app.html +12 -0
  224. package/src/web/app.js +217 -57
  225. package/src/web/src/chat.ts +4 -1
  226. package/src/web/src/composer.ts +3 -0
  227. package/src/web/src/voice.ts +141 -0
  228. package/src/web/sw.js +1 -1
  229. package/src/backends/mlx/memory-guard.ts +0 -40
  230. package/src/spec/prefix-state.ts +0 -45
  231. /package/docs/{planning → archive/planning}/release-notes-v0.3.0.md +0 -0
@@ -13,8 +13,11 @@ import * as ops from "../mlx/ops";
13
13
  import { toLogprobs } from "../sampler";
14
14
  import { Glm52DecoderLayer, Glm52Model } from "../model/glm52";
15
15
  import { MLACache } from "../model/glm52-cache";
16
- import { rmsNormF32Mlx } from "../model/glm52-mla";
17
- import type { DraftProvider, DraftSource } from "./source";
16
+ import { Glm52MtpGraph } from "./glm52-mtp-graph";
17
+ import type { DraftProvider, DraftSource, GroupedDraftProvider, TargetView, DraftRowSampling, DraftRowCheckpoint, DraftRowGroup, DraftPrefillGroup } from "./source";
18
+ import { Glm52MtpRows, type Glm52MtpRowState } from "./glm52-mtp-rows";
19
+ import { captureGlm52MtpState, restoreGlm52MtpState } from "./glm52-mtp-state";
20
+ import { applyStateChanges, disposeResources } from "../engine/resources";
18
21
 
19
22
  type Sampler = (logprobs: MlxArray, step: number) => MlxArray;
20
23
 
@@ -26,6 +29,11 @@ function lastToken(feed: readonly number[]): number {
26
29
  }
27
30
 
28
31
  export class Glm52NativeMtpProvider implements DraftProvider {
32
+ readonly grouped: GroupedDraftProvider = {
33
+ checkpointNamespace: () => "glm52-native-mtp-v1",
34
+ open: options => this.#openRows(options.target, options.sampling, options.checkpoints),
35
+ openPrefill: options => this.#openRows(options.target, null, options.checkpoints),
36
+ };
29
37
  readonly id = "glm52-native-mtp";
30
38
  readonly weightsBytes = 0;
31
39
  readonly #layer: Glm52DecoderLayer;
@@ -62,6 +70,34 @@ export class Glm52NativeMtpProvider implements DraftProvider {
62
70
  return new Glm52NativeMtpSource(this.model, this.#layer, opts.sampler);
63
71
  }
64
72
 
73
+ #openRows(target: TargetView, sampling: DraftRowSampling | null,
74
+ checkpoints: readonly (DraftRowCheckpoint | null)[]): DraftRowGroup & DraftPrefillGroup {
75
+ if (target.identity !== this.model) throw new Error("native MTP provider was opened for a different target");
76
+ const rows = new Glm52MtpRows(new Glm52MtpGraph(this.model, this.#layer), sampling);
77
+ const prepareAppend = (checkpoints: readonly (DraftRowCheckpoint | null)[]) => {
78
+ const states: (Glm52MtpRowState | null)[] = [];
79
+ try {
80
+ for (const checkpoint of checkpoints) states.push(checkpoint ? restoreGlm52MtpState(checkpoint, () => rows.makeCache()) : null);
81
+ return rows.prepareAppend(states);
82
+ } finally { disposeResources(states.flatMap(state => state ? [state.cache, state.hidden] : [])); }
83
+ };
84
+ const append = (states: readonly (DraftRowCheckpoint | null)[]) => applyStateChanges([() => prepareAppend(states)]);
85
+ try { append(checkpoints); } catch (error) { rows.dispose(); throw error; }
86
+ return {
87
+ namespace: "glm52-native-mtp-v1", prefillMode: "full", tapLayers: [],
88
+ get rowCount() { return rows.rowCount; },
89
+ append, prepareAppend, prefill: (tokens, context) => rows.prefill(tokens, context!),
90
+ materialize: rows.materialize.bind(rows), filterRows: rows.filterRows.bind(rows),
91
+ draft: rows.draft.bind(rows), commit: rows.commit.bind(rows),
92
+ capture(row) {
93
+ const state = rows.extractRow(row);
94
+ try { return captureGlm52MtpState(state); }
95
+ finally { state.cache.dispose(); state.hidden.dispose(); }
96
+ },
97
+ dispose: rows.dispose.bind(rows),
98
+ };
99
+ }
100
+
65
101
  dispose(): void {
66
102
  // The target model owns every shared/MTP weight and the expert tier.
67
103
  }
@@ -72,6 +108,7 @@ export class Glm52NativeMtpSource implements DraftSource {
72
108
  readonly pinTargetKernelFamily = true;
73
109
  readonly weightsBytes = 0;
74
110
  #cache: MLACache;
111
+ readonly #graph: Glm52MtpGraph;
75
112
  #roundStart = 0;
76
113
  #lastDraftCount = 0;
77
114
  #closed = false;
@@ -81,6 +118,7 @@ export class Glm52NativeMtpSource implements DraftSource {
81
118
  readonly layer: Glm52DecoderLayer,
82
119
  readonly sampler: Sampler,
83
120
  ) {
121
+ this.#graph = new Glm52MtpGraph(model, layer);
84
122
  this.#cache = new MLACache({
85
123
  kvLoraRank: model.glmConfig.kvLoraRank,
86
124
  ropeHeadDim: model.glmConfig.qkRopeHeadDim,
@@ -256,142 +294,22 @@ export class Glm52NativeMtpSource implements DraftSource {
256
294
  hidden: MlxArray,
257
295
  sampleStep: number,
258
296
  ): Promise<{ hidden: MlxArray; token: number }> {
259
- const config = this.model.glmConfig;
260
- const ids = ops.fromInt32([token], [1, 1]);
261
- let embedded: MlxArray | null = null;
262
- let embeddedNorm: MlxArray | null = null;
263
- let hiddenNorm: MlxArray | null = null;
264
- let joined: MlxArray | null = null;
265
- let projected: MlxArray | null = null;
266
- let output: MlxArray | null = null;
267
- let headInput: MlxArray | null = null;
268
- let logits: MlxArray | null = null;
269
- let logprobs: MlxArray | null = null;
270
- let sampled: MlxArray | null = null;
271
- try {
272
- embedded = this.model.weights.embedding(
273
- ids,
274
- "model.embed_tokens.weight",
275
- config.vocabSize,
276
- config.hiddenSize,
277
- );
278
- embeddedNorm = rmsNormF32Mlx(
279
- embedded,
280
- this.model.weights.tensor(
281
- `model.layers.${config.numHiddenLayers}.enorm.weight`,
282
- ),
283
- config.rmsNormEps,
284
- );
285
- hiddenNorm = rmsNormF32Mlx(
286
- hidden,
287
- this.model.weights.tensor(
288
- `model.layers.${config.numHiddenLayers}.hnorm.weight`,
289
- ),
290
- config.rmsNormEps,
291
- );
292
- joined = ops.concatAxis([embeddedNorm, hiddenNorm], 2);
293
- projected = this.model.weights.linear(
294
- joined,
295
- `model.layers.${config.numHiddenLayers}.eh_proj.weight`,
296
- config.hiddenSize,
297
- 2 * config.hiddenSize,
298
- );
299
- output = await this.layer.forwardAsync(projected, this.#cache, null);
300
- headInput = rmsNormF32Mlx(
301
- output,
302
- this.model.weights.tensor(
303
- `model.layers.${config.numHiddenLayers}.shared_head.norm.weight`,
304
- ),
305
- config.rmsNormEps,
306
- );
307
- logits = this.model.logitsFromHidden(headInput);
308
- // Sampler contract is [1, V] — same 3-D top-k hazard as
309
- // qwen-mtp-source #sample (2026-08-20); see the comment there.
310
- {
311
- const V = logits.shape[logits.shape.length - 1]!;
312
- const flat = ops.reshape(logits, [1, V]);
313
- logits.dispose();
314
- logits = flat;
315
- }
316
- logprobs = toLogprobs(logits);
317
- sampled = this.sampler(logprobs, sampleStep);
318
- return {
319
- hidden: ops.contiguous(output),
320
- token: ops.itemUint32(sampled),
321
- };
322
- } finally {
323
- ids.dispose();
324
- embedded?.dispose();
325
- embeddedNorm?.dispose();
326
- hiddenNorm?.dispose();
327
- joined?.dispose();
328
- projected?.dispose();
329
- output?.dispose();
330
- headInput?.dispose();
331
- logits?.dispose();
332
- logprobs?.dispose();
333
- sampled?.dispose();
334
- }
297
+ using ids = ops.fromInt32([token], [1, 1]);
298
+ using output = await this.#graph.forward(ids, hidden, this.#cache);
299
+ using logits = this.#graph.project(output);
300
+ // Sampling consumes [B,V], independently of the graph's token dimension.
301
+ using flat = ops.reshape(logits, [1, logits.shape.at(-1)!]);
302
+ using logprobs = toLogprobs(flat);
303
+ using sampled = this.sampler(logprobs, sampleStep);
304
+ return { hidden: ops.contiguous(output), token: ops.itemUint32(sampled) };
335
305
  }
336
306
 
337
- async #absorb(
338
- acceptedTokens: readonly number[],
339
- verifiedHidden: MlxArray,
340
- ): Promise<void> {
341
- const config = this.model.glmConfig;
342
- const k = acceptedTokens.length;
343
- const ids = ops.fromInt32([...acceptedTokens], [1, k]);
344
- let embedded: MlxArray | null = null;
345
- let embeddedNorm: MlxArray | null = null;
346
- let trueRows: MlxArray | null = null;
347
- let hiddenNorm: MlxArray | null = null;
348
- let joined: MlxArray | null = null;
349
- let projected: MlxArray | null = null;
350
- let output: MlxArray | null = null;
351
- try {
352
- embedded = this.model.weights.embedding(
353
- ids,
354
- "model.embed_tokens.weight",
355
- config.vocabSize,
356
- config.hiddenSize,
357
- );
358
- embeddedNorm = rmsNormF32Mlx(
359
- embedded,
360
- this.model.weights.tensor(
361
- `model.layers.${config.numHiddenLayers}.enorm.weight`,
362
- ),
363
- config.rmsNormEps,
364
- );
365
- trueRows = verifiedHidden.slice(
366
- [0, 0, 0],
367
- [1, k, config.hiddenSize],
368
- );
369
- hiddenNorm = rmsNormF32Mlx(
370
- trueRows,
371
- this.model.weights.tensor(
372
- `model.layers.${config.numHiddenLayers}.hnorm.weight`,
373
- ),
374
- config.rmsNormEps,
375
- );
376
- joined = ops.concatAxis([embeddedNorm, hiddenNorm], 2);
377
- projected = this.model.weights.linear(
378
- joined,
379
- `model.layers.${config.numHiddenLayers}.eh_proj.weight`,
380
- config.hiddenSize,
381
- 2 * config.hiddenSize,
382
- );
383
- output = await this.layer.forwardAsync(projected, this.#cache, null);
384
- output.eval();
385
- } finally {
386
- ids.dispose();
387
- embedded?.dispose();
388
- embeddedNorm?.dispose();
389
- trueRows?.dispose();
390
- hiddenNorm?.dispose();
391
- joined?.dispose();
392
- projected?.dispose();
393
- output?.dispose();
394
- }
307
+ async #absorb(acceptedTokens: readonly number[], verifiedHidden: MlxArray): Promise<void> {
308
+ const count = acceptedTokens.length;
309
+ using ids = ops.fromInt32([...acceptedTokens], [1, count]);
310
+ using trueRows = verifiedHidden.slice([0, 0, 0], [1, count, this.model.glmConfig.hiddenSize]);
311
+ using output = await this.#graph.forward(ids, trueRows, this.#cache);
312
+ output.eval();
395
313
  }
396
314
 
397
315
  #checkOpen(): void {
@@ -0,0 +1,33 @@
1
+ import type { MlxArray } from "../mlx/array";
2
+ import type { MLACache } from "../model/glm52-cache";
3
+ import { cloneAttachments } from "../backends/mlx/checkpoint-state";
4
+ import { disposeResources } from "../engine/resources";
5
+ import type { DraftRowCheckpoint } from "./source";
6
+ import type { Glm52MtpRowState } from "./glm52-mtp-rows";
7
+
8
+ export function captureGlm52MtpState(state: Glm52MtpRowState): DraftRowCheckpoint {
9
+ const held: MlxArray[] = [];
10
+ try {
11
+ if (state.cache.offset) {
12
+ const cache = state.cache.fetch(); held.push(cache.latent, cache.rope);
13
+ }
14
+ held.push(state.hidden.slice([0, 0, 0], [...state.hidden.shape]));
15
+ return { processedTokens: state.processedTokens, attachment: {
16
+ schema: "glm52-native-mtp-v1", metadata: { processedTokens: state.processedTokens, draftOffset: state.cache.offset },
17
+ tensors: held.splice(0),
18
+ } };
19
+ } finally { disposeResources(held); }
20
+ }
21
+
22
+ export function restoreGlm52MtpState(checkpoint: DraftRowCheckpoint, makeCache: () => MLACache): Glm52MtpRowState {
23
+ const { attachment, processedTokens } = checkpoint;
24
+ if (attachment.schema !== "glm52-native-mtp-v1" || attachment.metadata.processedTokens !== processedTokens)
25
+ throw new Error("invalid native GLM MTP checkpoint alignment");
26
+ const held = cloneAttachments([attachment])[0]!.tensors;
27
+ let cache: MLACache | null = makeCache();
28
+ try {
29
+ const offset = attachment.metadata.draftOffset as number;
30
+ if (offset) { cache.restoreCompressedState(held[0]!, held[1]!, null, offset); held.splice(0, 2); }
31
+ const state = { cache, hidden: held.pop()!, processedTokens }; cache = null; return state;
32
+ } finally { disposeResources([...held, ...(cache ? [cache] : [])]); }
33
+ }
@@ -38,8 +38,86 @@
38
38
  // speculation is structurally never worse than plain decode by more than the
39
39
  // JS scan (~µs against 30k-token histories).
40
40
 
41
- import { flagOn } from "../runtime-config";
42
- import type { DraftProvider, DraftSource } from "./source";
41
+ import { runtimeFlag } from "../runtime-config";
42
+ import type { MlxArray } from "../mlx/array";
43
+ import * as ops from "../mlx/ops";
44
+ import type { CheckpointAttachment } from "../backends/mlx/checkpoint-state";
45
+ import type { DraftProvider, DraftSource, DraftRowCheckpoint, DraftRowGroup, DraftPrefillGroup, GroupedDraftProvider, DraftRowConstraints } from "./source";
46
+ import { applyStateChanges } from "../engine/resources";
47
+
48
+ /** The matching policy is independent of the executor's feed convention. */
49
+ function proposeNgram(h: readonly number[], max: number, min: number, n: number): number[] {
50
+ const L = h.length;
51
+ for (let k = Math.min(max, L - 1); k >= min; k--) {
52
+ const tailAt = L - k;
53
+ search: for (let i = 0; i + k < L; i++) {
54
+ for (let j = 0; j < k; j++) if (h[i + j] !== h[tailAt + j]) continue search;
55
+ return h.slice(i + k, i + k + n);
56
+ }
57
+ }
58
+ return [];
59
+ }
60
+
61
+ const namespace = (max: number, min: number) => `ngram-history-v1:${max}:${min}`;
62
+ const captureHistory = (history: readonly number[], max: number, min: number): CheckpointAttachment => ({
63
+ schema: "ngram-history-v1", metadata: { max, min }, tensors: [ops.fromInt32([...history], [history.length])],
64
+ });
65
+ function restoreHistory(state: DraftRowCheckpoint, max: number, min: number): number[] {
66
+ const { attachment, processedTokens } = state;
67
+ if (attachment.schema !== "ngram-history-v1" || attachment.metadata.max !== max || attachment.metadata.min !== min ||
68
+ attachment.tensors.length !== 1 || attachment.tensors[0]!.shape.length !== 1 || attachment.tensors[0]!.shape[0] !== processedTokens)
69
+ throw new Error("ngram checkpoint does not match its token coverage or lookup policy");
70
+ return attachment.tensors[0]!.toIntTokens();
71
+ }
72
+
73
+ /** Each row owns only its committed token history. Verification and sampling
74
+ * remain in the shared method; target storage and scheduling are not needed. */
75
+ class NgramRows implements DraftRowGroup, DraftPrefillGroup {
76
+ readonly prefillMode = "tail-split" as const;
77
+ readonly tapLayers: readonly number[] = [];
78
+ readonly namespace: string;
79
+ #histories: number[][] = [];
80
+ #drafts: number[][] = [];
81
+ constructor(readonly max: number, readonly min: number, checkpoints: readonly (DraftRowCheckpoint | null)[],
82
+ readonly constraints?: DraftRowConstraints) {
83
+ this.namespace = namespace(max, min);
84
+ this.append(checkpoints);
85
+ }
86
+ get rowCount() { return this.#histories.length; }
87
+ prepareAppend(checkpoints: readonly (DraftRowCheckpoint | null)[]) {
88
+ let next = [...this.#histories, ...checkpoints.map(state => state ? restoreHistory(state, this.max, this.min) : [])];
89
+ return { commit: () => { const old = this.#histories; this.#histories = next; next = old; }, dispose: () => { next = []; } };
90
+ }
91
+ append(checkpoints: readonly (DraftRowCheckpoint | null)[]) { applyStateChanges([() => this.prepareAppend(checkpoints)]); }
92
+ filterRows(keep: readonly number[]) { this.#histories = keep.map(row => this.#histories[row]!); this.#drafts = []; }
93
+ prefill(tokens: MlxArray): void {
94
+ using packed = ops.contiguous(tokens);
95
+ const ids = packed.toIntTokens(), length = tokens.shape[1]!;
96
+ for (let row = 0; row < this.rowCount; row++) this.#histories[row]!.push(...ids.slice(row * length, (row + 1) * length));
97
+ }
98
+ materialize(): void {}
99
+ draft(pending: readonly number[], depth: number): number[][] | Promise<number[][]> {
100
+ if (this.constraints) {
101
+ for (const [row, history] of this.#histories.entries()) history.push(pending[row]!);
102
+ return Promise.all(this.#histories.map((_, row) => this.constraints!.propose(row, depth)))
103
+ .then(proposals => this.#drafts = proposals);
104
+ }
105
+ this.#drafts = this.#histories.map((history, row) => {
106
+ history.push(pending[row]!);
107
+ return proposeNgram(history, this.max, this.min, depth);
108
+ });
109
+ return this.#drafts;
110
+ }
111
+ commit(accepted: readonly number[]) {
112
+ for (let row = 0; row < this.rowCount; row++) this.#histories[row]!.push(...this.#drafts[row]!.slice(0, accepted[row]!));
113
+ this.#drafts = [];
114
+ }
115
+ capture(row: number): DraftRowCheckpoint {
116
+ const history = this.#histories[row]!;
117
+ return { processedTokens: history.length, attachment: captureHistory(history, this.max, this.min) };
118
+ }
119
+ dispose() { this.#histories = []; this.#drafts = []; }
120
+ }
43
121
 
44
122
  export interface NgramOptions {
45
123
  /** Longest suffix k-gram tried first (Saxena max_ngram_size). */
@@ -53,6 +131,12 @@ export class NgramProvider implements DraftProvider {
53
131
  readonly weightsBytes = 0;
54
132
  readonly max: number;
55
133
  readonly min: number;
134
+ readonly grouped: GroupedDraftProvider = {
135
+ checkpointNamespace: () => namespace(this.max, this.min),
136
+ supportsTargetAdapters: true,
137
+ open: options => new NgramRows(this.max, this.min, options.checkpoints),
138
+ openPrefill: options => new NgramRows(this.max, this.min, options.checkpoints),
139
+ };
56
140
 
57
141
  constructor(opts: NgramOptions = {}) {
58
142
  this.max = Math.max(1, opts.max ?? 3);
@@ -73,24 +157,33 @@ class NgramSource implements DraftSource {
73
157
  readonly weightsBytes = 0;
74
158
  #hist: number[] = [];
75
159
  #lastDrafts: number[] = [];
160
+ readonly #tailSplit = runtimeFlag("MLX_BUN_PREFILL_TAIL_SPLIT", true);
161
+ readonly checkpoint: NonNullable<DraftSource["checkpoint"]>;
76
162
 
77
163
  constructor(
78
164
  private readonly max: number,
79
165
  private readonly min: number,
80
- ) {}
166
+ ) {
167
+ this.checkpoint = {
168
+ namespace: namespace(max, min),
169
+ capture: () => captureHistory(this.#hist, this.max, this.min),
170
+ restore: (processedTokens, attachment) => {
171
+ this.#hist = restoreHistory({ processedTokens, attachment }, this.max, this.min);
172
+ },
173
+ };
174
+ }
81
175
 
82
176
  /** Test hook — the reconstructed prompt+emitted stream (see header). */
83
177
  get history(): readonly number[] {
84
178
  return this.#hist;
85
179
  }
86
180
 
87
- prefill(promptIds: number[]): void {
181
+ prefill(promptIds: number[], _context?: Parameters<DraftSource["prefill"]>[1], processedTokens?: number): void {
88
182
  // Mirror the serve loop's prefill shape (two-model.ts does the same): under
89
183
  // the oracle tail split the last prompt token is never prefilled — it IS
90
184
  // the first feed. Legacy shape (kill switch / 1-token prompt): full prompt,
91
185
  // and the sampled token0 arrives as the first feed.
92
- const tailSplit = flagOn("MLX_BUN_PREFILL_TAIL_SPLIT", true);
93
- const upTo = tailSplit && promptIds.length > 1 ? promptIds.length - 1 : promptIds.length;
186
+ const upTo = processedTokens ?? (this.#tailSplit && promptIds.length > 1 ? promptIds.length - 1 : promptIds.length);
94
187
  this.#hist = promptIds.slice(0, upTo);
95
188
  }
96
189
 
@@ -111,22 +204,23 @@ class NgramSource implements DraftSource {
111
204
  /** Longest-k-first, first-occurrence prompt lookup (Saxena/vLLM order):
112
205
  * find the trailing k-gram earlier in history, propose what followed it. */
113
206
  #propose(n: number): number[] {
114
- const h = this.#hist;
115
- const L = h.length;
116
- const kTop = Math.min(this.max, L - 1);
117
- for (let k = kTop; k >= this.min; k--) {
118
- const tailAt = L - k;
119
- // First occurrence, left to right; i + k < L both excludes the trailing
120
- // self-match and guarantees at least one continuation token.
121
- search: for (let i = 0; i + k < L; i++) {
122
- for (let j = 0; j < k; j++) {
123
- if (h[i + j] !== h[tailAt + j]) continue search;
124
- }
125
- return h.slice(i + k, i + k + n);
126
- }
127
- }
128
- return [];
207
+ return proposeNgram(this.#hist, this.max, this.min, n);
129
208
  }
130
209
 
131
210
  dispose(): void {}
132
211
  }
212
+
213
+ /** Constraint candidates share committed-history ownership with prompt lookup.
214
+ * They have no serial draft implementation and never perform ngram lookup. */
215
+ export function constraintDraftProvider(): { id: string; grouped: GroupedDraftProvider } {
216
+ return {
217
+ id: "grammar",
218
+ grouped: {
219
+ checkpointNamespace: () => namespace(3, 1),
220
+ supportsTargetAdapters: true,
221
+ openPrefill: options => new NgramRows(3, 1, options.checkpoints),
222
+ open: options => new NgramRows(3, 1, options.checkpoints,
223
+ options.constraints ?? { propose: async () => [] }),
224
+ },
225
+ };
226
+ }
@@ -0,0 +1,73 @@
1
+ import type { MlxArray } from "../mlx/array";
2
+ import * as ops from "../mlx/ops";
3
+ import { KVCache, type Cache } from "../model/gemma4-base";
4
+ import { MlxProjectedContextRows } from "../backends/mlx/projected-context-rows";
5
+ import { captureCacheAttachment, restoreCacheAttachment } from "../backends/mlx/cache-attachment";
6
+ import { applyStateChanges, disposeResources } from "../engine/resources";
7
+ import type { PreparedStateChange } from "../contracts/resources";
8
+ import type { DraftPrefillGroup, DraftRowGroup, DraftRowCheckpoint, GroupedDraftProvider, TargetView } from "./source";
9
+
10
+ export interface ProjectedDraftMethod {
11
+ namespace: string;
12
+ schema: string;
13
+ layers: number;
14
+ tapLayers: readonly number[];
15
+ project(hidden: MlxArray, positions: number | MlxArray): { k: MlxArray; v: MlxArray }[];
16
+ draft(context: ReturnType<MlxProjectedContextRows["readAttention"]>, pending: readonly number[], positions: number | MlxArray, depth: number): number[][];
17
+ }
18
+
19
+ /** Provider numerics compose with shared context membership and persistence. */
20
+ class ProjectedDraftRows implements DraftPrefillGroup, DraftRowGroup {
21
+ readonly prefillMode = "tail-split" as const;
22
+ readonly tapLayers: readonly number[];
23
+ readonly context: MlxProjectedContextRows;
24
+ constructor(readonly method: ProjectedDraftMethod,
25
+ checkpoints: readonly (DraftRowCheckpoint | null)[]) {
26
+ this.tapLayers = method.tapLayers;
27
+ this.context = new MlxProjectedContextRows(method.layers, { project: method.project });
28
+ try { this.append(checkpoints); } catch (error) { this.dispose(); throw error; }
29
+ }
30
+ get namespace(): string { return this.method.namespace; }
31
+ get rowCount(): number { return this.context.state.rowCount; }
32
+ prepareAppend(checkpoints: readonly (DraftRowCheckpoint | null)[]): PreparedStateChange {
33
+ const restored: Cache[][] = [];
34
+ const empty = () => Array.from({ length: this.method.layers }, () => new KVCache());
35
+ try {
36
+ for (const checkpoint of checkpoints) restored.push(checkpoint ? restoreCacheAttachment(checkpoint.attachment, empty) : empty());
37
+ return this.context.prepareAppend(restored);
38
+ } finally { disposeResources(restored.flat()); }
39
+ }
40
+ append(checkpoints: readonly (DraftRowCheckpoint | null)[]): void {
41
+ if (checkpoints.length) applyStateChanges([() => this.prepareAppend(checkpoints)]);
42
+ }
43
+ prefill(tokens: MlxArray, hidden?: MlxArray): void {
44
+ this.context.append(hidden!, Array(this.rowCount).fill(tokens.shape[1]!)); this.materialize();
45
+ }
46
+ materialize(): void { this.context.materialize(); }
47
+ filterRows(keep: readonly number[]): void { this.context.state.filterRows(keep); }
48
+ capture(row: number): DraftRowCheckpoint {
49
+ const caches = this.context.state.extractRow(row);
50
+ try {
51
+ const processedTokens = caches[0]!.offset;
52
+ return { processedTokens, attachment: captureCacheAttachment(this.method.schema, caches, { processedTokens }) };
53
+ } finally { disposeResources(caches); }
54
+ }
55
+ draft(pending: readonly number[], depth: number): number[][] {
56
+ if (!depth) return pending.map(() => []);
57
+ const positions = this.context.positions, donors = this.context.readAttention();
58
+ using offsets = positions.every(position => position === positions[0]) ? null : ops.fromInt32([...positions], [pending.length]);
59
+ try { return this.method.draft(donors, pending, offsets ?? positions[0]!, depth); }
60
+ finally { donors.dispose(); }
61
+ }
62
+ commit(accepted: readonly number[], hidden: MlxArray): void {
63
+ this.context.append(hidden, accepted.map(count => count + 1));
64
+ }
65
+ dispose(): void { this.context.dispose(); }
66
+ }
67
+
68
+ export function projectedDraftGroups(namespace: string, bind: (target: TargetView) => ProjectedDraftMethod): GroupedDraftProvider {
69
+ const open = ({ target, checkpoints }: {
70
+ target: TargetView; checkpoints: readonly (DraftRowCheckpoint | null)[];
71
+ }) => new ProjectedDraftRows(bind(target), checkpoints);
72
+ return { checkpointNamespace: () => namespace, open, openPrefill: open };
73
+ }