mlx-bun 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -32
- package/docs/README.md +58 -42
- package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +3617 -0
- package/docs/archive/investigations/openjarvis-fit.md +198 -0
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +1092 -0
- package/docs/archive/planning/release-notes-v0.4.0.md +45 -0
- package/docs/archive/planning/release-notes-v0.5.0.md +37 -0
- package/docs/design/batching.md +1535 -50
- package/docs/design/decode-speed-program.md +394 -1171
- package/docs/design/generic-model-support.md +195 -3
- package/docs/design/kv-cache.md +538 -225
- package/docs/design/speculative-decoding.md +552 -50
- package/docs/design/turboquant.md +140 -17
- package/docs/design/unified-engine-frontier-plan.md +259 -150
- package/docs/planning/PRODUCT_ROADMAP.md +21 -6
- package/docs/planning/public-readiness-handoff.md +244 -0
- package/docs/planning/public-readiness.md +149 -0
- package/docs/reference/benchmarks.md +3537 -1112
- package/docs/reference/cli.md +102 -2
- package/docs/reference/distribution.md +45 -13
- package/docs/reference/environment.md +159 -13
- package/docs/reference/library-api.md +18 -4
- package/docs/reference/memory.md +1 -1
- package/docs/reference/models.md +28 -7
- package/docs/reference/server-api.md +257 -33
- package/docs/reference/server-config.md +372 -151
- package/docs/reference/troubleshooting.md +13 -21
- package/package.json +2 -2
- package/src/audio/audiotoolbox.ts +112 -0
- package/src/audio/conformer.ts +8 -0
- package/src/audio/mic-capture.ts +98 -0
- package/src/audio/silero-vad.ts +379 -0
- package/src/audio/whisper-decode.ts +774 -0
- package/src/audio/whisper-fast.ts +732 -0
- package/src/audio/whisper-format.ts +73 -0
- package/src/audio/whisper-mel.ts +185 -0
- package/src/audio/whisper-timing.ts +274 -0
- package/src/audio/whisper-tokenizer.ts +239 -0
- package/src/audio/whisper-transcribe.ts +410 -0
- package/src/backends/mlx/assistant-target.ts +46 -0
- package/src/backends/mlx/autoregressive-draft-rows.ts +120 -0
- package/src/backends/mlx/autoregressive.ts +22 -1
- package/src/backends/mlx/batch-group.ts +464 -236
- package/src/backends/mlx/cache-attachment.ts +67 -0
- package/src/backends/mlx/cache-layout.ts +31 -0
- package/src/backends/mlx/checkpoint-state.ts +53 -0
- package/src/backends/mlx/continuation-persistence.ts +107 -0
- package/src/backends/mlx/continuation-request.ts +43 -0
- package/src/backends/mlx/continuation.ts +77 -0
- package/src/backends/mlx/draft-target.ts +7 -22
- package/src/backends/mlx/encoder-cache.ts +25 -0
- package/src/backends/mlx/fill-append.ts +21 -8
- package/src/backends/mlx/fill-group.ts +497 -0
- package/src/backends/mlx/full-kv-row-append.ts +32 -0
- package/src/backends/mlx/full-kv-row-donor.ts +54 -0
- package/src/backends/mlx/full-transitioning-kv-rows.ts +65 -0
- package/src/backends/mlx/gateway-binding.ts +122 -19
- package/src/backends/mlx/kv-codec.ts +9 -0
- package/src/backends/mlx/kv-maintenance.ts +161 -0
- package/src/backends/mlx/kv-tensor-rows.ts +212 -0
- package/src/backends/mlx/mixed-iteration.ts +62 -0
- package/src/backends/mlx/model-prompt.ts +46 -32
- package/src/backends/mlx/model-serving.ts +5 -4
- package/src/backends/mlx/paged-cache-codec.ts +27 -0
- package/src/backends/mlx/prefill-cohort.ts +100 -0
- package/src/backends/mlx/prefill-policy.ts +26 -0
- package/src/backends/mlx/prefill-rows.ts +244 -0
- package/src/backends/mlx/prefill.ts +1 -1
- package/src/backends/mlx/projected-context-rows.ts +72 -0
- package/src/backends/mlx/prompt-input.ts +29 -0
- package/src/backends/mlx/qwen-prompt-input.ts +40 -0
- package/src/backends/mlx/request-state-policy.ts +54 -0
- package/src/backends/mlx/rollback.ts +45 -4
- package/src/backends/mlx/row-sampling.ts +49 -0
- package/src/backends/mlx/serial-executor.ts +51 -73
- package/src/backends/mlx/speculative-group.ts +438 -0
- package/src/backends/mlx/speculative-round.ts +161 -0
- package/src/backends/mlx/speculative.ts +24 -7
- package/src/backends/mlx/state-rows.ts +104 -0
- package/src/backends/mlx/state-views.ts +21 -1
- package/src/backends/mlx/target-layout-capability.ts +35 -0
- package/src/backends/mlx/transitioning-kv-rows.ts +173 -0
- package/src/cli.ts +333 -36
- package/src/contracts/execution.ts +21 -0
- package/src/contracts/object-cache.ts +8 -0
- package/src/contracts/pi-provider.ts +1 -1
- package/src/contracts/pi.ts +3 -1
- package/src/contracts/prefix-cache.ts +33 -0
- package/src/contracts/resources.ts +7 -0
- package/src/contracts/sampling.ts +23 -0
- package/src/contracts/scheduling.ts +27 -0
- package/src/engine/execution-plan.ts +38 -25
- package/src/engine/execution-tasks.ts +44 -0
- package/src/engine/namespaced-cache.ts +13 -0
- package/src/engine/resources.ts +12 -1
- package/src/engine/scheduler.ts +32 -7
- package/src/fill/fill-session.ts +14 -3
- package/src/fit.ts +29 -15
- package/src/generate.ts +105 -195
- package/src/grammar.ts +13 -0
- package/src/inference/draft-acceptance.ts +41 -0
- package/src/inference/draft-output.ts +49 -0
- package/src/inference/prefill.ts +6 -0
- package/src/inference/rollback.ts +2 -2
- package/src/kv-scheme.ts +92 -12
- package/src/kv-store.ts +183 -93
- package/src/lab/README.md +5 -1
- package/src/lab/paged-kv/paged-attention.ts +117 -0
- package/src/lab/paged-kv/paged-kv-rows.ts +111 -0
- package/src/lab/paged-kv/paged-kv.ts +126 -46
- package/src/md-modules.d.ts +5 -0
- package/src/mlx/array.ts +39 -1
- package/src/mlx/ffi.ts +19 -0
- package/src/mlx/normalized-argmax.ts +116 -0
- package/src/mlx/ops.ts +97 -10
- package/src/model/artifact-identity.ts +73 -0
- package/src/model/batched-kv.ts +235 -0
- package/src/model/batched-mask.ts +12 -4
- package/src/model/batched-quant.ts +8 -2
- package/src/model/batched-quantized-kv.ts +144 -0
- package/src/model/batched-rotating-quant.ts +76 -15
- package/src/model/batched-rotating-state.ts +76 -0
- package/src/model/batched-rotating.ts +64 -22
- package/src/model/batched-row-storage.ts +52 -0
- package/src/model/batched-ssm.ts +52 -0
- package/src/model/batched-turboquant-kv.ts +94 -0
- package/src/model/compiled-decode.ts +7 -1
- package/src/model/decoded-kv-donor.ts +12 -0
- package/src/model/delayed-quantized-kv.ts +72 -0
- package/src/model/delayed-rotating-quantized-kv.ts +93 -0
- package/src/model/delayed-turboquant-kv.ts +57 -0
- package/src/model/factory.ts +26 -0
- package/src/model/fingerprint.ts +12 -0
- package/src/model/full-prefill-padding.ts +56 -0
- package/src/model/full-prefill-row.ts +105 -0
- package/src/model/gemma4-base.ts +183 -82
- package/src/model/gemma4.ts +116 -28
- package/src/model/generated/gemma4-12b.ts +4 -1
- package/src/model/generated/gemma4-26b.ts +4 -1
- package/src/model/generated/gemma4-e4b.ts +4 -1
- package/src/model/glm52-cache.ts +80 -4
- package/src/model/glm52-mla.ts +12 -6
- package/src/model/glm52.ts +19 -40
- package/src/model/kv-attention-view.ts +70 -0
- package/src/model/minicpm5.ts +11 -7
- package/src/model/profile.ts +29 -4
- package/src/model/quantized-append-attention.ts +31 -0
- package/src/model/qwen3-delta-state.ts +74 -0
- package/src/model/qwen3-delta.ts +59 -36
- package/src/model/qwen3-mrope.ts +7 -9
- package/src/model/qwen3_5.ts +177 -75
- package/src/model/rotating-kv-donor.ts +35 -0
- package/src/model/rotating-kv-layout.ts +262 -0
- package/src/model/rotating-row-transaction.ts +35 -0
- package/src/model/speculative-rotating-kv.ts +101 -0
- package/src/model/ssm-prefill-padding.ts +39 -0
- package/src/model/support.ts +8 -0
- package/src/model/token-groups.ts +66 -0
- package/src/model/trellis-linear.ts +4 -4
- package/src/model/turboquant-codec.ts +89 -0
- package/src/model/universal/dense.ts +14 -13
- package/src/model/whisper.ts +435 -0
- package/src/native/mic_capture.swift +92 -0
- package/src/native-pack.ts +6 -5
- package/src/pi-web.ts +9 -1
- package/src/preflight.ts +31 -15
- package/src/prompt-cache.ts +306 -86
- package/src/runtime-config.ts +1 -5
- package/src/safetensors.ts +9 -2
- package/src/sampler.ts +37 -25
- package/src/serve/audio-routes.ts +323 -0
- package/src/serve/chat-request.ts +2 -0
- package/src/serve/chat-stage.ts +19 -14
- package/src/serve/completion-executor.ts +9 -1
- package/src/serve/discovery-routes.ts +17 -1
- package/src/serve/generated-token-history.ts +57 -0
- package/src/serve/generation-gateway.ts +88 -38
- package/src/serve/model-binding.ts +7 -1
- package/src/serve/model-host.ts +3 -4
- package/src/serve/preparation.ts +3 -3
- package/src/serve/prompt-response-trace.ts +14 -0
- package/src/serve/request-plan.ts +6 -3
- package/src/serve/request-prep.ts +40 -25
- package/src/serve/text-completion-stage.ts +2 -2
- package/src/serve/transcription-server.ts +64 -0
- package/src/serve/transcription-service.ts +502 -0
- package/src/server.ts +185 -261
- package/src/spec/assistant-rows.ts +93 -0
- package/src/spec/assistant-source.ts +28 -18
- package/src/spec/cache-identity.ts +22 -0
- package/src/spec/deepspec-rows.ts +29 -0
- package/src/spec/deepspec-source.ts +12 -10
- package/src/spec/dflash-source.ts +28 -4
- package/src/spec/drafter.ts +153 -142
- package/src/spec/dspark/deepspec-module.ts +95 -120
- package/src/spec/dspark/module-dflash.ts +126 -170
- package/src/spec/glm52-mtp-graph.ts +29 -0
- package/src/spec/glm52-mtp-rows.ts +132 -0
- package/src/spec/glm52-mtp-source.ts +54 -136
- package/src/spec/glm52-mtp-state.ts +33 -0
- package/src/spec/ngram-source.ts +115 -21
- package/src/spec/projected-draft-rows.ts +73 -0
- package/src/spec/qwen-mtp-module.ts +171 -0
- package/src/spec/qwen-mtp-rows.ts +230 -0
- package/src/spec/qwen-mtp-source.ts +84 -226
- package/src/spec/qwen-mtp-state.ts +30 -0
- package/src/spec/serve-loop.ts +73 -69
- package/src/spec/source.ts +93 -15
- package/src/spec/two-model-rows.ts +62 -0
- package/src/spec/two-model.ts +45 -103
- package/src/ssd-cache.ts +107 -36
- package/src/ssd-durability.ts +29 -27
- package/src/storage/host-buffer.ts +24 -0
- package/src/storage/kv-writer.ts +83 -0
- package/src/storage/kv-writer.worker.js +264 -0
- package/src/storage/retention-policy.ts +37 -0
- package/src/tiered-prompt-cache.ts +45 -0
- package/src/vision/embedder.ts +8 -0
- package/src/vision/prompt.ts +101 -65
- package/src/vision/qwen3vl-prompt.ts +101 -64
- package/src/vision/qwen3vl-tower.ts +11 -0
- package/src/vision/siglip.ts +8 -0
- package/src/web/app.html +12 -0
- package/src/web/app.js +217 -57
- package/src/web/src/chat.ts +4 -1
- package/src/web/src/composer.ts +3 -0
- package/src/web/src/voice.ts +141 -0
- package/src/web/sw.js +1 -1
- package/src/backends/mlx/memory-guard.ts +0 -40
- package/src/spec/prefix-state.ts +0 -45
- /package/docs/{planning → archive/planning}/release-notes-v0.3.0.md +0 -0
|
@@ -13,8 +13,11 @@ import * as ops from "../mlx/ops";
|
|
|
13
13
|
import { toLogprobs } from "../sampler";
|
|
14
14
|
import { Glm52DecoderLayer, Glm52Model } from "../model/glm52";
|
|
15
15
|
import { MLACache } from "../model/glm52-cache";
|
|
16
|
-
import {
|
|
17
|
-
import type { DraftProvider, DraftSource } from "./source";
|
|
16
|
+
import { Glm52MtpGraph } from "./glm52-mtp-graph";
|
|
17
|
+
import type { DraftProvider, DraftSource, GroupedDraftProvider, TargetView, DraftRowSampling, DraftRowCheckpoint, DraftRowGroup, DraftPrefillGroup } from "./source";
|
|
18
|
+
import { Glm52MtpRows, type Glm52MtpRowState } from "./glm52-mtp-rows";
|
|
19
|
+
import { captureGlm52MtpState, restoreGlm52MtpState } from "./glm52-mtp-state";
|
|
20
|
+
import { applyStateChanges, disposeResources } from "../engine/resources";
|
|
18
21
|
|
|
19
22
|
type Sampler = (logprobs: MlxArray, step: number) => MlxArray;
|
|
20
23
|
|
|
@@ -26,6 +29,11 @@ function lastToken(feed: readonly number[]): number {
|
|
|
26
29
|
}
|
|
27
30
|
|
|
28
31
|
export class Glm52NativeMtpProvider implements DraftProvider {
|
|
32
|
+
readonly grouped: GroupedDraftProvider = {
|
|
33
|
+
checkpointNamespace: () => "glm52-native-mtp-v1",
|
|
34
|
+
open: options => this.#openRows(options.target, options.sampling, options.checkpoints),
|
|
35
|
+
openPrefill: options => this.#openRows(options.target, null, options.checkpoints),
|
|
36
|
+
};
|
|
29
37
|
readonly id = "glm52-native-mtp";
|
|
30
38
|
readonly weightsBytes = 0;
|
|
31
39
|
readonly #layer: Glm52DecoderLayer;
|
|
@@ -62,6 +70,34 @@ export class Glm52NativeMtpProvider implements DraftProvider {
|
|
|
62
70
|
return new Glm52NativeMtpSource(this.model, this.#layer, opts.sampler);
|
|
63
71
|
}
|
|
64
72
|
|
|
73
|
+
#openRows(target: TargetView, sampling: DraftRowSampling | null,
|
|
74
|
+
checkpoints: readonly (DraftRowCheckpoint | null)[]): DraftRowGroup & DraftPrefillGroup {
|
|
75
|
+
if (target.identity !== this.model) throw new Error("native MTP provider was opened for a different target");
|
|
76
|
+
const rows = new Glm52MtpRows(new Glm52MtpGraph(this.model, this.#layer), sampling);
|
|
77
|
+
const prepareAppend = (checkpoints: readonly (DraftRowCheckpoint | null)[]) => {
|
|
78
|
+
const states: (Glm52MtpRowState | null)[] = [];
|
|
79
|
+
try {
|
|
80
|
+
for (const checkpoint of checkpoints) states.push(checkpoint ? restoreGlm52MtpState(checkpoint, () => rows.makeCache()) : null);
|
|
81
|
+
return rows.prepareAppend(states);
|
|
82
|
+
} finally { disposeResources(states.flatMap(state => state ? [state.cache, state.hidden] : [])); }
|
|
83
|
+
};
|
|
84
|
+
const append = (states: readonly (DraftRowCheckpoint | null)[]) => applyStateChanges([() => prepareAppend(states)]);
|
|
85
|
+
try { append(checkpoints); } catch (error) { rows.dispose(); throw error; }
|
|
86
|
+
return {
|
|
87
|
+
namespace: "glm52-native-mtp-v1", prefillMode: "full", tapLayers: [],
|
|
88
|
+
get rowCount() { return rows.rowCount; },
|
|
89
|
+
append, prepareAppend, prefill: (tokens, context) => rows.prefill(tokens, context!),
|
|
90
|
+
materialize: rows.materialize.bind(rows), filterRows: rows.filterRows.bind(rows),
|
|
91
|
+
draft: rows.draft.bind(rows), commit: rows.commit.bind(rows),
|
|
92
|
+
capture(row) {
|
|
93
|
+
const state = rows.extractRow(row);
|
|
94
|
+
try { return captureGlm52MtpState(state); }
|
|
95
|
+
finally { state.cache.dispose(); state.hidden.dispose(); }
|
|
96
|
+
},
|
|
97
|
+
dispose: rows.dispose.bind(rows),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
65
101
|
dispose(): void {
|
|
66
102
|
// The target model owns every shared/MTP weight and the expert tier.
|
|
67
103
|
}
|
|
@@ -72,6 +108,7 @@ export class Glm52NativeMtpSource implements DraftSource {
|
|
|
72
108
|
readonly pinTargetKernelFamily = true;
|
|
73
109
|
readonly weightsBytes = 0;
|
|
74
110
|
#cache: MLACache;
|
|
111
|
+
readonly #graph: Glm52MtpGraph;
|
|
75
112
|
#roundStart = 0;
|
|
76
113
|
#lastDraftCount = 0;
|
|
77
114
|
#closed = false;
|
|
@@ -81,6 +118,7 @@ export class Glm52NativeMtpSource implements DraftSource {
|
|
|
81
118
|
readonly layer: Glm52DecoderLayer,
|
|
82
119
|
readonly sampler: Sampler,
|
|
83
120
|
) {
|
|
121
|
+
this.#graph = new Glm52MtpGraph(model, layer);
|
|
84
122
|
this.#cache = new MLACache({
|
|
85
123
|
kvLoraRank: model.glmConfig.kvLoraRank,
|
|
86
124
|
ropeHeadDim: model.glmConfig.qkRopeHeadDim,
|
|
@@ -256,142 +294,22 @@ export class Glm52NativeMtpSource implements DraftSource {
|
|
|
256
294
|
hidden: MlxArray,
|
|
257
295
|
sampleStep: number,
|
|
258
296
|
): Promise<{ hidden: MlxArray; token: number }> {
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
let headInput: MlxArray | null = null;
|
|
268
|
-
let logits: MlxArray | null = null;
|
|
269
|
-
let logprobs: MlxArray | null = null;
|
|
270
|
-
let sampled: MlxArray | null = null;
|
|
271
|
-
try {
|
|
272
|
-
embedded = this.model.weights.embedding(
|
|
273
|
-
ids,
|
|
274
|
-
"model.embed_tokens.weight",
|
|
275
|
-
config.vocabSize,
|
|
276
|
-
config.hiddenSize,
|
|
277
|
-
);
|
|
278
|
-
embeddedNorm = rmsNormF32Mlx(
|
|
279
|
-
embedded,
|
|
280
|
-
this.model.weights.tensor(
|
|
281
|
-
`model.layers.${config.numHiddenLayers}.enorm.weight`,
|
|
282
|
-
),
|
|
283
|
-
config.rmsNormEps,
|
|
284
|
-
);
|
|
285
|
-
hiddenNorm = rmsNormF32Mlx(
|
|
286
|
-
hidden,
|
|
287
|
-
this.model.weights.tensor(
|
|
288
|
-
`model.layers.${config.numHiddenLayers}.hnorm.weight`,
|
|
289
|
-
),
|
|
290
|
-
config.rmsNormEps,
|
|
291
|
-
);
|
|
292
|
-
joined = ops.concatAxis([embeddedNorm, hiddenNorm], 2);
|
|
293
|
-
projected = this.model.weights.linear(
|
|
294
|
-
joined,
|
|
295
|
-
`model.layers.${config.numHiddenLayers}.eh_proj.weight`,
|
|
296
|
-
config.hiddenSize,
|
|
297
|
-
2 * config.hiddenSize,
|
|
298
|
-
);
|
|
299
|
-
output = await this.layer.forwardAsync(projected, this.#cache, null);
|
|
300
|
-
headInput = rmsNormF32Mlx(
|
|
301
|
-
output,
|
|
302
|
-
this.model.weights.tensor(
|
|
303
|
-
`model.layers.${config.numHiddenLayers}.shared_head.norm.weight`,
|
|
304
|
-
),
|
|
305
|
-
config.rmsNormEps,
|
|
306
|
-
);
|
|
307
|
-
logits = this.model.logitsFromHidden(headInput);
|
|
308
|
-
// Sampler contract is [1, V] — same 3-D top-k hazard as
|
|
309
|
-
// qwen-mtp-source #sample (2026-08-20); see the comment there.
|
|
310
|
-
{
|
|
311
|
-
const V = logits.shape[logits.shape.length - 1]!;
|
|
312
|
-
const flat = ops.reshape(logits, [1, V]);
|
|
313
|
-
logits.dispose();
|
|
314
|
-
logits = flat;
|
|
315
|
-
}
|
|
316
|
-
logprobs = toLogprobs(logits);
|
|
317
|
-
sampled = this.sampler(logprobs, sampleStep);
|
|
318
|
-
return {
|
|
319
|
-
hidden: ops.contiguous(output),
|
|
320
|
-
token: ops.itemUint32(sampled),
|
|
321
|
-
};
|
|
322
|
-
} finally {
|
|
323
|
-
ids.dispose();
|
|
324
|
-
embedded?.dispose();
|
|
325
|
-
embeddedNorm?.dispose();
|
|
326
|
-
hiddenNorm?.dispose();
|
|
327
|
-
joined?.dispose();
|
|
328
|
-
projected?.dispose();
|
|
329
|
-
output?.dispose();
|
|
330
|
-
headInput?.dispose();
|
|
331
|
-
logits?.dispose();
|
|
332
|
-
logprobs?.dispose();
|
|
333
|
-
sampled?.dispose();
|
|
334
|
-
}
|
|
297
|
+
using ids = ops.fromInt32([token], [1, 1]);
|
|
298
|
+
using output = await this.#graph.forward(ids, hidden, this.#cache);
|
|
299
|
+
using logits = this.#graph.project(output);
|
|
300
|
+
// Sampling consumes [B,V], independently of the graph's token dimension.
|
|
301
|
+
using flat = ops.reshape(logits, [1, logits.shape.at(-1)!]);
|
|
302
|
+
using logprobs = toLogprobs(flat);
|
|
303
|
+
using sampled = this.sampler(logprobs, sampleStep);
|
|
304
|
+
return { hidden: ops.contiguous(output), token: ops.itemUint32(sampled) };
|
|
335
305
|
}
|
|
336
306
|
|
|
337
|
-
async #absorb(
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
const ids = ops.fromInt32([...acceptedTokens], [1, k]);
|
|
344
|
-
let embedded: MlxArray | null = null;
|
|
345
|
-
let embeddedNorm: MlxArray | null = null;
|
|
346
|
-
let trueRows: MlxArray | null = null;
|
|
347
|
-
let hiddenNorm: MlxArray | null = null;
|
|
348
|
-
let joined: MlxArray | null = null;
|
|
349
|
-
let projected: MlxArray | null = null;
|
|
350
|
-
let output: MlxArray | null = null;
|
|
351
|
-
try {
|
|
352
|
-
embedded = this.model.weights.embedding(
|
|
353
|
-
ids,
|
|
354
|
-
"model.embed_tokens.weight",
|
|
355
|
-
config.vocabSize,
|
|
356
|
-
config.hiddenSize,
|
|
357
|
-
);
|
|
358
|
-
embeddedNorm = rmsNormF32Mlx(
|
|
359
|
-
embedded,
|
|
360
|
-
this.model.weights.tensor(
|
|
361
|
-
`model.layers.${config.numHiddenLayers}.enorm.weight`,
|
|
362
|
-
),
|
|
363
|
-
config.rmsNormEps,
|
|
364
|
-
);
|
|
365
|
-
trueRows = verifiedHidden.slice(
|
|
366
|
-
[0, 0, 0],
|
|
367
|
-
[1, k, config.hiddenSize],
|
|
368
|
-
);
|
|
369
|
-
hiddenNorm = rmsNormF32Mlx(
|
|
370
|
-
trueRows,
|
|
371
|
-
this.model.weights.tensor(
|
|
372
|
-
`model.layers.${config.numHiddenLayers}.hnorm.weight`,
|
|
373
|
-
),
|
|
374
|
-
config.rmsNormEps,
|
|
375
|
-
);
|
|
376
|
-
joined = ops.concatAxis([embeddedNorm, hiddenNorm], 2);
|
|
377
|
-
projected = this.model.weights.linear(
|
|
378
|
-
joined,
|
|
379
|
-
`model.layers.${config.numHiddenLayers}.eh_proj.weight`,
|
|
380
|
-
config.hiddenSize,
|
|
381
|
-
2 * config.hiddenSize,
|
|
382
|
-
);
|
|
383
|
-
output = await this.layer.forwardAsync(projected, this.#cache, null);
|
|
384
|
-
output.eval();
|
|
385
|
-
} finally {
|
|
386
|
-
ids.dispose();
|
|
387
|
-
embedded?.dispose();
|
|
388
|
-
embeddedNorm?.dispose();
|
|
389
|
-
trueRows?.dispose();
|
|
390
|
-
hiddenNorm?.dispose();
|
|
391
|
-
joined?.dispose();
|
|
392
|
-
projected?.dispose();
|
|
393
|
-
output?.dispose();
|
|
394
|
-
}
|
|
307
|
+
async #absorb(acceptedTokens: readonly number[], verifiedHidden: MlxArray): Promise<void> {
|
|
308
|
+
const count = acceptedTokens.length;
|
|
309
|
+
using ids = ops.fromInt32([...acceptedTokens], [1, count]);
|
|
310
|
+
using trueRows = verifiedHidden.slice([0, 0, 0], [1, count, this.model.glmConfig.hiddenSize]);
|
|
311
|
+
using output = await this.#graph.forward(ids, trueRows, this.#cache);
|
|
312
|
+
output.eval();
|
|
395
313
|
}
|
|
396
314
|
|
|
397
315
|
#checkOpen(): void {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { MlxArray } from "../mlx/array";
|
|
2
|
+
import type { MLACache } from "../model/glm52-cache";
|
|
3
|
+
import { cloneAttachments } from "../backends/mlx/checkpoint-state";
|
|
4
|
+
import { disposeResources } from "../engine/resources";
|
|
5
|
+
import type { DraftRowCheckpoint } from "./source";
|
|
6
|
+
import type { Glm52MtpRowState } from "./glm52-mtp-rows";
|
|
7
|
+
|
|
8
|
+
export function captureGlm52MtpState(state: Glm52MtpRowState): DraftRowCheckpoint {
|
|
9
|
+
const held: MlxArray[] = [];
|
|
10
|
+
try {
|
|
11
|
+
if (state.cache.offset) {
|
|
12
|
+
const cache = state.cache.fetch(); held.push(cache.latent, cache.rope);
|
|
13
|
+
}
|
|
14
|
+
held.push(state.hidden.slice([0, 0, 0], [...state.hidden.shape]));
|
|
15
|
+
return { processedTokens: state.processedTokens, attachment: {
|
|
16
|
+
schema: "glm52-native-mtp-v1", metadata: { processedTokens: state.processedTokens, draftOffset: state.cache.offset },
|
|
17
|
+
tensors: held.splice(0),
|
|
18
|
+
} };
|
|
19
|
+
} finally { disposeResources(held); }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function restoreGlm52MtpState(checkpoint: DraftRowCheckpoint, makeCache: () => MLACache): Glm52MtpRowState {
|
|
23
|
+
const { attachment, processedTokens } = checkpoint;
|
|
24
|
+
if (attachment.schema !== "glm52-native-mtp-v1" || attachment.metadata.processedTokens !== processedTokens)
|
|
25
|
+
throw new Error("invalid native GLM MTP checkpoint alignment");
|
|
26
|
+
const held = cloneAttachments([attachment])[0]!.tensors;
|
|
27
|
+
let cache: MLACache | null = makeCache();
|
|
28
|
+
try {
|
|
29
|
+
const offset = attachment.metadata.draftOffset as number;
|
|
30
|
+
if (offset) { cache.restoreCompressedState(held[0]!, held[1]!, null, offset); held.splice(0, 2); }
|
|
31
|
+
const state = { cache, hidden: held.pop()!, processedTokens }; cache = null; return state;
|
|
32
|
+
} finally { disposeResources([...held, ...(cache ? [cache] : [])]); }
|
|
33
|
+
}
|
package/src/spec/ngram-source.ts
CHANGED
|
@@ -38,8 +38,86 @@
|
|
|
38
38
|
// speculation is structurally never worse than plain decode by more than the
|
|
39
39
|
// JS scan (~µs against 30k-token histories).
|
|
40
40
|
|
|
41
|
-
import {
|
|
42
|
-
import type {
|
|
41
|
+
import { runtimeFlag } from "../runtime-config";
|
|
42
|
+
import type { MlxArray } from "../mlx/array";
|
|
43
|
+
import * as ops from "../mlx/ops";
|
|
44
|
+
import type { CheckpointAttachment } from "../backends/mlx/checkpoint-state";
|
|
45
|
+
import type { DraftProvider, DraftSource, DraftRowCheckpoint, DraftRowGroup, DraftPrefillGroup, GroupedDraftProvider, DraftRowConstraints } from "./source";
|
|
46
|
+
import { applyStateChanges } from "../engine/resources";
|
|
47
|
+
|
|
48
|
+
/** The matching policy is independent of the executor's feed convention. */
|
|
49
|
+
function proposeNgram(h: readonly number[], max: number, min: number, n: number): number[] {
|
|
50
|
+
const L = h.length;
|
|
51
|
+
for (let k = Math.min(max, L - 1); k >= min; k--) {
|
|
52
|
+
const tailAt = L - k;
|
|
53
|
+
search: for (let i = 0; i + k < L; i++) {
|
|
54
|
+
for (let j = 0; j < k; j++) if (h[i + j] !== h[tailAt + j]) continue search;
|
|
55
|
+
return h.slice(i + k, i + k + n);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const namespace = (max: number, min: number) => `ngram-history-v1:${max}:${min}`;
|
|
62
|
+
const captureHistory = (history: readonly number[], max: number, min: number): CheckpointAttachment => ({
|
|
63
|
+
schema: "ngram-history-v1", metadata: { max, min }, tensors: [ops.fromInt32([...history], [history.length])],
|
|
64
|
+
});
|
|
65
|
+
function restoreHistory(state: DraftRowCheckpoint, max: number, min: number): number[] {
|
|
66
|
+
const { attachment, processedTokens } = state;
|
|
67
|
+
if (attachment.schema !== "ngram-history-v1" || attachment.metadata.max !== max || attachment.metadata.min !== min ||
|
|
68
|
+
attachment.tensors.length !== 1 || attachment.tensors[0]!.shape.length !== 1 || attachment.tensors[0]!.shape[0] !== processedTokens)
|
|
69
|
+
throw new Error("ngram checkpoint does not match its token coverage or lookup policy");
|
|
70
|
+
return attachment.tensors[0]!.toIntTokens();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Each row owns only its committed token history. Verification and sampling
|
|
74
|
+
* remain in the shared method; target storage and scheduling are not needed. */
|
|
75
|
+
class NgramRows implements DraftRowGroup, DraftPrefillGroup {
|
|
76
|
+
readonly prefillMode = "tail-split" as const;
|
|
77
|
+
readonly tapLayers: readonly number[] = [];
|
|
78
|
+
readonly namespace: string;
|
|
79
|
+
#histories: number[][] = [];
|
|
80
|
+
#drafts: number[][] = [];
|
|
81
|
+
constructor(readonly max: number, readonly min: number, checkpoints: readonly (DraftRowCheckpoint | null)[],
|
|
82
|
+
readonly constraints?: DraftRowConstraints) {
|
|
83
|
+
this.namespace = namespace(max, min);
|
|
84
|
+
this.append(checkpoints);
|
|
85
|
+
}
|
|
86
|
+
get rowCount() { return this.#histories.length; }
|
|
87
|
+
prepareAppend(checkpoints: readonly (DraftRowCheckpoint | null)[]) {
|
|
88
|
+
let next = [...this.#histories, ...checkpoints.map(state => state ? restoreHistory(state, this.max, this.min) : [])];
|
|
89
|
+
return { commit: () => { const old = this.#histories; this.#histories = next; next = old; }, dispose: () => { next = []; } };
|
|
90
|
+
}
|
|
91
|
+
append(checkpoints: readonly (DraftRowCheckpoint | null)[]) { applyStateChanges([() => this.prepareAppend(checkpoints)]); }
|
|
92
|
+
filterRows(keep: readonly number[]) { this.#histories = keep.map(row => this.#histories[row]!); this.#drafts = []; }
|
|
93
|
+
prefill(tokens: MlxArray): void {
|
|
94
|
+
using packed = ops.contiguous(tokens);
|
|
95
|
+
const ids = packed.toIntTokens(), length = tokens.shape[1]!;
|
|
96
|
+
for (let row = 0; row < this.rowCount; row++) this.#histories[row]!.push(...ids.slice(row * length, (row + 1) * length));
|
|
97
|
+
}
|
|
98
|
+
materialize(): void {}
|
|
99
|
+
draft(pending: readonly number[], depth: number): number[][] | Promise<number[][]> {
|
|
100
|
+
if (this.constraints) {
|
|
101
|
+
for (const [row, history] of this.#histories.entries()) history.push(pending[row]!);
|
|
102
|
+
return Promise.all(this.#histories.map((_, row) => this.constraints!.propose(row, depth)))
|
|
103
|
+
.then(proposals => this.#drafts = proposals);
|
|
104
|
+
}
|
|
105
|
+
this.#drafts = this.#histories.map((history, row) => {
|
|
106
|
+
history.push(pending[row]!);
|
|
107
|
+
return proposeNgram(history, this.max, this.min, depth);
|
|
108
|
+
});
|
|
109
|
+
return this.#drafts;
|
|
110
|
+
}
|
|
111
|
+
commit(accepted: readonly number[]) {
|
|
112
|
+
for (let row = 0; row < this.rowCount; row++) this.#histories[row]!.push(...this.#drafts[row]!.slice(0, accepted[row]!));
|
|
113
|
+
this.#drafts = [];
|
|
114
|
+
}
|
|
115
|
+
capture(row: number): DraftRowCheckpoint {
|
|
116
|
+
const history = this.#histories[row]!;
|
|
117
|
+
return { processedTokens: history.length, attachment: captureHistory(history, this.max, this.min) };
|
|
118
|
+
}
|
|
119
|
+
dispose() { this.#histories = []; this.#drafts = []; }
|
|
120
|
+
}
|
|
43
121
|
|
|
44
122
|
export interface NgramOptions {
|
|
45
123
|
/** Longest suffix k-gram tried first (Saxena max_ngram_size). */
|
|
@@ -53,6 +131,12 @@ export class NgramProvider implements DraftProvider {
|
|
|
53
131
|
readonly weightsBytes = 0;
|
|
54
132
|
readonly max: number;
|
|
55
133
|
readonly min: number;
|
|
134
|
+
readonly grouped: GroupedDraftProvider = {
|
|
135
|
+
checkpointNamespace: () => namespace(this.max, this.min),
|
|
136
|
+
supportsTargetAdapters: true,
|
|
137
|
+
open: options => new NgramRows(this.max, this.min, options.checkpoints),
|
|
138
|
+
openPrefill: options => new NgramRows(this.max, this.min, options.checkpoints),
|
|
139
|
+
};
|
|
56
140
|
|
|
57
141
|
constructor(opts: NgramOptions = {}) {
|
|
58
142
|
this.max = Math.max(1, opts.max ?? 3);
|
|
@@ -73,24 +157,33 @@ class NgramSource implements DraftSource {
|
|
|
73
157
|
readonly weightsBytes = 0;
|
|
74
158
|
#hist: number[] = [];
|
|
75
159
|
#lastDrafts: number[] = [];
|
|
160
|
+
readonly #tailSplit = runtimeFlag("MLX_BUN_PREFILL_TAIL_SPLIT", true);
|
|
161
|
+
readonly checkpoint: NonNullable<DraftSource["checkpoint"]>;
|
|
76
162
|
|
|
77
163
|
constructor(
|
|
78
164
|
private readonly max: number,
|
|
79
165
|
private readonly min: number,
|
|
80
|
-
) {
|
|
166
|
+
) {
|
|
167
|
+
this.checkpoint = {
|
|
168
|
+
namespace: namespace(max, min),
|
|
169
|
+
capture: () => captureHistory(this.#hist, this.max, this.min),
|
|
170
|
+
restore: (processedTokens, attachment) => {
|
|
171
|
+
this.#hist = restoreHistory({ processedTokens, attachment }, this.max, this.min);
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
}
|
|
81
175
|
|
|
82
176
|
/** Test hook — the reconstructed prompt+emitted stream (see header). */
|
|
83
177
|
get history(): readonly number[] {
|
|
84
178
|
return this.#hist;
|
|
85
179
|
}
|
|
86
180
|
|
|
87
|
-
prefill(promptIds: number[]): void {
|
|
181
|
+
prefill(promptIds: number[], _context?: Parameters<DraftSource["prefill"]>[1], processedTokens?: number): void {
|
|
88
182
|
// Mirror the serve loop's prefill shape (two-model.ts does the same): under
|
|
89
183
|
// the oracle tail split the last prompt token is never prefilled — it IS
|
|
90
184
|
// the first feed. Legacy shape (kill switch / 1-token prompt): full prompt,
|
|
91
185
|
// and the sampled token0 arrives as the first feed.
|
|
92
|
-
const
|
|
93
|
-
const upTo = tailSplit && promptIds.length > 1 ? promptIds.length - 1 : promptIds.length;
|
|
186
|
+
const upTo = processedTokens ?? (this.#tailSplit && promptIds.length > 1 ? promptIds.length - 1 : promptIds.length);
|
|
94
187
|
this.#hist = promptIds.slice(0, upTo);
|
|
95
188
|
}
|
|
96
189
|
|
|
@@ -111,22 +204,23 @@ class NgramSource implements DraftSource {
|
|
|
111
204
|
/** Longest-k-first, first-occurrence prompt lookup (Saxena/vLLM order):
|
|
112
205
|
* find the trailing k-gram earlier in history, propose what followed it. */
|
|
113
206
|
#propose(n: number): number[] {
|
|
114
|
-
|
|
115
|
-
const L = h.length;
|
|
116
|
-
const kTop = Math.min(this.max, L - 1);
|
|
117
|
-
for (let k = kTop; k >= this.min; k--) {
|
|
118
|
-
const tailAt = L - k;
|
|
119
|
-
// First occurrence, left to right; i + k < L both excludes the trailing
|
|
120
|
-
// self-match and guarantees at least one continuation token.
|
|
121
|
-
search: for (let i = 0; i + k < L; i++) {
|
|
122
|
-
for (let j = 0; j < k; j++) {
|
|
123
|
-
if (h[i + j] !== h[tailAt + j]) continue search;
|
|
124
|
-
}
|
|
125
|
-
return h.slice(i + k, i + k + n);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
return [];
|
|
207
|
+
return proposeNgram(this.#hist, this.max, this.min, n);
|
|
129
208
|
}
|
|
130
209
|
|
|
131
210
|
dispose(): void {}
|
|
132
211
|
}
|
|
212
|
+
|
|
213
|
+
/** Constraint candidates share committed-history ownership with prompt lookup.
|
|
214
|
+
* They have no serial draft implementation and never perform ngram lookup. */
|
|
215
|
+
export function constraintDraftProvider(): { id: string; grouped: GroupedDraftProvider } {
|
|
216
|
+
return {
|
|
217
|
+
id: "grammar",
|
|
218
|
+
grouped: {
|
|
219
|
+
checkpointNamespace: () => namespace(3, 1),
|
|
220
|
+
supportsTargetAdapters: true,
|
|
221
|
+
openPrefill: options => new NgramRows(3, 1, options.checkpoints),
|
|
222
|
+
open: options => new NgramRows(3, 1, options.checkpoints,
|
|
223
|
+
options.constraints ?? { propose: async () => [] }),
|
|
224
|
+
},
|
|
225
|
+
};
|
|
226
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { MlxArray } from "../mlx/array";
|
|
2
|
+
import * as ops from "../mlx/ops";
|
|
3
|
+
import { KVCache, type Cache } from "../model/gemma4-base";
|
|
4
|
+
import { MlxProjectedContextRows } from "../backends/mlx/projected-context-rows";
|
|
5
|
+
import { captureCacheAttachment, restoreCacheAttachment } from "../backends/mlx/cache-attachment";
|
|
6
|
+
import { applyStateChanges, disposeResources } from "../engine/resources";
|
|
7
|
+
import type { PreparedStateChange } from "../contracts/resources";
|
|
8
|
+
import type { DraftPrefillGroup, DraftRowGroup, DraftRowCheckpoint, GroupedDraftProvider, TargetView } from "./source";
|
|
9
|
+
|
|
10
|
+
export interface ProjectedDraftMethod {
|
|
11
|
+
namespace: string;
|
|
12
|
+
schema: string;
|
|
13
|
+
layers: number;
|
|
14
|
+
tapLayers: readonly number[];
|
|
15
|
+
project(hidden: MlxArray, positions: number | MlxArray): { k: MlxArray; v: MlxArray }[];
|
|
16
|
+
draft(context: ReturnType<MlxProjectedContextRows["readAttention"]>, pending: readonly number[], positions: number | MlxArray, depth: number): number[][];
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Provider numerics compose with shared context membership and persistence. */
|
|
20
|
+
class ProjectedDraftRows implements DraftPrefillGroup, DraftRowGroup {
|
|
21
|
+
readonly prefillMode = "tail-split" as const;
|
|
22
|
+
readonly tapLayers: readonly number[];
|
|
23
|
+
readonly context: MlxProjectedContextRows;
|
|
24
|
+
constructor(readonly method: ProjectedDraftMethod,
|
|
25
|
+
checkpoints: readonly (DraftRowCheckpoint | null)[]) {
|
|
26
|
+
this.tapLayers = method.tapLayers;
|
|
27
|
+
this.context = new MlxProjectedContextRows(method.layers, { project: method.project });
|
|
28
|
+
try { this.append(checkpoints); } catch (error) { this.dispose(); throw error; }
|
|
29
|
+
}
|
|
30
|
+
get namespace(): string { return this.method.namespace; }
|
|
31
|
+
get rowCount(): number { return this.context.state.rowCount; }
|
|
32
|
+
prepareAppend(checkpoints: readonly (DraftRowCheckpoint | null)[]): PreparedStateChange {
|
|
33
|
+
const restored: Cache[][] = [];
|
|
34
|
+
const empty = () => Array.from({ length: this.method.layers }, () => new KVCache());
|
|
35
|
+
try {
|
|
36
|
+
for (const checkpoint of checkpoints) restored.push(checkpoint ? restoreCacheAttachment(checkpoint.attachment, empty) : empty());
|
|
37
|
+
return this.context.prepareAppend(restored);
|
|
38
|
+
} finally { disposeResources(restored.flat()); }
|
|
39
|
+
}
|
|
40
|
+
append(checkpoints: readonly (DraftRowCheckpoint | null)[]): void {
|
|
41
|
+
if (checkpoints.length) applyStateChanges([() => this.prepareAppend(checkpoints)]);
|
|
42
|
+
}
|
|
43
|
+
prefill(tokens: MlxArray, hidden?: MlxArray): void {
|
|
44
|
+
this.context.append(hidden!, Array(this.rowCount).fill(tokens.shape[1]!)); this.materialize();
|
|
45
|
+
}
|
|
46
|
+
materialize(): void { this.context.materialize(); }
|
|
47
|
+
filterRows(keep: readonly number[]): void { this.context.state.filterRows(keep); }
|
|
48
|
+
capture(row: number): DraftRowCheckpoint {
|
|
49
|
+
const caches = this.context.state.extractRow(row);
|
|
50
|
+
try {
|
|
51
|
+
const processedTokens = caches[0]!.offset;
|
|
52
|
+
return { processedTokens, attachment: captureCacheAttachment(this.method.schema, caches, { processedTokens }) };
|
|
53
|
+
} finally { disposeResources(caches); }
|
|
54
|
+
}
|
|
55
|
+
draft(pending: readonly number[], depth: number): number[][] {
|
|
56
|
+
if (!depth) return pending.map(() => []);
|
|
57
|
+
const positions = this.context.positions, donors = this.context.readAttention();
|
|
58
|
+
using offsets = positions.every(position => position === positions[0]) ? null : ops.fromInt32([...positions], [pending.length]);
|
|
59
|
+
try { return this.method.draft(donors, pending, offsets ?? positions[0]!, depth); }
|
|
60
|
+
finally { donors.dispose(); }
|
|
61
|
+
}
|
|
62
|
+
commit(accepted: readonly number[], hidden: MlxArray): void {
|
|
63
|
+
this.context.append(hidden, accepted.map(count => count + 1));
|
|
64
|
+
}
|
|
65
|
+
dispose(): void { this.context.dispose(); }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function projectedDraftGroups(namespace: string, bind: (target: TargetView) => ProjectedDraftMethod): GroupedDraftProvider {
|
|
69
|
+
const open = ({ target, checkpoints }: {
|
|
70
|
+
target: TargetView; checkpoints: readonly (DraftRowCheckpoint | null)[];
|
|
71
|
+
}) => new ProjectedDraftRows(bind(target), checkpoints);
|
|
72
|
+
return { checkpointNamespace: () => namespace, open, openPrefill: open };
|
|
73
|
+
}
|