mlx-bun 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/README.md +4 -1
- package/docs/archive/investigations/openjarvis-fit.md +198 -0
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +1092 -0
- package/docs/design/batching.md +1321 -49
- package/docs/design/decode-speed-program.md +331 -1163
- package/docs/design/generic-model-support.md +79 -3
- package/docs/design/kv-cache.md +198 -54
- package/docs/design/speculative-decoding.md +387 -24
- package/docs/design/turboquant.md +123 -13
- package/docs/design/unified-engine-frontier-plan.md +144 -142
- package/docs/planning/PRODUCT_ROADMAP.md +16 -0
- package/docs/planning/release-notes-v0.4.0.md +45 -0
- package/docs/reference/benchmarks.md +2457 -6
- package/docs/reference/cli.md +41 -2
- package/docs/reference/distribution.md +16 -9
- package/docs/reference/environment.md +55 -5
- package/docs/reference/models.md +13 -4
- package/docs/reference/server-api.md +79 -25
- package/docs/reference/server-config.md +178 -84
- package/docs/reference/troubleshooting.md +13 -21
- package/package.json +1 -1
- package/src/backends/mlx/assistant-target.ts +46 -0
- package/src/backends/mlx/autoregressive-draft-rows.ts +120 -0
- package/src/backends/mlx/batch-group.ts +368 -224
- package/src/backends/mlx/cache-attachment.ts +67 -0
- package/src/backends/mlx/cache-layout.ts +31 -0
- package/src/backends/mlx/checkpoint-state.ts +53 -0
- package/src/backends/mlx/continuation-persistence.ts +106 -0
- package/src/backends/mlx/continuation-request.ts +43 -0
- package/src/backends/mlx/continuation.ts +77 -0
- package/src/backends/mlx/draft-target.ts +7 -22
- package/src/backends/mlx/full-kv-row-append.ts +32 -0
- package/src/backends/mlx/full-kv-row-donor.ts +54 -0
- package/src/backends/mlx/full-transitioning-kv-rows.ts +65 -0
- package/src/backends/mlx/gateway-binding.ts +77 -18
- package/src/backends/mlx/kv-codec.ts +9 -0
- package/src/backends/mlx/kv-maintenance.ts +133 -0
- package/src/backends/mlx/kv-tensor-rows.ts +212 -0
- package/src/backends/mlx/model-serving.ts +1 -1
- package/src/backends/mlx/prefill-cohort.ts +89 -0
- package/src/backends/mlx/prefill-rows.ts +200 -0
- package/src/backends/mlx/prefill.ts +1 -1
- package/src/backends/mlx/projected-context-rows.ts +72 -0
- package/src/backends/mlx/request-state-policy.ts +41 -0
- package/src/backends/mlx/rollback.ts +41 -3
- package/src/backends/mlx/row-sampling.ts +49 -0
- package/src/backends/mlx/serial-executor.ts +35 -66
- package/src/backends/mlx/speculative-group.ts +359 -0
- package/src/backends/mlx/speculative-round.ts +161 -0
- package/src/backends/mlx/state-rows.ts +104 -0
- package/src/backends/mlx/state-views.ts +21 -1
- package/src/backends/mlx/target-layout-capability.ts +35 -0
- package/src/backends/mlx/transitioning-kv-rows.ts +165 -0
- package/src/cli.ts +11 -36
- package/src/contracts/execution.ts +11 -0
- package/src/contracts/prefix-cache.ts +23 -0
- package/src/contracts/resources.ts +7 -0
- package/src/contracts/sampling.ts +23 -0
- package/src/contracts/scheduling.ts +18 -0
- package/src/engine/execution-plan.ts +18 -16
- package/src/engine/resources.ts +12 -1
- package/src/engine/scheduler.ts +10 -5
- package/src/fit.ts +29 -15
- package/src/generate.ts +77 -173
- package/src/inference/draft-acceptance.ts +41 -0
- package/src/inference/draft-output.ts +49 -0
- package/src/inference/rollback.ts +2 -2
- package/src/kv-scheme.ts +70 -12
- package/src/kv-store.ts +70 -16
- package/src/lab/paged-kv/paged-kv-rows.ts +83 -0
- package/src/lab/paged-kv/paged-kv.ts +25 -11
- package/src/mlx/array.ts +2 -1
- package/src/model/artifact-identity.ts +73 -0
- package/src/model/batched-kv.ts +235 -0
- package/src/model/batched-mask.ts +12 -4
- package/src/model/batched-quant.ts +8 -2
- package/src/model/batched-quantized-kv.ts +144 -0
- package/src/model/batched-rotating-quant.ts +76 -15
- package/src/model/batched-rotating-state.ts +76 -0
- package/src/model/batched-rotating.ts +64 -22
- package/src/model/batched-row-storage.ts +52 -0
- package/src/model/batched-ssm.ts +52 -0
- package/src/model/batched-turboquant-kv.ts +93 -0
- package/src/model/compiled-decode.ts +7 -1
- package/src/model/decoded-kv-donor.ts +12 -0
- package/src/model/delayed-quantized-kv.ts +72 -0
- package/src/model/delayed-rotating-quantized-kv.ts +93 -0
- package/src/model/delayed-turboquant-kv.ts +55 -0
- package/src/model/full-prefill-padding.ts +56 -0
- package/src/model/full-prefill-row.ts +105 -0
- package/src/model/gemma4-base.ts +162 -80
- package/src/model/gemma4.ts +34 -26
- package/src/model/generated/gemma4-12b.ts +4 -1
- package/src/model/generated/gemma4-26b.ts +4 -1
- package/src/model/generated/gemma4-e4b.ts +4 -1
- package/src/model/glm52-cache.ts +80 -4
- package/src/model/glm52-mla.ts +12 -6
- package/src/model/glm52.ts +19 -40
- package/src/model/kv-attention-view.ts +67 -0
- package/src/model/minicpm5.ts +11 -7
- package/src/model/qwen3-delta-state.ts +74 -0
- package/src/model/qwen3-delta.ts +59 -36
- package/src/model/qwen3_5.ts +82 -42
- package/src/model/rotating-kv-donor.ts +35 -0
- package/src/model/rotating-kv-layout.ts +262 -0
- package/src/model/rotating-row-transaction.ts +35 -0
- package/src/model/speculative-rotating-kv.ts +101 -0
- package/src/model/ssm-prefill-padding.ts +39 -0
- package/src/model/trellis-linear.ts +2 -2
- package/src/model/turboquant-codec.ts +83 -0
- package/src/model/universal/dense.ts +14 -13
- package/src/preflight.ts +31 -15
- package/src/prompt-cache.ts +42 -44
- package/src/sampler.ts +36 -25
- package/src/serve/chat-stage.ts +2 -2
- package/src/serve/generation-gateway.ts +39 -30
- package/src/serve/model-host.ts +3 -4
- package/src/serve/request-plan.ts +4 -3
- package/src/serve/request-prep.ts +2 -4
- package/src/serve/text-completion-stage.ts +2 -2
- package/src/server.ts +64 -108
- package/src/spec/assistant-rows.ts +92 -0
- package/src/spec/assistant-source.ts +28 -18
- package/src/spec/cache-identity.ts +22 -0
- package/src/spec/deepspec-rows.ts +29 -0
- package/src/spec/deepspec-source.ts +12 -10
- package/src/spec/dflash-source.ts +28 -4
- package/src/spec/drafter.ts +153 -142
- package/src/spec/dspark/deepspec-module.ts +95 -120
- package/src/spec/dspark/module-dflash.ts +126 -170
- package/src/spec/glm52-mtp-graph.ts +29 -0
- package/src/spec/glm52-mtp-rows.ts +132 -0
- package/src/spec/glm52-mtp-source.ts +53 -136
- package/src/spec/glm52-mtp-state.ts +33 -0
- package/src/spec/ngram-source.ts +92 -20
- package/src/spec/projected-draft-rows.ts +73 -0
- package/src/spec/qwen-mtp-module.ts +171 -0
- package/src/spec/qwen-mtp-rows.ts +219 -0
- package/src/spec/qwen-mtp-source.ts +81 -226
- package/src/spec/qwen-mtp-state.ts +30 -0
- package/src/spec/serve-loop.ts +70 -68
- package/src/spec/source.ts +80 -15
- package/src/spec/two-model-rows.ts +61 -0
- package/src/spec/two-model.ts +43 -101
- package/src/ssd-cache.ts +33 -21
- package/src/ssd-durability.ts +11 -7
- package/src/web/app.js +57 -57
- package/src/backends/mlx/memory-guard.ts +0 -40
- package/src/spec/prefix-state.ts +0 -45
- /package/docs/{planning → archive/planning}/release-notes-v0.3.0.md +0 -0
package/src/ssd-cache.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CheckpointAttachment } from "./backends/mlx/checkpoint-state";
|
|
1
2
|
// SSD cold tier for the prompt/KV cache (docs/design/kv-cache.md).
|
|
2
3
|
//
|
|
3
4
|
// Files ARE the database: no sidecar index. Layout
|
|
@@ -22,10 +23,11 @@ import { randomUUID } from "node:crypto";
|
|
|
22
23
|
import { commonPrefixLength } from "./prompt-cache";
|
|
23
24
|
import {
|
|
24
25
|
saveKvCache, saveKvCacheAsync, loadKvCache, readKvHeader,
|
|
25
|
-
cacheHeadersTrimmable, legacyCacheCodecs, type CacheCodecProvider,
|
|
26
|
+
cacheHeadersTrimmable, cacheHeadersMinimumReusableOffset, legacyCacheCodecs, type CacheCodecProvider,
|
|
26
27
|
type KvSaveMeta, type KvLoadExpect, type LoadedKvCache,
|
|
27
28
|
} from "./kv-store";
|
|
28
29
|
import type { Cache } from "./model/gemma4-base";
|
|
30
|
+
import { minimumReusableOffset } from "./backends/mlx/state-views";
|
|
29
31
|
|
|
30
32
|
export interface SsdIndexEntry {
|
|
31
33
|
path: string;
|
|
@@ -39,6 +41,7 @@ export interface SsdIndexEntry {
|
|
|
39
41
|
* over a shorter usable one (the 2026-07-06 restart-0 defect: the
|
|
40
42
|
* [prompt+gen] file always won, always got rejected after restore). */
|
|
41
43
|
trimmable: boolean;
|
|
44
|
+
minimumReusableOffset?: number;
|
|
42
45
|
generationCheckpoint?: NonNullable<KvSaveMeta["generationCheckpoint"]>;
|
|
43
46
|
}
|
|
44
47
|
|
|
@@ -92,7 +95,7 @@ export class SsdCacheStore {
|
|
|
92
95
|
* trimmable descendant is equivalent to an exact boundary snapshot. */
|
|
93
96
|
hasDurablePrefix(tokens: number[], ns = ""): boolean {
|
|
94
97
|
return this.#index.some((entry) => {
|
|
95
|
-
if (entry.ns !== ns || entry.tokens.length < tokens.length) return false;
|
|
98
|
+
if (entry.ns !== ns || entry.tokens.length < tokens.length || tokens.length < (entry.minimumReusableOffset ?? 0)) return false;
|
|
96
99
|
if (commonPrefixLength(entry.tokens, tokens) !== tokens.length) return false;
|
|
97
100
|
return entry.tokens.length === tokens.length || entry.trimmable;
|
|
98
101
|
});
|
|
@@ -129,7 +132,8 @@ export class SsdCacheStore {
|
|
|
129
132
|
const st = statSync(path);
|
|
130
133
|
this.#index.push({
|
|
131
134
|
path, ns: h.ns ?? "", tokens: h.tokens, bytes: st.size, mtimeMs: st.mtimeMs,
|
|
132
|
-
trimmable: cacheHeadersTrimmable(h.caches, this.#codecs),
|
|
135
|
+
trimmable: !h.attachments?.length && cacheHeadersTrimmable(h.caches, this.#codecs),
|
|
136
|
+
minimumReusableOffset: cacheHeadersMinimumReusableOffset(h.caches),
|
|
133
137
|
...(h.generationCheckpoint
|
|
134
138
|
? { generationCheckpoint: h.generationCheckpoint }
|
|
135
139
|
: {}),
|
|
@@ -157,7 +161,7 @@ export class SsdCacheStore {
|
|
|
157
161
|
for (const e of this.#index) {
|
|
158
162
|
if (e.ns !== ns) continue;
|
|
159
163
|
const p = Math.min(commonPrefixLength(e.tokens, prompt), prompt.length - 1);
|
|
160
|
-
if (p === 0 || p <= bestLen) continue;
|
|
164
|
+
if (p === 0 || p <= bestLen || p < (e.minimumReusableOffset ?? 0)) continue;
|
|
161
165
|
if (e.tokens.length - p > 0 && !e.trimmable) continue;
|
|
162
166
|
bestLen = p;
|
|
163
167
|
best = e;
|
|
@@ -201,13 +205,13 @@ export class SsdCacheStore {
|
|
|
201
205
|
* the same ns whose tokens are a PREFIX of the new entry are superseded
|
|
202
206
|
* (the agent-conversation pattern: one growing entry, not N generations
|
|
203
207
|
* of it). Returns true if stored. */
|
|
204
|
-
store(tokens: number[], caches: Cache[], ns = ""): boolean {
|
|
208
|
+
store(tokens: number[], caches: Cache[], ns = "", attachments?: CheckpointAttachment[]): boolean {
|
|
205
209
|
const dir = join(this.#root, nsHash(ns));
|
|
206
210
|
const path = join(dir, `${randomUUID()}.mlxkv`);
|
|
207
211
|
try {
|
|
208
212
|
mkdirSync(dir, { recursive: true });
|
|
209
|
-
saveKvCache(path, tokens, caches, this.#meta(ns), this.#codecs);
|
|
210
|
-
return this.#indexStored(path, tokens, caches, ns);
|
|
213
|
+
saveKvCache(path, tokens, caches, { ...this.#meta(ns), attachments }, this.#codecs);
|
|
214
|
+
return this.#indexStored(path, tokens, caches, ns, undefined, attachments);
|
|
211
215
|
} catch (err) {
|
|
212
216
|
return this.#storeFailed(path, err);
|
|
213
217
|
}
|
|
@@ -224,13 +228,14 @@ export class SsdCacheStore {
|
|
|
224
228
|
async storeAsync(
|
|
225
229
|
tokens: number[], caches: Cache[], ns = "",
|
|
226
230
|
runStep?: <T>(step: () => T) => Promise<T>,
|
|
231
|
+
attachments?: CheckpointAttachment[],
|
|
227
232
|
): Promise<boolean> {
|
|
228
233
|
const dir = join(this.#root, nsHash(ns));
|
|
229
234
|
const path = join(dir, `${randomUUID()}.mlxkv`);
|
|
230
235
|
try {
|
|
231
236
|
mkdirSync(dir, { recursive: true });
|
|
232
|
-
await saveKvCacheAsync(path, tokens, caches, this.#meta(ns), runStep, this.#codecs);
|
|
233
|
-
return this.#indexStored(path, tokens, caches, ns);
|
|
237
|
+
await saveKvCacheAsync(path, tokens, caches, { ...this.#meta(ns), attachments }, runStep, this.#codecs);
|
|
238
|
+
return this.#indexStored(path, tokens, caches, ns, undefined, attachments);
|
|
234
239
|
} catch (err) {
|
|
235
240
|
return this.#storeFailed(path, err);
|
|
236
241
|
}
|
|
@@ -260,25 +265,30 @@ export class SsdCacheStore {
|
|
|
260
265
|
* its rename succeeds, so a crash during a write leaves the prior point. */
|
|
261
266
|
async storeGenerationCheckpoint(
|
|
262
267
|
tokens: number[], caches: Cache[], checkpoint: NonNullable<KvSaveMeta["generationCheckpoint"]>,
|
|
268
|
+
runStep?: <T>(step: () => T) => Promise<T>,
|
|
263
269
|
): Promise<boolean> {
|
|
264
270
|
const ns = `__generation_checkpoint__:${checkpoint.key}`;
|
|
265
271
|
const dir = join(this.#root, nsHash(ns));
|
|
266
272
|
const path = join(dir, `${randomUUID()}.mlxkv`);
|
|
267
273
|
try {
|
|
268
|
-
mkdirSync(dir, { recursive: true });
|
|
274
|
+
if (runStep) await runStep(() => mkdirSync(dir, { recursive: true }));
|
|
275
|
+
else mkdirSync(dir, { recursive: true });
|
|
269
276
|
await saveKvCacheAsync(path, tokens, caches, {
|
|
270
277
|
...this.#meta(ns), generationCheckpoint: checkpoint,
|
|
271
|
-
},
|
|
272
|
-
const
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
278
|
+
}, runStep, this.#codecs);
|
|
279
|
+
const index = () => {
|
|
280
|
+
const stored = this.#indexStored(path, tokens, caches, ns, checkpoint);
|
|
281
|
+
if (stored) {
|
|
282
|
+
for (const entry of [...this.#index]) {
|
|
283
|
+
if (entry.path === path) continue;
|
|
284
|
+
if (entry.generationCheckpoint?.key === checkpoint.key) this.remove(entry.path);
|
|
285
|
+
}
|
|
277
286
|
}
|
|
278
|
-
|
|
279
|
-
|
|
287
|
+
return stored;
|
|
288
|
+
};
|
|
289
|
+
return runStep ? await runStep(index) : index();
|
|
280
290
|
} catch (err) {
|
|
281
|
-
return this.#storeFailed(path, err);
|
|
291
|
+
return runStep ? await runStep(() => this.#storeFailed(path, err)) : this.#storeFailed(path, err);
|
|
282
292
|
}
|
|
283
293
|
}
|
|
284
294
|
|
|
@@ -299,6 +309,7 @@ export class SsdCacheStore {
|
|
|
299
309
|
#indexStored(
|
|
300
310
|
path: string, tokens: number[], caches: Cache[], ns: string,
|
|
301
311
|
generationCheckpoint?: NonNullable<KvSaveMeta["generationCheckpoint"]>,
|
|
312
|
+
attachments?: CheckpointAttachment[],
|
|
302
313
|
): boolean {
|
|
303
314
|
const st = statSync(path);
|
|
304
315
|
if (st.size > this.#opts.maxBytes) {
|
|
@@ -306,7 +317,7 @@ export class SsdCacheStore {
|
|
|
306
317
|
console.warn(`[ssd-cache] entry not stored: ${st.size} bytes exceeds the ${this.#opts.maxBytes}-byte cap`);
|
|
307
318
|
return false;
|
|
308
319
|
}
|
|
309
|
-
const trimmable = caches.every((c) => c.isTrimmable());
|
|
320
|
+
const trimmable = !attachments?.length && caches.every((c) => c.isTrimmable());
|
|
310
321
|
// Exact duplicates (same tokens) are replaced regardless of
|
|
311
322
|
// trimmability — the new file serves exactly the old one's matches.
|
|
312
323
|
for (const e of [...this.#index]) {
|
|
@@ -321,12 +332,13 @@ export class SsdCacheStore {
|
|
|
321
332
|
// unrestorable one landed.
|
|
322
333
|
if (trimmable) {
|
|
323
334
|
for (const e of [...this.#index]) {
|
|
324
|
-
if (e.ns !== ns || e.tokens.length >= tokens.length) continue;
|
|
335
|
+
if (e.ns !== ns || e.tokens.length >= tokens.length || e.tokens.length < minimumReusableOffset(caches)) continue;
|
|
325
336
|
if (commonPrefixLength(e.tokens, tokens) === e.tokens.length) this.remove(e.path);
|
|
326
337
|
}
|
|
327
338
|
}
|
|
328
339
|
this.#index.push({
|
|
329
340
|
path, ns, tokens, bytes: st.size, mtimeMs: Date.now(), trimmable,
|
|
341
|
+
minimumReusableOffset: minimumReusableOffset(caches),
|
|
330
342
|
...(generationCheckpoint ? { generationCheckpoint } : {}),
|
|
331
343
|
});
|
|
332
344
|
this.stats.spills++;
|
package/src/ssd-durability.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { cloneAttachments } from "./backends/mlx/checkpoint-state";
|
|
2
|
+
import { cleanupFailure, disposeResources } from "./engine/resources";
|
|
1
3
|
import type { Cache } from "./model/gemma4";
|
|
2
4
|
import type { PromptCacheEntry } from "./prompt-cache";
|
|
3
5
|
import type { SpillItem, SpillQueue } from "./kv-store";
|
|
4
6
|
|
|
5
7
|
export interface DurabilityGateway {
|
|
6
8
|
readonly busy: boolean;
|
|
7
|
-
|
|
9
|
+
runWhenIdle<T>(fn: () => Promise<T>): Promise<T>;
|
|
8
10
|
}
|
|
9
11
|
|
|
10
12
|
export interface DurabilityPromptCache {
|
|
@@ -128,15 +130,17 @@ export class SsdDurabilityCoordinator {
|
|
|
128
130
|
async #store(rec: DirtySnapshot): Promise<StoreOutcome> {
|
|
129
131
|
let snap: SpillItem | null = null;
|
|
130
132
|
try {
|
|
131
|
-
snap = await this.gateway.
|
|
133
|
+
snap = await this.gateway.runWhenIdle(async () => {
|
|
132
134
|
if (this.isAlreadyDurable(rec.tokens, rec.ns)) return null;
|
|
133
135
|
const entry = this.promptCache.findExact(rec.tokens, rec.ns);
|
|
134
136
|
if (!entry) return null;
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
137
|
+
const caches = this.cloneCaches(entry.caches);
|
|
138
|
+
try {
|
|
139
|
+
return { tokens: [...entry.tokens], caches, ns: rec.ns,
|
|
140
|
+
attachments: cloneAttachments(entry.attachments) };
|
|
141
|
+
} catch (error) {
|
|
142
|
+
return cleanupFailure(error, () => disposeResources(caches));
|
|
143
|
+
}
|
|
140
144
|
});
|
|
141
145
|
} catch {
|
|
142
146
|
return "failed";
|
package/src/web/app.js
CHANGED
|
@@ -994,14 +994,14 @@ function highlightIn(container) {
|
|
|
994
994
|
return;
|
|
995
995
|
const blocks = container.querySelectorAll("pre code");
|
|
996
996
|
for (const block of blocks) {
|
|
997
|
-
const
|
|
998
|
-
if (
|
|
997
|
+
const el = block;
|
|
998
|
+
if (el.dataset.highlighted)
|
|
999
999
|
continue;
|
|
1000
|
-
const hasLang = /\blanguage-/.test(
|
|
1001
|
-
if (!hasLang && (
|
|
1000
|
+
const hasLang = /\blanguage-/.test(el.className);
|
|
1001
|
+
if (!hasLang && (el.textContent || "").length > 20000)
|
|
1002
1002
|
continue;
|
|
1003
1003
|
try {
|
|
1004
|
-
hljs.highlightElement(
|
|
1004
|
+
hljs.highlightElement(el);
|
|
1005
1005
|
} catch {}
|
|
1006
1006
|
}
|
|
1007
1007
|
}
|
|
@@ -2611,12 +2611,12 @@ async function renderListBody() {
|
|
|
2611
2611
|
async function renderArticleLists(query) {
|
|
2612
2612
|
const listEl = $("mem-list");
|
|
2613
2613
|
if (!query) {
|
|
2614
|
-
const
|
|
2615
|
-
if (!
|
|
2614
|
+
const d = await getJson("/api/memory/list").catch(() => ({ ok: false }));
|
|
2615
|
+
if (!d.ok) {
|
|
2616
2616
|
listEl.innerHTML = `<div class="mem-list-empty">Couldn't load the vault contents.</div>`;
|
|
2617
2617
|
return;
|
|
2618
2618
|
}
|
|
2619
|
-
const articles =
|
|
2619
|
+
const articles = d.articles || [], reference = d.reference || [];
|
|
2620
2620
|
listEl.innerHTML = '<div class="mem-sec"><div class="mem-sec-title">Articles</div>' + (articles.length ? articles.map((a) => articleRowHtml(a)).join("") : '<div class="mem-list-empty">No articles yet — nightly synthesis writes here after a few conversations.</div>') + "</div>" + (reference.length ? '<div class="mem-sec"><div class="mem-sec-title">Reference</div>' + reference.map((r) => articleRowHtml(r, undefined, "\uD83D\uDCD8")).join("") + "</div>" : "");
|
|
2621
2621
|
wireArticleRows(listEl);
|
|
2622
2622
|
return;
|
|
@@ -2953,11 +2953,11 @@ var INTERACTIVE_SELECTOR = [
|
|
|
2953
2953
|
"[data-spotlight]"
|
|
2954
2954
|
].join(", ");
|
|
2955
2955
|
var MAX_ELEMENTS = 120;
|
|
2956
|
-
function isAgentChrome(
|
|
2957
|
-
return !!
|
|
2956
|
+
function isAgentChrome(el) {
|
|
2957
|
+
return !!el.closest("[data-ui-chrome]") || !!el.closest("#toasts");
|
|
2958
2958
|
}
|
|
2959
|
-
function isHiddenSelf(
|
|
2960
|
-
const html =
|
|
2959
|
+
function isHiddenSelf(el) {
|
|
2960
|
+
const html = el;
|
|
2961
2961
|
if (html.hidden)
|
|
2962
2962
|
return true;
|
|
2963
2963
|
const inline = html.style?.display;
|
|
@@ -2968,21 +2968,21 @@ function isHiddenSelf(el2) {
|
|
|
2968
2968
|
return true;
|
|
2969
2969
|
return false;
|
|
2970
2970
|
}
|
|
2971
|
-
function isVisible(
|
|
2972
|
-
const html =
|
|
2971
|
+
function isVisible(el) {
|
|
2972
|
+
const html = el;
|
|
2973
2973
|
if (html.offsetParent === null) {
|
|
2974
2974
|
const style = globalThis.getComputedStyle?.(html);
|
|
2975
2975
|
if (style && style.position === "fixed")
|
|
2976
2976
|
return true;
|
|
2977
2977
|
}
|
|
2978
|
-
for (let node =
|
|
2978
|
+
for (let node = el;node; node = node.parentElement) {
|
|
2979
2979
|
if (isHiddenSelf(node))
|
|
2980
2980
|
return false;
|
|
2981
2981
|
}
|
|
2982
2982
|
return true;
|
|
2983
2983
|
}
|
|
2984
|
-
function elementLabel(
|
|
2985
|
-
const html =
|
|
2984
|
+
function elementLabel(el) {
|
|
2985
|
+
const html = el;
|
|
2986
2986
|
const uiLabel = html.getAttribute("data-ui-label")?.trim();
|
|
2987
2987
|
if (uiLabel)
|
|
2988
2988
|
return uiLabel;
|
|
@@ -3017,13 +3017,13 @@ function captureUiSnapshot(route) {
|
|
|
3017
3017
|
const elements = [];
|
|
3018
3018
|
const seenRefs = new Set;
|
|
3019
3019
|
let index = 0;
|
|
3020
|
-
for (const
|
|
3021
|
-
if (isAgentChrome(
|
|
3020
|
+
for (const el of nodes) {
|
|
3021
|
+
if (isAgentChrome(el) || !isVisible(el))
|
|
3022
3022
|
continue;
|
|
3023
|
-
const label = elementLabel(
|
|
3023
|
+
const label = elementLabel(el);
|
|
3024
3024
|
if (!label)
|
|
3025
3025
|
continue;
|
|
3026
|
-
const html =
|
|
3026
|
+
const html = el;
|
|
3027
3027
|
const spotlightId = html.getAttribute("data-spotlight")?.trim() || undefined;
|
|
3028
3028
|
const existingRef = html.getAttribute("data-ui-ref");
|
|
3029
3029
|
const ref = existingRef ?? `ui-${route.replace(/[^a-zA-Z0-9]/g, "_") || "root"}-${index}`;
|
|
@@ -3067,12 +3067,12 @@ function findByLabel(snapshot, label) {
|
|
|
3067
3067
|
return;
|
|
3068
3068
|
let best;
|
|
3069
3069
|
let bestScore = 0;
|
|
3070
|
-
for (const
|
|
3071
|
-
const hay = [
|
|
3070
|
+
for (const el of snapshot.elements) {
|
|
3071
|
+
const hay = [el.label, el.spotlightId].filter(Boolean).join(" ");
|
|
3072
3072
|
const score = matchScore(hay, label);
|
|
3073
3073
|
if (score > bestScore) {
|
|
3074
3074
|
bestScore = score;
|
|
3075
|
-
best =
|
|
3075
|
+
best = el;
|
|
3076
3076
|
}
|
|
3077
3077
|
}
|
|
3078
3078
|
return bestScore >= 50 ? best : undefined;
|
|
@@ -3089,9 +3089,9 @@ function safeQuery(selector) {
|
|
|
3089
3089
|
}
|
|
3090
3090
|
function resolveSpotlightTarget(request, snapshot) {
|
|
3091
3091
|
if (request.ref) {
|
|
3092
|
-
const
|
|
3093
|
-
if (
|
|
3094
|
-
return { selector:
|
|
3092
|
+
const el = snapshot?.elements.find((e) => e.ref === request.ref);
|
|
3093
|
+
if (el)
|
|
3094
|
+
return { selector: el.selector, title: el.label, message: request.message };
|
|
3095
3095
|
const selector = `[data-ui-ref="${attrEscape(request.ref)}"]`;
|
|
3096
3096
|
if (safeQuery(selector)) {
|
|
3097
3097
|
return { selector, title: request.label ?? request.ref, message: request.message };
|
|
@@ -3101,9 +3101,9 @@ function resolveSpotlightTarget(request, snapshot) {
|
|
|
3101
3101
|
return { selector: request.selector, title: request.label ?? "Here", message: request.message };
|
|
3102
3102
|
}
|
|
3103
3103
|
if (request.label) {
|
|
3104
|
-
const
|
|
3105
|
-
if (
|
|
3106
|
-
return { selector:
|
|
3104
|
+
const el = findByLabel(snapshot, request.label);
|
|
3105
|
+
if (el)
|
|
3106
|
+
return { selector: el.selector, title: el.label, message: request.message };
|
|
3107
3107
|
let bestEl = null;
|
|
3108
3108
|
let bestScore = 0;
|
|
3109
3109
|
for (const node of document.querySelectorAll("[data-ui-label]")) {
|
|
@@ -3144,15 +3144,15 @@ var AUTO_DISMISS_MS = 3000;
|
|
|
3144
3144
|
var dismissTimer = null;
|
|
3145
3145
|
var dismissListeners = null;
|
|
3146
3146
|
function ensureOverlayEl() {
|
|
3147
|
-
let
|
|
3148
|
-
if (!
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
document.body.appendChild(
|
|
3147
|
+
let el = document.getElementById("assistant-spotlight");
|
|
3148
|
+
if (!el) {
|
|
3149
|
+
el = document.createElement("div");
|
|
3150
|
+
el.id = "assistant-spotlight";
|
|
3151
|
+
el.setAttribute("data-ui-chrome", "assistant");
|
|
3152
|
+
el.innerHTML = '<div class="asr-ring"></div><div class="asr-pop"><div class="asr-pop-text"></div></div>';
|
|
3153
|
+
document.body.appendChild(el);
|
|
3154
3154
|
}
|
|
3155
|
-
return
|
|
3155
|
+
return el;
|
|
3156
3156
|
}
|
|
3157
3157
|
function dismissSpotlight() {
|
|
3158
3158
|
if (dismissTimer) {
|
|
@@ -3163,9 +3163,9 @@ function dismissSpotlight() {
|
|
|
3163
3163
|
dismissListeners();
|
|
3164
3164
|
dismissListeners = null;
|
|
3165
3165
|
}
|
|
3166
|
-
const
|
|
3167
|
-
if (
|
|
3168
|
-
|
|
3166
|
+
const el = document.getElementById("assistant-spotlight");
|
|
3167
|
+
if (el)
|
|
3168
|
+
el.classList.remove("show");
|
|
3169
3169
|
}
|
|
3170
3170
|
function showSpotlight(resolved) {
|
|
3171
3171
|
const target = safeQuery(resolved.selector);
|
|
@@ -3332,7 +3332,7 @@ function createChatController() {
|
|
|
3332
3332
|
}
|
|
3333
3333
|
return false;
|
|
3334
3334
|
}
|
|
3335
|
-
function
|
|
3335
|
+
function currentView() {
|
|
3336
3336
|
if ($("mem-overlay")?.classList.contains("open"))
|
|
3337
3337
|
return "memory-panel";
|
|
3338
3338
|
if ($("hub-overlay")?.classList.contains("open"))
|
|
@@ -3345,16 +3345,16 @@ function createChatController() {
|
|
|
3345
3345
|
const route = currentRoute();
|
|
3346
3346
|
if (!isRouteId(route))
|
|
3347
3347
|
return;
|
|
3348
|
-
const ctx = buildAppContext(route,
|
|
3348
|
+
const ctx = buildAppContext(route, currentView());
|
|
3349
3349
|
lastSnapshot = ctx.snapshot;
|
|
3350
3350
|
send({ type: "context", context: ctx });
|
|
3351
3351
|
}
|
|
3352
3352
|
function watchWizardSteps() {
|
|
3353
3353
|
const mo = new MutationObserver(() => pushAppContext());
|
|
3354
3354
|
for (const id of ["q-steps", "f-steps", "d-steps"]) {
|
|
3355
|
-
const
|
|
3356
|
-
if (
|
|
3357
|
-
mo.observe(
|
|
3355
|
+
const el = $(id);
|
|
3356
|
+
if (el)
|
|
3357
|
+
mo.observe(el, { childList: true, subtree: true });
|
|
3358
3358
|
}
|
|
3359
3359
|
}
|
|
3360
3360
|
function setChatStatus(s) {
|
|
@@ -3548,8 +3548,8 @@ function createChatController() {
|
|
|
3548
3548
|
if (t)
|
|
3549
3549
|
return t;
|
|
3550
3550
|
if (isMemoryToolName(tool || "")) {
|
|
3551
|
-
const
|
|
3552
|
-
t = { kind: "memchip", wrap:
|
|
3551
|
+
const handle = memoryToolChip(a.bubble, tool, args);
|
|
3552
|
+
t = { kind: "memchip", wrap: handle.wrap, handle, chunks: "" };
|
|
3553
3553
|
a.tools.set(callId, t);
|
|
3554
3554
|
stick();
|
|
3555
3555
|
return t;
|
|
@@ -3763,9 +3763,9 @@ function createChatController() {
|
|
|
3763
3763
|
}
|
|
3764
3764
|
for (const t of item.tools || []) {
|
|
3765
3765
|
if (isMemoryToolName(t.name || "")) {
|
|
3766
|
-
const
|
|
3766
|
+
const handle = memoryToolChip(bubble, t.name, t.args);
|
|
3767
3767
|
if (t.result)
|
|
3768
|
-
|
|
3768
|
+
handle.setResult(t.result);
|
|
3769
3769
|
continue;
|
|
3770
3770
|
}
|
|
3771
3771
|
const wrap = el("div", "tool ok", bubble);
|
|
@@ -4081,9 +4081,9 @@ function createChatController() {
|
|
|
4081
4081
|
watchWizardSteps();
|
|
4082
4082
|
const overlayMo = new MutationObserver(() => pushAppContext());
|
|
4083
4083
|
for (const id of ["mem-overlay", "hub-overlay", "adapters-overlay"]) {
|
|
4084
|
-
const
|
|
4085
|
-
if (
|
|
4086
|
-
overlayMo.observe(
|
|
4084
|
+
const el = $(id);
|
|
4085
|
+
if (el)
|
|
4086
|
+
overlayMo.observe(el, { attributes: true, attributeFilter: ["class"] });
|
|
4087
4087
|
}
|
|
4088
4088
|
},
|
|
4089
4089
|
enter() {
|
|
@@ -4893,7 +4893,7 @@ function createStatusController() {
|
|
|
4893
4893
|
async function loadFit() {
|
|
4894
4894
|
let f;
|
|
4895
4895
|
try {
|
|
4896
|
-
f = await fetch("/fit").then((
|
|
4896
|
+
f = await fetch("/fit").then((r) => r.json());
|
|
4897
4897
|
} catch {
|
|
4898
4898
|
return;
|
|
4899
4899
|
}
|
|
@@ -5319,9 +5319,9 @@ async function openHubPanel() {
|
|
|
5319
5319
|
if (inFlightDownloads.size)
|
|
5320
5320
|
ensureDownloadPolling();
|
|
5321
5321
|
setTimeout(() => {
|
|
5322
|
-
const
|
|
5323
|
-
if (
|
|
5324
|
-
|
|
5322
|
+
const el2 = $("hub-search-input");
|
|
5323
|
+
if (el2)
|
|
5324
|
+
el2.focus();
|
|
5325
5325
|
}, 30);
|
|
5326
5326
|
}
|
|
5327
5327
|
function setOfflineFooter(offline) {
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import { DEFAULT_CHUNK, TRANSIENT_PER_TOKEN } from "../../fit";
|
|
2
|
-
import { kvBytesAt, type KvSchemeOptions } from "../../kv-scheme";
|
|
3
|
-
import { activeMemory, clearCache, maxRecommendedWorkingSetSize } from "../../mlx/ffi";
|
|
4
|
-
import type { RuntimeModel } from "../../model/factory";
|
|
5
|
-
import { Qwen35Model } from "../../model/qwen3_5";
|
|
6
|
-
import type { PromptCache } from "../../prompt-cache";
|
|
7
|
-
import { runtimeValue } from "../../runtime-config";
|
|
8
|
-
|
|
9
|
-
/** Admission's ceiling and the cache that can release memory under the GPU lease. */
|
|
10
|
-
export interface MlxMemoryBudget {
|
|
11
|
-
readonly usableBytes: number;
|
|
12
|
-
readonly kvOptions: KvSchemeOptions;
|
|
13
|
-
readonly promptCache: Pick<PromptCache, "relievePressure" | "totalBytes">;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/** Both execution paths enter after acquiring the request's reusable prefix.
|
|
17
|
-
* The caller owns the GPU lease and must close the scope on every exit. */
|
|
18
|
-
export function enterMlxMemoryGuard(model: RuntimeModel, budget: MlxMemoryBudget,
|
|
19
|
-
promptTokens: number, prefillChunkSize?: number): { check(): void; close(): void } {
|
|
20
|
-
const reserve = (prefillChunkSize ?? DEFAULT_CHUNK) * TRANSIENT_PER_TOKEN +
|
|
21
|
-
kvBytesAt(model.config, promptTokens, budget.kvOptions);
|
|
22
|
-
const ceiling = Math.min(budget.usableBytes, maxRecommendedWorkingSetSize());
|
|
23
|
-
const check = () => {
|
|
24
|
-
if (activeMemory() + reserve <= ceiling) return;
|
|
25
|
-
clearCache();
|
|
26
|
-
if (runtimeValue("MLX_BUN_PREFILL_MEM_LOG") === "1")
|
|
27
|
-
console.error(`[cache-memory] active=${activeMemory()} prompt=${budget.promptCache.totalBytes} reserve=${reserve} ceiling=${ceiling}`);
|
|
28
|
-
const count = budget.promptCache.relievePressure(() => {
|
|
29
|
-
clearCache();
|
|
30
|
-
return activeMemory() + reserve > ceiling;
|
|
31
|
-
});
|
|
32
|
-
if (count) console.log(`[prompt-cache] demoted ${count} entries for GPU headroom`);
|
|
33
|
-
if (activeMemory() + reserve > ceiling)
|
|
34
|
-
throw new Error("insufficient GPU headroom after prompt-cache demotion; saved generation checkpoints are retained");
|
|
35
|
-
};
|
|
36
|
-
const qwen = model instanceof Qwen35Model ? model : undefined;
|
|
37
|
-
const previous = qwen?.prefillMemoryGuard;
|
|
38
|
-
if (qwen) qwen.prefillMemoryGuard = check;
|
|
39
|
-
return { check, close() { if (qwen) qwen.prefillMemoryGuard = previous ?? null; } };
|
|
40
|
-
}
|
package/src/spec/prefix-state.ts
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import type { Cache } from "../model/gemma4";
|
|
2
|
-
import type { DisposableResource } from "../contracts/resources";
|
|
3
|
-
import { cleanupFailure } from "../engine/resources";
|
|
4
|
-
|
|
5
|
-
/** One most-recent, exact prefill boundary. Taking transfers ownership; a
|
|
6
|
-
* recurrent target cannot be trimmed to an arbitrary common prefix. The
|
|
7
|
-
* provider owns this state and releases it before its weights at shutdown. */
|
|
8
|
-
export interface SpeculativePrefixState extends DisposableResource {
|
|
9
|
-
readonly tokens: number[];
|
|
10
|
-
readonly targetIdentity: object;
|
|
11
|
-
readonly namespace: string;
|
|
12
|
-
readonly bytes: number;
|
|
13
|
-
readonly target: Cache[];
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export class SpeculativePrefixStore<T extends SpeculativePrefixState> {
|
|
17
|
-
#entry: T | null = null;
|
|
18
|
-
|
|
19
|
-
take(prompt: readonly number[], targetIdentity: object, namespace: string,
|
|
20
|
-
maxBytes: number): T | null {
|
|
21
|
-
const entry = this.#entry;
|
|
22
|
-
this.#entry = null;
|
|
23
|
-
if (!entry) return null;
|
|
24
|
-
if (entry.bytes <= maxBytes && entry.targetIdentity === targetIdentity &&
|
|
25
|
-
entry.namespace === namespace && entry.tokens.length < prompt.length &&
|
|
26
|
-
entry.tokens.every((token, index) => prompt[index] === token))
|
|
27
|
-
return entry;
|
|
28
|
-
entry.dispose();
|
|
29
|
-
return null;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/** Takes ownership even when the entry exceeds the configured RAM budget. */
|
|
33
|
-
put(entry: T, maxBytes: number): void {
|
|
34
|
-
try { this.dispose(); }
|
|
35
|
-
catch (error) { cleanupFailure(error, () => entry.dispose()); }
|
|
36
|
-
if (entry.bytes <= maxBytes && maxBytes > 0) this.#entry = entry;
|
|
37
|
-
else entry.dispose();
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
dispose(): void {
|
|
41
|
-
const entry = this.#entry;
|
|
42
|
-
this.#entry = null;
|
|
43
|
-
entry?.dispose();
|
|
44
|
-
}
|
|
45
|
-
}
|
|
File without changes
|