mlx-bun 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/docs/README.md +4 -1
  2. package/docs/archive/investigations/openjarvis-fit.md +198 -0
  3. package/docs/archive/investigations/qwen38-27b-campaign-log.md +1092 -0
  4. package/docs/design/batching.md +1321 -49
  5. package/docs/design/decode-speed-program.md +331 -1163
  6. package/docs/design/generic-model-support.md +79 -3
  7. package/docs/design/kv-cache.md +198 -54
  8. package/docs/design/speculative-decoding.md +387 -24
  9. package/docs/design/turboquant.md +123 -13
  10. package/docs/design/unified-engine-frontier-plan.md +144 -142
  11. package/docs/planning/PRODUCT_ROADMAP.md +16 -0
  12. package/docs/planning/release-notes-v0.4.0.md +45 -0
  13. package/docs/reference/benchmarks.md +2457 -6
  14. package/docs/reference/cli.md +41 -2
  15. package/docs/reference/distribution.md +16 -9
  16. package/docs/reference/environment.md +55 -5
  17. package/docs/reference/models.md +13 -4
  18. package/docs/reference/server-api.md +79 -25
  19. package/docs/reference/server-config.md +178 -84
  20. package/docs/reference/troubleshooting.md +13 -21
  21. package/package.json +1 -1
  22. package/src/backends/mlx/assistant-target.ts +46 -0
  23. package/src/backends/mlx/autoregressive-draft-rows.ts +120 -0
  24. package/src/backends/mlx/batch-group.ts +368 -224
  25. package/src/backends/mlx/cache-attachment.ts +67 -0
  26. package/src/backends/mlx/cache-layout.ts +31 -0
  27. package/src/backends/mlx/checkpoint-state.ts +53 -0
  28. package/src/backends/mlx/continuation-persistence.ts +106 -0
  29. package/src/backends/mlx/continuation-request.ts +43 -0
  30. package/src/backends/mlx/continuation.ts +77 -0
  31. package/src/backends/mlx/draft-target.ts +7 -22
  32. package/src/backends/mlx/full-kv-row-append.ts +32 -0
  33. package/src/backends/mlx/full-kv-row-donor.ts +54 -0
  34. package/src/backends/mlx/full-transitioning-kv-rows.ts +65 -0
  35. package/src/backends/mlx/gateway-binding.ts +77 -18
  36. package/src/backends/mlx/kv-codec.ts +9 -0
  37. package/src/backends/mlx/kv-maintenance.ts +133 -0
  38. package/src/backends/mlx/kv-tensor-rows.ts +212 -0
  39. package/src/backends/mlx/model-serving.ts +1 -1
  40. package/src/backends/mlx/prefill-cohort.ts +89 -0
  41. package/src/backends/mlx/prefill-rows.ts +200 -0
  42. package/src/backends/mlx/prefill.ts +1 -1
  43. package/src/backends/mlx/projected-context-rows.ts +72 -0
  44. package/src/backends/mlx/request-state-policy.ts +41 -0
  45. package/src/backends/mlx/rollback.ts +41 -3
  46. package/src/backends/mlx/row-sampling.ts +49 -0
  47. package/src/backends/mlx/serial-executor.ts +35 -66
  48. package/src/backends/mlx/speculative-group.ts +359 -0
  49. package/src/backends/mlx/speculative-round.ts +161 -0
  50. package/src/backends/mlx/state-rows.ts +104 -0
  51. package/src/backends/mlx/state-views.ts +21 -1
  52. package/src/backends/mlx/target-layout-capability.ts +35 -0
  53. package/src/backends/mlx/transitioning-kv-rows.ts +165 -0
  54. package/src/cli.ts +11 -36
  55. package/src/contracts/execution.ts +11 -0
  56. package/src/contracts/prefix-cache.ts +23 -0
  57. package/src/contracts/resources.ts +7 -0
  58. package/src/contracts/sampling.ts +23 -0
  59. package/src/contracts/scheduling.ts +18 -0
  60. package/src/engine/execution-plan.ts +18 -16
  61. package/src/engine/resources.ts +12 -1
  62. package/src/engine/scheduler.ts +10 -5
  63. package/src/fit.ts +29 -15
  64. package/src/generate.ts +77 -173
  65. package/src/inference/draft-acceptance.ts +41 -0
  66. package/src/inference/draft-output.ts +49 -0
  67. package/src/inference/rollback.ts +2 -2
  68. package/src/kv-scheme.ts +70 -12
  69. package/src/kv-store.ts +70 -16
  70. package/src/lab/paged-kv/paged-kv-rows.ts +83 -0
  71. package/src/lab/paged-kv/paged-kv.ts +25 -11
  72. package/src/mlx/array.ts +2 -1
  73. package/src/model/artifact-identity.ts +73 -0
  74. package/src/model/batched-kv.ts +235 -0
  75. package/src/model/batched-mask.ts +12 -4
  76. package/src/model/batched-quant.ts +8 -2
  77. package/src/model/batched-quantized-kv.ts +144 -0
  78. package/src/model/batched-rotating-quant.ts +76 -15
  79. package/src/model/batched-rotating-state.ts +76 -0
  80. package/src/model/batched-rotating.ts +64 -22
  81. package/src/model/batched-row-storage.ts +52 -0
  82. package/src/model/batched-ssm.ts +52 -0
  83. package/src/model/batched-turboquant-kv.ts +93 -0
  84. package/src/model/compiled-decode.ts +7 -1
  85. package/src/model/decoded-kv-donor.ts +12 -0
  86. package/src/model/delayed-quantized-kv.ts +72 -0
  87. package/src/model/delayed-rotating-quantized-kv.ts +93 -0
  88. package/src/model/delayed-turboquant-kv.ts +55 -0
  89. package/src/model/full-prefill-padding.ts +56 -0
  90. package/src/model/full-prefill-row.ts +105 -0
  91. package/src/model/gemma4-base.ts +162 -80
  92. package/src/model/gemma4.ts +34 -26
  93. package/src/model/generated/gemma4-12b.ts +4 -1
  94. package/src/model/generated/gemma4-26b.ts +4 -1
  95. package/src/model/generated/gemma4-e4b.ts +4 -1
  96. package/src/model/glm52-cache.ts +80 -4
  97. package/src/model/glm52-mla.ts +12 -6
  98. package/src/model/glm52.ts +19 -40
  99. package/src/model/kv-attention-view.ts +67 -0
  100. package/src/model/minicpm5.ts +11 -7
  101. package/src/model/qwen3-delta-state.ts +74 -0
  102. package/src/model/qwen3-delta.ts +59 -36
  103. package/src/model/qwen3_5.ts +82 -42
  104. package/src/model/rotating-kv-donor.ts +35 -0
  105. package/src/model/rotating-kv-layout.ts +262 -0
  106. package/src/model/rotating-row-transaction.ts +35 -0
  107. package/src/model/speculative-rotating-kv.ts +101 -0
  108. package/src/model/ssm-prefill-padding.ts +39 -0
  109. package/src/model/trellis-linear.ts +2 -2
  110. package/src/model/turboquant-codec.ts +83 -0
  111. package/src/model/universal/dense.ts +14 -13
  112. package/src/preflight.ts +31 -15
  113. package/src/prompt-cache.ts +42 -44
  114. package/src/sampler.ts +36 -25
  115. package/src/serve/chat-stage.ts +2 -2
  116. package/src/serve/generation-gateway.ts +39 -30
  117. package/src/serve/model-host.ts +3 -4
  118. package/src/serve/request-plan.ts +4 -3
  119. package/src/serve/request-prep.ts +2 -4
  120. package/src/serve/text-completion-stage.ts +2 -2
  121. package/src/server.ts +64 -108
  122. package/src/spec/assistant-rows.ts +92 -0
  123. package/src/spec/assistant-source.ts +28 -18
  124. package/src/spec/cache-identity.ts +22 -0
  125. package/src/spec/deepspec-rows.ts +29 -0
  126. package/src/spec/deepspec-source.ts +12 -10
  127. package/src/spec/dflash-source.ts +28 -4
  128. package/src/spec/drafter.ts +153 -142
  129. package/src/spec/dspark/deepspec-module.ts +95 -120
  130. package/src/spec/dspark/module-dflash.ts +126 -170
  131. package/src/spec/glm52-mtp-graph.ts +29 -0
  132. package/src/spec/glm52-mtp-rows.ts +132 -0
  133. package/src/spec/glm52-mtp-source.ts +53 -136
  134. package/src/spec/glm52-mtp-state.ts +33 -0
  135. package/src/spec/ngram-source.ts +92 -20
  136. package/src/spec/projected-draft-rows.ts +73 -0
  137. package/src/spec/qwen-mtp-module.ts +171 -0
  138. package/src/spec/qwen-mtp-rows.ts +219 -0
  139. package/src/spec/qwen-mtp-source.ts +81 -226
  140. package/src/spec/qwen-mtp-state.ts +30 -0
  141. package/src/spec/serve-loop.ts +70 -68
  142. package/src/spec/source.ts +80 -15
  143. package/src/spec/two-model-rows.ts +61 -0
  144. package/src/spec/two-model.ts +43 -101
  145. package/src/ssd-cache.ts +33 -21
  146. package/src/ssd-durability.ts +11 -7
  147. package/src/web/app.js +57 -57
  148. package/src/backends/mlx/memory-guard.ts +0 -40
  149. package/src/spec/prefix-state.ts +0 -45
  150. /package/docs/{planning → archive/planning}/release-notes-v0.3.0.md +0 -0
package/src/ssd-cache.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import type { CheckpointAttachment } from "./backends/mlx/checkpoint-state";
1
2
  // SSD cold tier for the prompt/KV cache (docs/design/kv-cache.md).
2
3
  //
3
4
  // Files ARE the database: no sidecar index. Layout
@@ -22,10 +23,11 @@ import { randomUUID } from "node:crypto";
22
23
  import { commonPrefixLength } from "./prompt-cache";
23
24
  import {
24
25
  saveKvCache, saveKvCacheAsync, loadKvCache, readKvHeader,
25
- cacheHeadersTrimmable, legacyCacheCodecs, type CacheCodecProvider,
26
+ cacheHeadersTrimmable, cacheHeadersMinimumReusableOffset, legacyCacheCodecs, type CacheCodecProvider,
26
27
  type KvSaveMeta, type KvLoadExpect, type LoadedKvCache,
27
28
  } from "./kv-store";
28
29
  import type { Cache } from "./model/gemma4-base";
30
+ import { minimumReusableOffset } from "./backends/mlx/state-views";
29
31
 
30
32
  export interface SsdIndexEntry {
31
33
  path: string;
@@ -39,6 +41,7 @@ export interface SsdIndexEntry {
39
41
  * over a shorter usable one (the 2026-07-06 restart-0 defect: the
40
42
  * [prompt+gen] file always won, always got rejected after restore). */
41
43
  trimmable: boolean;
44
+ minimumReusableOffset?: number;
42
45
  generationCheckpoint?: NonNullable<KvSaveMeta["generationCheckpoint"]>;
43
46
  }
44
47
 
@@ -92,7 +95,7 @@ export class SsdCacheStore {
92
95
  * trimmable descendant is equivalent to an exact boundary snapshot. */
93
96
  hasDurablePrefix(tokens: number[], ns = ""): boolean {
94
97
  return this.#index.some((entry) => {
95
- if (entry.ns !== ns || entry.tokens.length < tokens.length) return false;
98
+ if (entry.ns !== ns || entry.tokens.length < tokens.length || tokens.length < (entry.minimumReusableOffset ?? 0)) return false;
96
99
  if (commonPrefixLength(entry.tokens, tokens) !== tokens.length) return false;
97
100
  return entry.tokens.length === tokens.length || entry.trimmable;
98
101
  });
@@ -129,7 +132,8 @@ export class SsdCacheStore {
129
132
  const st = statSync(path);
130
133
  this.#index.push({
131
134
  path, ns: h.ns ?? "", tokens: h.tokens, bytes: st.size, mtimeMs: st.mtimeMs,
132
- trimmable: cacheHeadersTrimmable(h.caches, this.#codecs),
135
+ trimmable: !h.attachments?.length && cacheHeadersTrimmable(h.caches, this.#codecs),
136
+ minimumReusableOffset: cacheHeadersMinimumReusableOffset(h.caches),
133
137
  ...(h.generationCheckpoint
134
138
  ? { generationCheckpoint: h.generationCheckpoint }
135
139
  : {}),
@@ -157,7 +161,7 @@ export class SsdCacheStore {
157
161
  for (const e of this.#index) {
158
162
  if (e.ns !== ns) continue;
159
163
  const p = Math.min(commonPrefixLength(e.tokens, prompt), prompt.length - 1);
160
- if (p === 0 || p <= bestLen) continue;
164
+ if (p === 0 || p <= bestLen || p < (e.minimumReusableOffset ?? 0)) continue;
161
165
  if (e.tokens.length - p > 0 && !e.trimmable) continue;
162
166
  bestLen = p;
163
167
  best = e;
@@ -201,13 +205,13 @@ export class SsdCacheStore {
201
205
  * the same ns whose tokens are a PREFIX of the new entry are superseded
202
206
  * (the agent-conversation pattern: one growing entry, not N generations
203
207
  * of it). Returns true if stored. */
204
- store(tokens: number[], caches: Cache[], ns = ""): boolean {
208
+ store(tokens: number[], caches: Cache[], ns = "", attachments?: CheckpointAttachment[]): boolean {
205
209
  const dir = join(this.#root, nsHash(ns));
206
210
  const path = join(dir, `${randomUUID()}.mlxkv`);
207
211
  try {
208
212
  mkdirSync(dir, { recursive: true });
209
- saveKvCache(path, tokens, caches, this.#meta(ns), this.#codecs);
210
- return this.#indexStored(path, tokens, caches, ns);
213
+ saveKvCache(path, tokens, caches, { ...this.#meta(ns), attachments }, this.#codecs);
214
+ return this.#indexStored(path, tokens, caches, ns, undefined, attachments);
211
215
  } catch (err) {
212
216
  return this.#storeFailed(path, err);
213
217
  }
@@ -224,13 +228,14 @@ export class SsdCacheStore {
224
228
  async storeAsync(
225
229
  tokens: number[], caches: Cache[], ns = "",
226
230
  runStep?: <T>(step: () => T) => Promise<T>,
231
+ attachments?: CheckpointAttachment[],
227
232
  ): Promise<boolean> {
228
233
  const dir = join(this.#root, nsHash(ns));
229
234
  const path = join(dir, `${randomUUID()}.mlxkv`);
230
235
  try {
231
236
  mkdirSync(dir, { recursive: true });
232
- await saveKvCacheAsync(path, tokens, caches, this.#meta(ns), runStep, this.#codecs);
233
- return this.#indexStored(path, tokens, caches, ns);
237
+ await saveKvCacheAsync(path, tokens, caches, { ...this.#meta(ns), attachments }, runStep, this.#codecs);
238
+ return this.#indexStored(path, tokens, caches, ns, undefined, attachments);
234
239
  } catch (err) {
235
240
  return this.#storeFailed(path, err);
236
241
  }
@@ -260,25 +265,30 @@ export class SsdCacheStore {
260
265
  * its rename succeeds, so a crash during a write leaves the prior point. */
261
266
  async storeGenerationCheckpoint(
262
267
  tokens: number[], caches: Cache[], checkpoint: NonNullable<KvSaveMeta["generationCheckpoint"]>,
268
+ runStep?: <T>(step: () => T) => Promise<T>,
263
269
  ): Promise<boolean> {
264
270
  const ns = `__generation_checkpoint__:${checkpoint.key}`;
265
271
  const dir = join(this.#root, nsHash(ns));
266
272
  const path = join(dir, `${randomUUID()}.mlxkv`);
267
273
  try {
268
- mkdirSync(dir, { recursive: true });
274
+ if (runStep) await runStep(() => mkdirSync(dir, { recursive: true }));
275
+ else mkdirSync(dir, { recursive: true });
269
276
  await saveKvCacheAsync(path, tokens, caches, {
270
277
  ...this.#meta(ns), generationCheckpoint: checkpoint,
271
- }, undefined, this.#codecs);
272
- const stored = this.#indexStored(path, tokens, caches, ns, checkpoint);
273
- if (stored) {
274
- for (const entry of [...this.#index]) {
275
- if (entry.path === path) continue;
276
- if (entry.generationCheckpoint?.key === checkpoint.key) this.remove(entry.path);
278
+ }, runStep, this.#codecs);
279
+ const index = () => {
280
+ const stored = this.#indexStored(path, tokens, caches, ns, checkpoint);
281
+ if (stored) {
282
+ for (const entry of [...this.#index]) {
283
+ if (entry.path === path) continue;
284
+ if (entry.generationCheckpoint?.key === checkpoint.key) this.remove(entry.path);
285
+ }
277
286
  }
278
- }
279
- return stored;
287
+ return stored;
288
+ };
289
+ return runStep ? await runStep(index) : index();
280
290
  } catch (err) {
281
- return this.#storeFailed(path, err);
291
+ return runStep ? await runStep(() => this.#storeFailed(path, err)) : this.#storeFailed(path, err);
282
292
  }
283
293
  }
284
294
 
@@ -299,6 +309,7 @@ export class SsdCacheStore {
299
309
  #indexStored(
300
310
  path: string, tokens: number[], caches: Cache[], ns: string,
301
311
  generationCheckpoint?: NonNullable<KvSaveMeta["generationCheckpoint"]>,
312
+ attachments?: CheckpointAttachment[],
302
313
  ): boolean {
303
314
  const st = statSync(path);
304
315
  if (st.size > this.#opts.maxBytes) {
@@ -306,7 +317,7 @@ export class SsdCacheStore {
306
317
  console.warn(`[ssd-cache] entry not stored: ${st.size} bytes exceeds the ${this.#opts.maxBytes}-byte cap`);
307
318
  return false;
308
319
  }
309
- const trimmable = caches.every((c) => c.isTrimmable());
320
+ const trimmable = !attachments?.length && caches.every((c) => c.isTrimmable());
310
321
  // Exact duplicates (same tokens) are replaced regardless of
311
322
  // trimmability — the new file serves exactly the old one's matches.
312
323
  for (const e of [...this.#index]) {
@@ -321,12 +332,13 @@ export class SsdCacheStore {
321
332
  // unrestorable one landed.
322
333
  if (trimmable) {
323
334
  for (const e of [...this.#index]) {
324
- if (e.ns !== ns || e.tokens.length >= tokens.length) continue;
335
+ if (e.ns !== ns || e.tokens.length >= tokens.length || e.tokens.length < minimumReusableOffset(caches)) continue;
325
336
  if (commonPrefixLength(e.tokens, tokens) === e.tokens.length) this.remove(e.path);
326
337
  }
327
338
  }
328
339
  this.#index.push({
329
340
  path, ns, tokens, bytes: st.size, mtimeMs: Date.now(), trimmable,
341
+ minimumReusableOffset: minimumReusableOffset(caches),
330
342
  ...(generationCheckpoint ? { generationCheckpoint } : {}),
331
343
  });
332
344
  this.stats.spills++;
@@ -1,10 +1,12 @@
1
+ import { cloneAttachments } from "./backends/mlx/checkpoint-state";
2
+ import { cleanupFailure, disposeResources } from "./engine/resources";
1
3
  import type { Cache } from "./model/gemma4";
2
4
  import type { PromptCacheEntry } from "./prompt-cache";
3
5
  import type { SpillItem, SpillQueue } from "./kv-store";
4
6
 
5
7
  export interface DurabilityGateway {
6
8
  readonly busy: boolean;
7
- runExclusive<T>(fn: () => Promise<T>): Promise<T>;
9
+ runWhenIdle<T>(fn: () => Promise<T>): Promise<T>;
8
10
  }
9
11
 
10
12
  export interface DurabilityPromptCache {
@@ -128,15 +130,17 @@ export class SsdDurabilityCoordinator {
128
130
  async #store(rec: DirtySnapshot): Promise<StoreOutcome> {
129
131
  let snap: SpillItem | null = null;
130
132
  try {
131
- snap = await this.gateway.runExclusive(async () => {
133
+ snap = await this.gateway.runWhenIdle(async () => {
132
134
  if (this.isAlreadyDurable(rec.tokens, rec.ns)) return null;
133
135
  const entry = this.promptCache.findExact(rec.tokens, rec.ns);
134
136
  if (!entry) return null;
135
- return {
136
- tokens: [...entry.tokens],
137
- caches: this.cloneCaches(entry.caches),
138
- ns: rec.ns,
139
- };
137
+ const caches = this.cloneCaches(entry.caches);
138
+ try {
139
+ return { tokens: [...entry.tokens], caches, ns: rec.ns,
140
+ attachments: cloneAttachments(entry.attachments) };
141
+ } catch (error) {
142
+ return cleanupFailure(error, () => disposeResources(caches));
143
+ }
140
144
  });
141
145
  } catch {
142
146
  return "failed";
package/src/web/app.js CHANGED
@@ -994,14 +994,14 @@ function highlightIn(container) {
994
994
  return;
995
995
  const blocks = container.querySelectorAll("pre code");
996
996
  for (const block of blocks) {
997
- const el2 = block;
998
- if (el2.dataset.highlighted)
997
+ const el = block;
998
+ if (el.dataset.highlighted)
999
999
  continue;
1000
- const hasLang = /\blanguage-/.test(el2.className);
1001
- if (!hasLang && (el2.textContent || "").length > 20000)
1000
+ const hasLang = /\blanguage-/.test(el.className);
1001
+ if (!hasLang && (el.textContent || "").length > 20000)
1002
1002
  continue;
1003
1003
  try {
1004
- hljs.highlightElement(el2);
1004
+ hljs.highlightElement(el);
1005
1005
  } catch {}
1006
1006
  }
1007
1007
  }
@@ -2611,12 +2611,12 @@ async function renderListBody() {
2611
2611
  async function renderArticleLists(query) {
2612
2612
  const listEl = $("mem-list");
2613
2613
  if (!query) {
2614
- const d2 = await getJson("/api/memory/list").catch(() => ({ ok: false }));
2615
- if (!d2.ok) {
2614
+ const d = await getJson("/api/memory/list").catch(() => ({ ok: false }));
2615
+ if (!d.ok) {
2616
2616
  listEl.innerHTML = `<div class="mem-list-empty">Couldn't load the vault contents.</div>`;
2617
2617
  return;
2618
2618
  }
2619
- const articles = d2.articles || [], reference = d2.reference || [];
2619
+ const articles = d.articles || [], reference = d.reference || [];
2620
2620
  listEl.innerHTML = '<div class="mem-sec"><div class="mem-sec-title">Articles</div>' + (articles.length ? articles.map((a) => articleRowHtml(a)).join("") : '<div class="mem-list-empty">No articles yet — nightly synthesis writes here after a few conversations.</div>') + "</div>" + (reference.length ? '<div class="mem-sec"><div class="mem-sec-title">Reference</div>' + reference.map((r) => articleRowHtml(r, undefined, "\uD83D\uDCD8")).join("") + "</div>" : "");
2621
2621
  wireArticleRows(listEl);
2622
2622
  return;
@@ -2953,11 +2953,11 @@ var INTERACTIVE_SELECTOR = [
2953
2953
  "[data-spotlight]"
2954
2954
  ].join(", ");
2955
2955
  var MAX_ELEMENTS = 120;
2956
- function isAgentChrome(el2) {
2957
- return !!el2.closest("[data-ui-chrome]") || !!el2.closest("#toasts");
2956
+ function isAgentChrome(el) {
2957
+ return !!el.closest("[data-ui-chrome]") || !!el.closest("#toasts");
2958
2958
  }
2959
- function isHiddenSelf(el2) {
2960
- const html = el2;
2959
+ function isHiddenSelf(el) {
2960
+ const html = el;
2961
2961
  if (html.hidden)
2962
2962
  return true;
2963
2963
  const inline = html.style?.display;
@@ -2968,21 +2968,21 @@ function isHiddenSelf(el2) {
2968
2968
  return true;
2969
2969
  return false;
2970
2970
  }
2971
- function isVisible(el2) {
2972
- const html = el2;
2971
+ function isVisible(el) {
2972
+ const html = el;
2973
2973
  if (html.offsetParent === null) {
2974
2974
  const style = globalThis.getComputedStyle?.(html);
2975
2975
  if (style && style.position === "fixed")
2976
2976
  return true;
2977
2977
  }
2978
- for (let node = el2;node; node = node.parentElement) {
2978
+ for (let node = el;node; node = node.parentElement) {
2979
2979
  if (isHiddenSelf(node))
2980
2980
  return false;
2981
2981
  }
2982
2982
  return true;
2983
2983
  }
2984
- function elementLabel(el2) {
2985
- const html = el2;
2984
+ function elementLabel(el) {
2985
+ const html = el;
2986
2986
  const uiLabel = html.getAttribute("data-ui-label")?.trim();
2987
2987
  if (uiLabel)
2988
2988
  return uiLabel;
@@ -3017,13 +3017,13 @@ function captureUiSnapshot(route) {
3017
3017
  const elements = [];
3018
3018
  const seenRefs = new Set;
3019
3019
  let index = 0;
3020
- for (const el2 of nodes) {
3021
- if (isAgentChrome(el2) || !isVisible(el2))
3020
+ for (const el of nodes) {
3021
+ if (isAgentChrome(el) || !isVisible(el))
3022
3022
  continue;
3023
- const label = elementLabel(el2);
3023
+ const label = elementLabel(el);
3024
3024
  if (!label)
3025
3025
  continue;
3026
- const html = el2;
3026
+ const html = el;
3027
3027
  const spotlightId = html.getAttribute("data-spotlight")?.trim() || undefined;
3028
3028
  const existingRef = html.getAttribute("data-ui-ref");
3029
3029
  const ref = existingRef ?? `ui-${route.replace(/[^a-zA-Z0-9]/g, "_") || "root"}-${index}`;
@@ -3067,12 +3067,12 @@ function findByLabel(snapshot, label) {
3067
3067
  return;
3068
3068
  let best;
3069
3069
  let bestScore = 0;
3070
- for (const el2 of snapshot.elements) {
3071
- const hay = [el2.label, el2.spotlightId].filter(Boolean).join(" ");
3070
+ for (const el of snapshot.elements) {
3071
+ const hay = [el.label, el.spotlightId].filter(Boolean).join(" ");
3072
3072
  const score = matchScore(hay, label);
3073
3073
  if (score > bestScore) {
3074
3074
  bestScore = score;
3075
- best = el2;
3075
+ best = el;
3076
3076
  }
3077
3077
  }
3078
3078
  return bestScore >= 50 ? best : undefined;
@@ -3089,9 +3089,9 @@ function safeQuery(selector) {
3089
3089
  }
3090
3090
  function resolveSpotlightTarget(request, snapshot) {
3091
3091
  if (request.ref) {
3092
- const el2 = snapshot?.elements.find((e) => e.ref === request.ref);
3093
- if (el2)
3094
- return { selector: el2.selector, title: el2.label, message: request.message };
3092
+ const el = snapshot?.elements.find((e) => e.ref === request.ref);
3093
+ if (el)
3094
+ return { selector: el.selector, title: el.label, message: request.message };
3095
3095
  const selector = `[data-ui-ref="${attrEscape(request.ref)}"]`;
3096
3096
  if (safeQuery(selector)) {
3097
3097
  return { selector, title: request.label ?? request.ref, message: request.message };
@@ -3101,9 +3101,9 @@ function resolveSpotlightTarget(request, snapshot) {
3101
3101
  return { selector: request.selector, title: request.label ?? "Here", message: request.message };
3102
3102
  }
3103
3103
  if (request.label) {
3104
- const el2 = findByLabel(snapshot, request.label);
3105
- if (el2)
3106
- return { selector: el2.selector, title: el2.label, message: request.message };
3104
+ const el = findByLabel(snapshot, request.label);
3105
+ if (el)
3106
+ return { selector: el.selector, title: el.label, message: request.message };
3107
3107
  let bestEl = null;
3108
3108
  let bestScore = 0;
3109
3109
  for (const node of document.querySelectorAll("[data-ui-label]")) {
@@ -3144,15 +3144,15 @@ var AUTO_DISMISS_MS = 3000;
3144
3144
  var dismissTimer = null;
3145
3145
  var dismissListeners = null;
3146
3146
  function ensureOverlayEl() {
3147
- let el2 = document.getElementById("assistant-spotlight");
3148
- if (!el2) {
3149
- el2 = document.createElement("div");
3150
- el2.id = "assistant-spotlight";
3151
- el2.setAttribute("data-ui-chrome", "assistant");
3152
- el2.innerHTML = '<div class="asr-ring"></div><div class="asr-pop"><div class="asr-pop-text"></div></div>';
3153
- document.body.appendChild(el2);
3147
+ let el = document.getElementById("assistant-spotlight");
3148
+ if (!el) {
3149
+ el = document.createElement("div");
3150
+ el.id = "assistant-spotlight";
3151
+ el.setAttribute("data-ui-chrome", "assistant");
3152
+ el.innerHTML = '<div class="asr-ring"></div><div class="asr-pop"><div class="asr-pop-text"></div></div>';
3153
+ document.body.appendChild(el);
3154
3154
  }
3155
- return el2;
3155
+ return el;
3156
3156
  }
3157
3157
  function dismissSpotlight() {
3158
3158
  if (dismissTimer) {
@@ -3163,9 +3163,9 @@ function dismissSpotlight() {
3163
3163
  dismissListeners();
3164
3164
  dismissListeners = null;
3165
3165
  }
3166
- const el2 = document.getElementById("assistant-spotlight");
3167
- if (el2)
3168
- el2.classList.remove("show");
3166
+ const el = document.getElementById("assistant-spotlight");
3167
+ if (el)
3168
+ el.classList.remove("show");
3169
3169
  }
3170
3170
  function showSpotlight(resolved) {
3171
3171
  const target = safeQuery(resolved.selector);
@@ -3332,7 +3332,7 @@ function createChatController() {
3332
3332
  }
3333
3333
  return false;
3334
3334
  }
3335
- function currentView2() {
3335
+ function currentView() {
3336
3336
  if ($("mem-overlay")?.classList.contains("open"))
3337
3337
  return "memory-panel";
3338
3338
  if ($("hub-overlay")?.classList.contains("open"))
@@ -3345,16 +3345,16 @@ function createChatController() {
3345
3345
  const route = currentRoute();
3346
3346
  if (!isRouteId(route))
3347
3347
  return;
3348
- const ctx = buildAppContext(route, currentView2());
3348
+ const ctx = buildAppContext(route, currentView());
3349
3349
  lastSnapshot = ctx.snapshot;
3350
3350
  send({ type: "context", context: ctx });
3351
3351
  }
3352
3352
  function watchWizardSteps() {
3353
3353
  const mo = new MutationObserver(() => pushAppContext());
3354
3354
  for (const id of ["q-steps", "f-steps", "d-steps"]) {
3355
- const el2 = $(id);
3356
- if (el2)
3357
- mo.observe(el2, { childList: true, subtree: true });
3355
+ const el = $(id);
3356
+ if (el)
3357
+ mo.observe(el, { childList: true, subtree: true });
3358
3358
  }
3359
3359
  }
3360
3360
  function setChatStatus(s) {
@@ -3548,8 +3548,8 @@ function createChatController() {
3548
3548
  if (t)
3549
3549
  return t;
3550
3550
  if (isMemoryToolName(tool || "")) {
3551
- const handle2 = memoryToolChip(a.bubble, tool, args);
3552
- t = { kind: "memchip", wrap: handle2.wrap, handle: handle2, chunks: "" };
3551
+ const handle = memoryToolChip(a.bubble, tool, args);
3552
+ t = { kind: "memchip", wrap: handle.wrap, handle, chunks: "" };
3553
3553
  a.tools.set(callId, t);
3554
3554
  stick();
3555
3555
  return t;
@@ -3763,9 +3763,9 @@ function createChatController() {
3763
3763
  }
3764
3764
  for (const t of item.tools || []) {
3765
3765
  if (isMemoryToolName(t.name || "")) {
3766
- const handle2 = memoryToolChip(bubble, t.name, t.args);
3766
+ const handle = memoryToolChip(bubble, t.name, t.args);
3767
3767
  if (t.result)
3768
- handle2.setResult(t.result);
3768
+ handle.setResult(t.result);
3769
3769
  continue;
3770
3770
  }
3771
3771
  const wrap = el("div", "tool ok", bubble);
@@ -4081,9 +4081,9 @@ function createChatController() {
4081
4081
  watchWizardSteps();
4082
4082
  const overlayMo = new MutationObserver(() => pushAppContext());
4083
4083
  for (const id of ["mem-overlay", "hub-overlay", "adapters-overlay"]) {
4084
- const el2 = $(id);
4085
- if (el2)
4086
- overlayMo.observe(el2, { attributes: true, attributeFilter: ["class"] });
4084
+ const el = $(id);
4085
+ if (el)
4086
+ overlayMo.observe(el, { attributes: true, attributeFilter: ["class"] });
4087
4087
  }
4088
4088
  },
4089
4089
  enter() {
@@ -4893,7 +4893,7 @@ function createStatusController() {
4893
4893
  async function loadFit() {
4894
4894
  let f;
4895
4895
  try {
4896
- f = await fetch("/fit").then((r2) => r2.json());
4896
+ f = await fetch("/fit").then((r) => r.json());
4897
4897
  } catch {
4898
4898
  return;
4899
4899
  }
@@ -5319,9 +5319,9 @@ async function openHubPanel() {
5319
5319
  if (inFlightDownloads.size)
5320
5320
  ensureDownloadPolling();
5321
5321
  setTimeout(() => {
5322
- const el22 = $("hub-search-input");
5323
- if (el22)
5324
- el22.focus();
5322
+ const el2 = $("hub-search-input");
5323
+ if (el2)
5324
+ el2.focus();
5325
5325
  }, 30);
5326
5326
  }
5327
5327
  function setOfflineFooter(offline) {
@@ -1,40 +0,0 @@
1
- import { DEFAULT_CHUNK, TRANSIENT_PER_TOKEN } from "../../fit";
2
- import { kvBytesAt, type KvSchemeOptions } from "../../kv-scheme";
3
- import { activeMemory, clearCache, maxRecommendedWorkingSetSize } from "../../mlx/ffi";
4
- import type { RuntimeModel } from "../../model/factory";
5
- import { Qwen35Model } from "../../model/qwen3_5";
6
- import type { PromptCache } from "../../prompt-cache";
7
- import { runtimeValue } from "../../runtime-config";
8
-
9
- /** Admission's ceiling and the cache that can release memory under the GPU lease. */
10
- export interface MlxMemoryBudget {
11
- readonly usableBytes: number;
12
- readonly kvOptions: KvSchemeOptions;
13
- readonly promptCache: Pick<PromptCache, "relievePressure" | "totalBytes">;
14
- }
15
-
16
- /** Both execution paths enter after acquiring the request's reusable prefix.
17
- * The caller owns the GPU lease and must close the scope on every exit. */
18
- export function enterMlxMemoryGuard(model: RuntimeModel, budget: MlxMemoryBudget,
19
- promptTokens: number, prefillChunkSize?: number): { check(): void; close(): void } {
20
- const reserve = (prefillChunkSize ?? DEFAULT_CHUNK) * TRANSIENT_PER_TOKEN +
21
- kvBytesAt(model.config, promptTokens, budget.kvOptions);
22
- const ceiling = Math.min(budget.usableBytes, maxRecommendedWorkingSetSize());
23
- const check = () => {
24
- if (activeMemory() + reserve <= ceiling) return;
25
- clearCache();
26
- if (runtimeValue("MLX_BUN_PREFILL_MEM_LOG") === "1")
27
- console.error(`[cache-memory] active=${activeMemory()} prompt=${budget.promptCache.totalBytes} reserve=${reserve} ceiling=${ceiling}`);
28
- const count = budget.promptCache.relievePressure(() => {
29
- clearCache();
30
- return activeMemory() + reserve > ceiling;
31
- });
32
- if (count) console.log(`[prompt-cache] demoted ${count} entries for GPU headroom`);
33
- if (activeMemory() + reserve > ceiling)
34
- throw new Error("insufficient GPU headroom after prompt-cache demotion; saved generation checkpoints are retained");
35
- };
36
- const qwen = model instanceof Qwen35Model ? model : undefined;
37
- const previous = qwen?.prefillMemoryGuard;
38
- if (qwen) qwen.prefillMemoryGuard = check;
39
- return { check, close() { if (qwen) qwen.prefillMemoryGuard = previous ?? null; } };
40
- }
@@ -1,45 +0,0 @@
1
- import type { Cache } from "../model/gemma4";
2
- import type { DisposableResource } from "../contracts/resources";
3
- import { cleanupFailure } from "../engine/resources";
4
-
5
- /** One most-recent, exact prefill boundary. Taking transfers ownership; a
6
- * recurrent target cannot be trimmed to an arbitrary common prefix. The
7
- * provider owns this state and releases it before its weights at shutdown. */
8
- export interface SpeculativePrefixState extends DisposableResource {
9
- readonly tokens: number[];
10
- readonly targetIdentity: object;
11
- readonly namespace: string;
12
- readonly bytes: number;
13
- readonly target: Cache[];
14
- }
15
-
16
- export class SpeculativePrefixStore<T extends SpeculativePrefixState> {
17
- #entry: T | null = null;
18
-
19
- take(prompt: readonly number[], targetIdentity: object, namespace: string,
20
- maxBytes: number): T | null {
21
- const entry = this.#entry;
22
- this.#entry = null;
23
- if (!entry) return null;
24
- if (entry.bytes <= maxBytes && entry.targetIdentity === targetIdentity &&
25
- entry.namespace === namespace && entry.tokens.length < prompt.length &&
26
- entry.tokens.every((token, index) => prompt[index] === token))
27
- return entry;
28
- entry.dispose();
29
- return null;
30
- }
31
-
32
- /** Takes ownership even when the entry exceeds the configured RAM budget. */
33
- put(entry: T, maxBytes: number): void {
34
- try { this.dispose(); }
35
- catch (error) { cleanupFailure(error, () => entry.dispose()); }
36
- if (entry.bytes <= maxBytes && maxBytes > 0) this.#entry = entry;
37
- else entry.dispose();
38
- }
39
-
40
- dispose(): void {
41
- const entry = this.#entry;
42
- this.#entry = null;
43
- entry?.dispose();
44
- }
45
- }