mlx-bun 0.0.11 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +43 -13
  2. package/THIRD_PARTY_LICENSES.md +9 -0
  3. package/docs/design/colibri-glm52-port.md +1479 -0
  4. package/docs/design/pre-colibri-stabilization.md +329 -0
  5. package/docs/investigations/colibri-oracle-pin.md +1284 -0
  6. package/docs/investigations/indexshare-performance-spike.md +357 -0
  7. package/docs/planning/release-notes-v0.0.12.md +80 -0
  8. package/docs/reference/cli.md +28 -2
  9. package/docs/reference/distribution.md +25 -1
  10. package/docs/reference/embedding.md +5 -2
  11. package/docs/reference/features-matrix.md +4 -2
  12. package/docs/reference/library-api.md +41 -13
  13. package/docs/reference/models.md +49 -2
  14. package/docs/reference/server-api.md +90 -9
  15. package/docs/reference/server-config.md +21 -14
  16. package/docs/reference/training.md +10 -1
  17. package/package.json +1 -1
  18. package/src/anthropic.ts +54 -44
  19. package/src/chat-template.ts +155 -10
  20. package/src/cli.ts +127 -8
  21. package/src/config.ts +71 -26
  22. package/src/diffusion/diffusion-generate.ts +14 -8
  23. package/src/download.ts +103 -12
  24. package/src/eval/ifeval.ts +63 -120
  25. package/src/eval/runner.ts +14 -5
  26. package/src/eval/tasks/ifeval.ts +167 -31
  27. package/src/expert-io.ts +391 -0
  28. package/src/expert-residency.ts +1134 -0
  29. package/src/expert-usage.ts +415 -0
  30. package/src/generate.ts +114 -17
  31. package/src/hf-push.ts +1 -1
  32. package/src/index.ts +14 -2
  33. package/src/jobs/runner.ts +37 -18
  34. package/src/kv-store.ts +147 -8
  35. package/src/lora.ts +77 -14
  36. package/src/media-fetch.ts +198 -56
  37. package/src/mlx/array.ts +43 -0
  38. package/src/mlx/autograd.ts +51 -14
  39. package/src/mlx/checkpoint.ts +46 -13
  40. package/src/mlx/custom-vjp.ts +83 -35
  41. package/src/model/factory.ts +96 -5
  42. package/src/model/gemma4-base.ts +32 -0
  43. package/src/model/glm52-atlas.ts +511 -0
  44. package/src/model/glm52-cache.ts +706 -0
  45. package/src/model/glm52-config.ts +262 -0
  46. package/src/model/glm52-container.ts +305 -0
  47. package/src/model/glm52-coupling.ts +388 -0
  48. package/src/model/glm52-dsa.ts +602 -0
  49. package/src/model/glm52-expert-layout.ts +253 -0
  50. package/src/model/glm52-layout.ts +152 -0
  51. package/src/model/glm52-memory.ts +473 -0
  52. package/src/model/glm52-mla.ts +786 -0
  53. package/src/model/glm52-moe.ts +223 -0
  54. package/src/model/glm52-pilot.ts +537 -0
  55. package/src/model/glm52-reference.ts +445 -0
  56. package/src/model/glm52-residency.ts +557 -0
  57. package/src/model/glm52-resident-weights.ts +135 -0
  58. package/src/model/glm52-streamed-experts.ts +453 -0
  59. package/src/model/glm52-streamed-metal.ts +610 -0
  60. package/src/model/glm52-weights.ts +503 -0
  61. package/src/model/glm52.ts +1153 -0
  62. package/src/model/support.ts +8 -1
  63. package/src/native/expert_io.c +523 -0
  64. package/src/native-pack.ts +6 -4
  65. package/src/responses.ts +62 -49
  66. package/src/serve/batch-scheduler.ts +95 -16
  67. package/src/serve/generation-gateway.ts +41 -16
  68. package/src/server.ts +362 -81
  69. package/src/spec/glm52-mtp-source.ts +392 -0
  70. package/src/spec/serve-loop.ts +99 -18
  71. package/src/spec/source.ts +31 -6
  72. package/src/ssd-cache.ts +6 -2
  73. package/src/tool-call.ts +30 -2
  74. package/src/train/job.ts +0 -10
  75. package/src/train/lora-params.ts +17 -6
  76. package/src/train/trainer.ts +59 -26
  77. package/src/web/app.html +13 -2
  78. package/src/web/app.js +30 -3
  79. package/src/web/src/status.ts +51 -3
  80. package/src/web-tools.ts +54 -2
  81. package/src/weights.ts +82 -19
@@ -13,10 +13,9 @@
13
13
  // apply.
14
14
  //
15
15
  // DNS names are resolved and every resolved address is checked before the
16
- // fetch. The fetch itself re-resolves, so a DNS-rebinding attacker who
17
- // controls TTLs can theoretically race the check — accepted for a local
18
- // single-user server (fixing it needs pinned-address dialing, which
19
- // bun's fetch doesn't expose).
16
+ // fetch. The request then dials one of those checked IP literals while
17
+ // preserving the logical Host header and HTTPS SNI, so the transport cannot
18
+ // re-resolve to a private address between validation and connect.
20
19
  //
21
20
  // This module must stay free of mlx imports so tests/media-fetch.test.ts
22
21
  // runs model-free.
@@ -26,6 +25,8 @@ import { lookup } from "node:dns/promises";
26
25
  export interface MediaFetchPolicy {
27
26
  /** Permit private/loopback/link-local destinations (LAN hosts). */
28
27
  allowPrivate: boolean;
28
+ /** Optional caller-specific explanation when private destinations fail. */
29
+ privateDestinationHint?: string;
29
30
  /** Wall-clock budget for the whole fetch — every redirect hop plus the
30
31
  * body read share one AbortSignal.timeout. */
31
32
  timeoutMs: number;
@@ -35,6 +36,39 @@ export interface MediaFetchPolicy {
35
36
  maxRedirects: number;
36
37
  }
37
38
 
39
+ export type RestrictedFetch = (
40
+ input: string | URL,
41
+ init?: BunFetchRequestInit,
42
+ ) => Promise<Response>;
43
+
44
+ export type ResolveHost = (
45
+ hostname: string,
46
+ ) => Promise<ReadonlyArray<{ address: string }>>;
47
+
48
+ export interface RestrictedFetchDependencies {
49
+ fetch: RestrictedFetch;
50
+ resolve: ResolveHost;
51
+ }
52
+
53
+ export interface RestrictedFetchOptions {
54
+ /** Optional caller cancellation, combined with the policy timeout. */
55
+ signal?: AbortSignal;
56
+ /** Request headers applied on every validated redirect hop. */
57
+ headers?: Record<string, string>;
58
+ /** Test seam for deterministic DNS/redirect/body behavior. */
59
+ dependencies?: Partial<RestrictedFetchDependencies>;
60
+ /** Media callers reject non-2xx before reading; web_fetch returns the body. */
61
+ rejectHttpErrors?: boolean;
62
+ }
63
+
64
+ export interface RestrictedFetchResult {
65
+ ok: boolean;
66
+ status: number;
67
+ finalUrl: string;
68
+ contentType: string;
69
+ bytes: Uint8Array;
70
+ }
71
+
38
72
  export function defaultMediaFetchPolicy(): MediaFetchPolicy {
39
73
  return {
40
74
  allowPrivate: process.env.MLX_BUN_ALLOW_PRIVATE_MEDIA === "1",
@@ -50,6 +84,10 @@ const PRIVATE_HINT =
50
84
  "private/loopback hosts are blocked by default; start the server with " +
51
85
  "--allow-private-media (or MLX_BUN_ALLOW_PRIVATE_MEDIA=1) to fetch from LAN hosts";
52
86
 
87
+ function privateHint(policy: MediaFetchPolicy): string {
88
+ return policy.privateDestinationHint ?? PRIVATE_HINT;
89
+ }
90
+
53
91
  function parseIpv4(s: string): number[] | null {
54
92
  const parts = s.split(".");
55
93
  if (parts.length !== 4) return null;
@@ -155,19 +193,22 @@ export function checkMediaUrl(u: URL, policy: MediaFetchPolicy): string | null {
155
193
  if (policy.allowPrivate) return null;
156
194
  const lower = host.toLowerCase();
157
195
  if (lower === "localhost" || lower.endsWith(".localhost"))
158
- return `destination "${host}" — ${PRIVATE_HINT}`;
196
+ return `destination "${host}" — ${privateHint(policy)}`;
159
197
  // WHATWG URL parsing already canonicalized numeric IPv4 forms
160
198
  // (http://2130706433/, http://0x7f000001/ → 127.0.0.1).
161
199
  if (parseIpv4(lower) || parseIpv6(lower)) {
162
- if (isBlockedAddress(lower)) return `destination "${host}" — ${PRIVATE_HINT}`;
200
+ if (isBlockedAddress(lower))
201
+ return `destination "${host}" — ${privateHint(policy)}`;
163
202
  }
164
203
  return null;
165
204
  }
166
205
 
167
- async function assertResolvesPublic(
206
+ export async function assertResolvesPublic(
168
207
  hostname: string,
169
208
  kind: string,
170
209
  policy: MediaFetchPolicy,
210
+ resolve: ResolveHost = async (host) =>
211
+ await lookup(host, { all: true, verbatim: true }),
171
212
  ): Promise<void> {
172
213
  if (policy.allowPrivate) return;
173
214
  const host =
@@ -177,52 +218,105 @@ async function assertResolvesPublic(
177
218
  if (parseIpv4(host) || parseIpv6(host)) return; // literal — checked sync
178
219
  let addrs: { address: string }[];
179
220
  try {
180
- addrs = await lookup(host, { all: true, verbatim: true });
221
+ addrs = [...await resolve(host)];
181
222
  } catch {
182
223
  throw new Error(`${kind} fetch failed: could not resolve host "${host}"`);
183
224
  }
184
225
  for (const a of addrs) {
185
226
  if (isBlockedAddress(a.address))
186
227
  throw new Error(
187
- `${kind} url rejected: host "${host}" resolves to ${a.address} — ${PRIVATE_HINT}`,
228
+ `${kind} url rejected: host "${host}" resolves to ${a.address} — ${privateHint(policy)}`,
188
229
  );
189
230
  }
190
231
  }
191
232
 
192
- /** Fetch image/audio bytes for an OpenAI-style image_url/audio_url part.
193
- * data: URLs decode locally (no policy — the bytes came in the request);
194
- * http(s) URLs go through the destination policy above, with every
195
- * redirect hop re-validated, one wall-clock timeout, and a streaming
196
- * size cap. Throws Error with a client-presentable message — callers
197
- * surface it as a 400 (the prompt-build catch in server.ts). */
198
- export async function fetchMediaBytes(
199
- url: string,
200
- kind: "image" | "audio",
201
- policy: MediaFetchPolicy = defaultMediaFetchPolicy(),
202
- ): Promise<Uint8Array> {
203
- if (url.startsWith("data:")) {
204
- const comma = url.indexOf(",");
205
- if (comma === -1) throw new Error("malformed data: URL");
206
- const meta = url.slice(0, comma);
207
- const body = url.slice(comma + 1);
208
- if (!meta.includes("base64")) throw new Error("data: URL must be base64");
209
- return Uint8Array.from(Buffer.from(body, "base64"));
233
+ async function resolveFetchAddresses(
234
+ hostname: string,
235
+ kind: string,
236
+ policy: MediaFetchPolicy,
237
+ resolve: ResolveHost,
238
+ ): Promise<string[]> {
239
+ const host =
240
+ hostname.startsWith("[") && hostname.endsWith("]")
241
+ ? hostname.slice(1, -1)
242
+ : hostname;
243
+ if (parseIpv4(host) || parseIpv6(host)) return [host];
244
+ if (policy.allowPrivate) return [];
245
+ let addrs: { address: string }[];
246
+ try {
247
+ addrs = [...await resolve(host)];
248
+ } catch {
249
+ throw new Error(`${kind} fetch failed: could not resolve host "${host}"`);
210
250
  }
251
+ if (addrs.length === 0)
252
+ throw new Error(`${kind} fetch failed: host "${host}" resolved to no addresses`);
253
+ for (const a of addrs) {
254
+ if (isBlockedAddress(a.address))
255
+ throw new Error(
256
+ `${kind} url rejected: host "${host}" resolves to ${a.address} — ${privateHint(policy)}`,
257
+ );
258
+ }
259
+ return addrs.map((a) => a.address);
260
+ }
261
+
262
+ /**
263
+ * Fetch an http(s) response through the shared public-destination policy.
264
+ * Every redirect is handled manually and revalidated, caller cancellation
265
+ * shares one signal with the wall-clock timeout, and the body is streamed
266
+ * into a byte-capped buffer.
267
+ */
268
+ export async function fetchRestrictedHttpBytes(
269
+ url: string,
270
+ kind: string,
271
+ policy: MediaFetchPolicy,
272
+ options: RestrictedFetchOptions = {},
273
+ ): Promise<RestrictedFetchResult> {
211
274
  let current: URL;
212
275
  try {
213
276
  current = new URL(url);
214
277
  } catch {
215
278
  throw new Error(`unsupported ${kind} url scheme: ${url.slice(0, 16)}`);
216
279
  }
280
+
281
+ const fetchImpl = options.dependencies?.fetch ??
282
+ ((input: string | URL, init?: BunFetchRequestInit) => fetch(input, init));
283
+ const resolve = options.dependencies?.resolve ??
284
+ (async (host: string) => await lookup(host, { all: true, verbatim: true }));
285
+ const timeout = AbortSignal.timeout(policy.timeoutMs);
286
+ const signal = options.signal
287
+ ? AbortSignal.any([options.signal, timeout])
288
+ : timeout;
217
289
  const capMb = Math.round(policy.maxBytes / (1024 * 1024));
218
- const signal = AbortSignal.timeout(policy.timeoutMs);
219
290
  let overCap = false;
291
+
220
292
  try {
221
293
  for (let hop = 0; ; hop++) {
222
294
  const reason = checkMediaUrl(current, policy);
223
295
  if (reason) throw new Error(`${kind} url rejected: ${reason}`);
224
- await assertResolvesPublic(current.hostname, kind, policy);
225
- const res = await fetch(current, { redirect: "manual", signal });
296
+ const addresses = await resolveFetchAddresses(
297
+ current.hostname,
298
+ kind,
299
+ policy,
300
+ resolve,
301
+ );
302
+ const logicalHostname = current.hostname.replace(/^\[|\]$/g, "");
303
+ const connectUrl = new URL(current);
304
+ if (!policy.allowPrivate && addresses.length > 0) {
305
+ const address = addresses[0]!;
306
+ connectUrl.hostname = address.includes(":") ? `[${address}]` : address;
307
+ }
308
+ const headers = new Headers(options.headers);
309
+ if (connectUrl.hostname !== current.hostname && !headers.has("host"))
310
+ headers.set("host", current.host);
311
+ const init: BunFetchRequestInit = {
312
+ redirect: "manual",
313
+ signal,
314
+ headers,
315
+ };
316
+ if (current.protocol === "https:" && connectUrl.hostname !== current.hostname) {
317
+ init.tls = { serverName: logicalHostname };
318
+ }
319
+ const res = await fetchImpl(connectUrl, init);
226
320
  if (res.status >= 300 && res.status < 400) {
227
321
  const loc = res.headers.get("location");
228
322
  if (!loc)
@@ -239,46 +333,94 @@ export async function fetchMediaBytes(
239
333
  }
240
334
  continue;
241
335
  }
242
- if (!res.ok) throw new Error(`${kind} fetch failed: ${res.status} ${current}`);
336
+
337
+ if (options.rejectHttpErrors && !res.ok) {
338
+ try {
339
+ await res.body?.cancel();
340
+ } catch {}
341
+ throw new Error(`${kind} fetch failed: ${res.status} ${current}`);
342
+ }
343
+
243
344
  const declared = Number(res.headers.get("content-length") ?? "0");
244
- if (declared > policy.maxBytes) {
345
+ if (Number.isFinite(declared) && declared > policy.maxBytes) {
245
346
  overCap = true;
246
347
  try {
247
348
  await res.body?.cancel();
248
349
  } catch {}
249
350
  throw new Error(`${kind} response exceeds the ${capMb} MB limit`);
250
351
  }
251
- if (!res.body) return new Uint8Array(await res.arrayBuffer());
252
- // stream + count: Content-Length is optional (chunked) and spoofable
253
- const reader = res.body.getReader();
254
- const chunks: Uint8Array[] = [];
255
- let total = 0;
256
- for (;;) {
257
- const { done, value } = await reader.read();
258
- if (done) break;
259
- total += value.byteLength;
260
- if (total > policy.maxBytes) {
352
+
353
+ let bytes: Uint8Array;
354
+ if (!res.body) {
355
+ bytes = new Uint8Array(await res.arrayBuffer());
356
+ if (bytes.byteLength > policy.maxBytes) {
261
357
  overCap = true;
262
- try {
263
- await reader.cancel();
264
- } catch {}
265
358
  throw new Error(`${kind} response exceeds the ${capMb} MB limit`);
266
359
  }
267
- chunks.push(value);
268
- }
269
- const out = new Uint8Array(total);
270
- let off = 0;
271
- for (const c of chunks) {
272
- out.set(c, off);
273
- off += c.byteLength;
360
+ } else {
361
+ const reader = res.body.getReader();
362
+ const chunks: Uint8Array[] = [];
363
+ let total = 0;
364
+ for (;;) {
365
+ const { done, value } = await reader.read();
366
+ if (done) break;
367
+ total += value.byteLength;
368
+ if (total > policy.maxBytes) {
369
+ overCap = true;
370
+ try {
371
+ await reader.cancel();
372
+ } catch {}
373
+ throw new Error(`${kind} response exceeds the ${capMb} MB limit`);
374
+ }
375
+ chunks.push(value);
376
+ }
377
+ bytes = new Uint8Array(total);
378
+ let offset = 0;
379
+ for (const chunk of chunks) {
380
+ bytes.set(chunk, offset);
381
+ offset += chunk.byteLength;
382
+ }
274
383
  }
275
- return out;
384
+
385
+ return {
386
+ ok: res.ok,
387
+ status: res.status,
388
+ finalUrl: current.href,
389
+ contentType: res.headers.get("content-type") ?? "",
390
+ bytes,
391
+ };
276
392
  }
277
- } catch (e) {
278
- if (signal.aborted && !overCap)
393
+ } catch (error) {
394
+ if (timeout.aborted && !options.signal?.aborted && !overCap)
279
395
  throw new Error(
280
396
  `${kind} fetch timed out after ${policy.timeoutMs} ms: ${url.slice(0, 200)}`,
281
397
  );
282
- throw e;
398
+ throw error;
399
+ }
400
+ }
401
+
402
+ /** Fetch image/audio bytes for an OpenAI-style image_url/audio_url part.
403
+ * data: URLs decode locally (no policy — the bytes came in the request);
404
+ * http(s) URLs go through the destination policy above, with every
405
+ * redirect hop re-validated, one wall-clock timeout, and a streaming
406
+ * size cap. Throws Error with a client-presentable message — callers
407
+ * surface it as a 400 (the prompt-build catch in server.ts). */
408
+ export async function fetchMediaBytes(
409
+ url: string,
410
+ kind: "image" | "audio",
411
+ policy: MediaFetchPolicy = defaultMediaFetchPolicy(),
412
+ ): Promise<Uint8Array> {
413
+ if (url.startsWith("data:")) {
414
+ const comma = url.indexOf(",");
415
+ if (comma === -1) throw new Error("malformed data: URL");
416
+ const meta = url.slice(0, comma);
417
+ const body = url.slice(comma + 1);
418
+ if (!meta.includes("base64")) throw new Error("data: URL must be base64");
419
+ return Uint8Array.from(Buffer.from(body, "base64"));
283
420
  }
421
+ return (
422
+ await fetchRestrictedHttpBytes(url, kind, policy, {
423
+ rejectHttpErrors: true,
424
+ })
425
+ ).bytes;
284
426
  }
package/src/mlx/array.ts CHANGED
@@ -292,6 +292,49 @@ export class MlxArray {
292
292
  return out;
293
293
  }
294
294
 
295
+ /** Read an f32/f16/bf16 array numerically as float32 without enqueueing a
296
+ * GPU cast. This evaluates the ORIGINAL array, then expands 16-bit values
297
+ * on the host. Use this for small readbacks after later GPU work has
298
+ * already been dispatched: toFloat32() would append astype(float32) behind
299
+ * that work and turn the readback into an accidental full-step barrier. */
300
+ toFloat32Host(): Float32Array {
301
+ this.eval();
302
+ const dt = this.dtype;
303
+ if (dt === Dtype.float32) {
304
+ const p = C.mlx_array_data_float32(this.handle);
305
+ return new Float32Array(toArrayBuffer(p!, 0, this.size * 4)).slice();
306
+ }
307
+ if (dt === Dtype.bfloat16) {
308
+ const p = C.mlx_array_data_bfloat16(this.handle);
309
+ const src = new Uint16Array(toArrayBuffer(p!, 0, this.size * 2));
310
+ const words = new Uint32Array(src.length);
311
+ for (let i = 0; i < src.length; i++) words[i] = src[i]! << 16;
312
+ return new Float32Array(words.buffer);
313
+ }
314
+ if (dt === Dtype.float16) {
315
+ const p = C.mlx_array_data_float16(this.handle);
316
+ const src = new Uint16Array(toArrayBuffer(p!, 0, this.size * 2));
317
+ const out = new Float32Array(src.length);
318
+ for (let i = 0; i < src.length; i++) {
319
+ const bits = src[i]!;
320
+ const sign = bits & 0x8000;
321
+ const exponent = (bits >>> 10) & 0x1f;
322
+ const fraction = bits & 0x03ff;
323
+ let value: number;
324
+ if (exponent === 0) {
325
+ value = fraction === 0 ? 0 : fraction * 2 ** -24;
326
+ } else if (exponent === 0x1f) {
327
+ value = fraction === 0 ? Infinity : NaN;
328
+ } else {
329
+ value = (1 + fraction / 1024) * 2 ** (exponent - 15);
330
+ }
331
+ out[i] = sign ? -value : value;
332
+ }
333
+ return out;
334
+ }
335
+ throw new Error(`toFloat32Host: unsupported dtype ${this.dtypeName}`);
336
+ }
337
+
295
338
  /** Basic slice: start/stop per dimension (stride 1). */
296
339
  slice(start: number[], stop: number[], stream: MlxHandle = gpuStream): MlxArray {
297
340
  const s = new Int32Array(start);
@@ -15,6 +15,23 @@ import { JSCallback, ptr, read } from "bun:ffi";
15
15
  import { C, takeMlxError } from "./ffi";
16
16
  import { MlxArray } from "./array";
17
17
 
18
+ type CallbackHandle = Pick<JSCallback, "ptr" | "close">;
19
+ type NativeCall<F extends (...args: any[]) => any> =
20
+ (...args: Parameters<F>) => ReturnType<F>;
21
+
22
+ /** Constructor-only dependency seam for deterministic native failure tests. */
23
+ export interface ValueAndGradConstructorHooks {
24
+ callback?: (
25
+ callback: ConstructorParameters<typeof JSCallback>[0],
26
+ definition: ConstructorParameters<typeof JSCallback>[1],
27
+ ) => CallbackHandle;
28
+ closureNew?: NativeCall<typeof C.mlx_closure_new_func_payload>;
29
+ transformedNew?: NativeCall<typeof C.mlx_closure_value_and_grad_new>;
30
+ transform?: NativeCall<typeof C.mlx_value_and_grad>;
31
+ transformedFree?: NativeCall<typeof C.mlx_closure_value_and_grad_free>;
32
+ closureFree?: NativeCall<typeof C.mlx_closure_free>;
33
+ }
34
+
18
35
  /** A reusable value-and-grad over a JS-built loss graph.
19
36
  *
20
37
  * `loss(primals)` receives the input arrays in argument order and must
@@ -28,18 +45,30 @@ import { MlxArray } from "./array";
28
45
  * are owned by mlx — do NOT dispose them; the returned loss IS disposed
29
46
  * of by mlx's graph. */
30
47
  export class ValueAndGrad {
31
- readonly #cb: JSCallback;
48
+ readonly #cb: CallbackHandle;
32
49
  readonly #closure: bigint;
33
50
  readonly #vag: bigint;
34
51
  readonly #argIdx: Int32Array;
52
+ readonly #freeClosure: NativeCall<typeof C.mlx_closure_free>;
53
+ readonly #freeVag: NativeCall<typeof C.mlx_closure_value_and_grad_free>;
35
54
  #closureError: string | null = null;
36
55
  #disposed = false;
37
56
 
38
- constructor(loss: (primals: MlxArray[]) => MlxArray, argIdx: number[]) {
57
+ constructor(
58
+ loss: (primals: MlxArray[]) => MlxArray,
59
+ argIdx: number[],
60
+ hooks: ValueAndGradConstructorHooks = {},
61
+ ) {
39
62
  if (argIdx.length === 0) throw new Error("ValueAndGrad: argIdx must be non-empty");
40
63
  this.#argIdx = new Int32Array(argIdx);
64
+ const makeCallback = hooks.callback ?? ((callback, definition) => new JSCallback(callback, definition));
65
+ const closureNew = hooks.closureNew ?? C.mlx_closure_new_func_payload;
66
+ const transformedNew = hooks.transformedNew ?? C.mlx_closure_value_and_grad_new;
67
+ const transform = hooks.transform ?? C.mlx_value_and_grad;
68
+ this.#freeClosure = hooks.closureFree ?? C.mlx_closure_free;
69
+ this.#freeVag = hooks.transformedFree ?? C.mlx_closure_value_and_grad_free;
41
70
 
42
- this.#cb = new JSCallback(
71
+ const cb = makeCallback(
43
72
  (outPtr: number, inVec: bigint, _payload: number): number => {
44
73
  try {
45
74
  // Read the primals out of the input vector_array. mlx_vector_array_get
@@ -73,16 +102,24 @@ export class ValueAndGrad {
73
102
  { args: ["ptr", "u64", "ptr"], returns: "i32" },
74
103
  );
75
104
 
76
- this.#closure = C.mlx_closure_new_func_payload(this.#cb.ptr as never, null, null);
77
-
78
- const vagSlot = new BigUint64Array([C.mlx_closure_value_and_grad_new()]);
79
- if (C.mlx_value_and_grad(ptr(vagSlot), this.#closure, ptr(this.#argIdx), BigInt(this.#argIdx.length)) !== 0) {
80
- // best-effort cleanup before throwing
81
- C.mlx_closure_free(this.#closure);
82
- this.#cb.close();
83
- throw new Error("mlx_value_and_grad failed");
105
+ let closure: bigint | null = null;
106
+ let vagSlot: BigUint64Array | null = null;
107
+ try {
108
+ closure = closureNew(cb.ptr as never, null, null);
109
+ vagSlot = new BigUint64Array([transformedNew()]);
110
+ if (transform(ptr(vagSlot), closure, ptr(this.#argIdx), BigInt(this.#argIdx.length)) !== 0)
111
+ throw new Error("mlx_value_and_grad failed");
112
+ this.#cb = cb;
113
+ this.#closure = closure;
114
+ this.#vag = read.u64(ptr(vagSlot), 0);
115
+ } catch (e) {
116
+ // mlx-c out-params are initialized live handles. On failure the slot
117
+ // still owns whichever handle it contains and must be freed explicitly.
118
+ if (vagSlot) this.#freeVag(read.u64(ptr(vagSlot), 0));
119
+ if (closure !== null) this.#freeClosure(closure);
120
+ cb.close();
121
+ throw e;
84
122
  }
85
- this.#vag = read.u64(ptr(vagSlot), 0);
86
123
  }
87
124
 
88
125
  /** Run the loss + backward on `primals` (in argument order). Returns the
@@ -136,8 +173,8 @@ export class ValueAndGrad {
136
173
  dispose(): void {
137
174
  if (this.#disposed) return;
138
175
  this.#disposed = true;
139
- C.mlx_closure_value_and_grad_free(this.#vag);
140
- C.mlx_closure_free(this.#closure);
176
+ this.#freeVag(this.#vag);
177
+ this.#freeClosure(this.#closure);
141
178
  this.#cb.close();
142
179
  }
143
180
 
@@ -15,6 +15,22 @@ import { JSCallback, ptr, read } from "bun:ffi";
15
15
  import { C, takeMlxError } from "./ffi";
16
16
  import { MlxArray } from "./array";
17
17
 
18
+ type CallbackHandle = Pick<JSCallback, "ptr" | "close">;
19
+ type NativeCall<F extends (...args: any[]) => any> =
20
+ (...args: Parameters<F>) => ReturnType<F>;
21
+
22
+ /** Constructor-only dependency seam for deterministic native failure tests. */
23
+ export interface CheckpointConstructorHooks {
24
+ callback?: (
25
+ callback: ConstructorParameters<typeof JSCallback>[0],
26
+ definition: ConstructorParameters<typeof JSCallback>[1],
27
+ ) => CallbackHandle;
28
+ closureNew?: NativeCall<typeof C.mlx_closure_new_func_payload>;
29
+ transformedNew?: NativeCall<typeof C.mlx_closure_new>;
30
+ transform?: NativeCall<typeof C.mlx_checkpoint>;
31
+ closureFree?: NativeCall<typeof C.mlx_closure_free>;
32
+ }
33
+
18
34
  /** A reusable checkpointed wrapper over a JS-built subgraph.
19
35
  *
20
36
  * `fn(inputs)` receives the input arrays (in order) and returns one or more
@@ -24,14 +40,24 @@ import { MlxArray } from "./array";
24
40
  * AND to any outer-trace leaves captured by `fn` (e.g. swapped-in LoRA
25
41
  * primals), exactly as if `fn` had been inlined. */
26
42
  export class Checkpoint {
27
- readonly #cb: JSCallback;
43
+ readonly #cb: CallbackHandle;
28
44
  readonly #closure: bigint;
29
45
  readonly #ckpt: bigint;
46
+ readonly #freeClosure: NativeCall<typeof C.mlx_closure_free>;
30
47
  #closureError: string | null = null;
31
48
  #disposed = false;
32
49
 
33
- constructor(fn: (inputs: MlxArray[]) => MlxArray[]) {
34
- this.#cb = new JSCallback(
50
+ constructor(
51
+ fn: (inputs: MlxArray[]) => MlxArray[],
52
+ hooks: CheckpointConstructorHooks = {},
53
+ ) {
54
+ const makeCallback = hooks.callback ?? ((callback, definition) => new JSCallback(callback, definition));
55
+ const closureNew = hooks.closureNew ?? C.mlx_closure_new_func_payload;
56
+ const transformedNew = hooks.transformedNew ?? C.mlx_closure_new;
57
+ const transform = hooks.transform ?? C.mlx_checkpoint;
58
+ this.#freeClosure = hooks.closureFree ?? C.mlx_closure_free;
59
+
60
+ const cb = makeCallback(
35
61
  (outPtr: number, inVec: bigint, _payload: number): number => {
36
62
  try {
37
63
  const n = Number(C.mlx_vector_array_size(inVec));
@@ -58,15 +84,22 @@ export class Checkpoint {
58
84
  { args: ["ptr", "u64", "ptr"], returns: "i32" },
59
85
  );
60
86
 
61
- this.#closure = C.mlx_closure_new_func_payload(this.#cb.ptr as never, null, null);
62
-
63
- const slot = new BigUint64Array([C.mlx_closure_new()]);
64
- if (C.mlx_checkpoint(ptr(slot), this.#closure) !== 0) {
65
- C.mlx_closure_free(this.#closure);
66
- this.#cb.close();
67
- throw new Error(`mlx_checkpoint failed: ${takeMlxError() ?? ""}`);
87
+ let closure: bigint | null = null;
88
+ let slot: BigUint64Array | null = null;
89
+ try {
90
+ closure = closureNew(cb.ptr as never, null, null);
91
+ slot = new BigUint64Array([transformedNew()]);
92
+ if (transform(ptr(slot), closure) !== 0)
93
+ throw new Error(`mlx_checkpoint failed: ${takeMlxError() ?? ""}`);
94
+ this.#cb = cb;
95
+ this.#closure = closure;
96
+ this.#ckpt = read.u64(ptr(slot), 0);
97
+ } catch (e) {
98
+ if (slot) this.#freeClosure(read.u64(ptr(slot), 0));
99
+ if (closure !== null) this.#freeClosure(closure);
100
+ cb.close();
101
+ throw e;
68
102
  }
69
- this.#ckpt = read.u64(ptr(slot), 0);
70
103
  }
71
104
 
72
105
  /** Apply the checkpointed fn. Returns its outputs as owned MlxArrays. */
@@ -105,8 +138,8 @@ export class Checkpoint {
105
138
  dispose(): void {
106
139
  if (this.#disposed) return;
107
140
  this.#disposed = true;
108
- C.mlx_closure_free(this.#ckpt);
109
- C.mlx_closure_free(this.#closure);
141
+ this.#freeClosure(this.#ckpt);
142
+ this.#freeClosure(this.#closure);
110
143
  this.#cb.close();
111
144
  }
112
145