mlx-bun 0.0.11 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -13
- package/THIRD_PARTY_LICENSES.md +9 -0
- package/docs/design/colibri-glm52-port.md +1479 -0
- package/docs/design/pre-colibri-stabilization.md +329 -0
- package/docs/investigations/colibri-oracle-pin.md +1284 -0
- package/docs/investigations/indexshare-performance-spike.md +357 -0
- package/docs/planning/release-notes-v0.0.12.md +80 -0
- package/docs/reference/cli.md +28 -2
- package/docs/reference/distribution.md +25 -1
- package/docs/reference/embedding.md +5 -2
- package/docs/reference/features-matrix.md +4 -2
- package/docs/reference/library-api.md +41 -13
- package/docs/reference/models.md +49 -2
- package/docs/reference/server-api.md +90 -9
- package/docs/reference/server-config.md +21 -14
- package/docs/reference/training.md +10 -1
- package/package.json +1 -1
- package/src/anthropic.ts +54 -44
- package/src/chat-template.ts +155 -10
- package/src/cli.ts +127 -8
- package/src/config.ts +71 -26
- package/src/diffusion/diffusion-generate.ts +14 -8
- package/src/download.ts +103 -12
- package/src/eval/ifeval.ts +63 -120
- package/src/eval/runner.ts +14 -5
- package/src/eval/tasks/ifeval.ts +167 -31
- package/src/expert-io.ts +391 -0
- package/src/expert-residency.ts +1134 -0
- package/src/expert-usage.ts +415 -0
- package/src/generate.ts +114 -17
- package/src/hf-push.ts +1 -1
- package/src/index.ts +14 -2
- package/src/jobs/runner.ts +37 -18
- package/src/kv-store.ts +147 -8
- package/src/lora.ts +77 -14
- package/src/media-fetch.ts +198 -56
- package/src/mlx/array.ts +43 -0
- package/src/mlx/autograd.ts +51 -14
- package/src/mlx/checkpoint.ts +46 -13
- package/src/mlx/custom-vjp.ts +83 -35
- package/src/model/factory.ts +96 -5
- package/src/model/gemma4-base.ts +32 -0
- package/src/model/glm52-atlas.ts +511 -0
- package/src/model/glm52-cache.ts +706 -0
- package/src/model/glm52-config.ts +262 -0
- package/src/model/glm52-container.ts +305 -0
- package/src/model/glm52-coupling.ts +388 -0
- package/src/model/glm52-dsa.ts +602 -0
- package/src/model/glm52-expert-layout.ts +253 -0
- package/src/model/glm52-layout.ts +152 -0
- package/src/model/glm52-memory.ts +473 -0
- package/src/model/glm52-mla.ts +786 -0
- package/src/model/glm52-moe.ts +223 -0
- package/src/model/glm52-pilot.ts +537 -0
- package/src/model/glm52-reference.ts +445 -0
- package/src/model/glm52-residency.ts +557 -0
- package/src/model/glm52-resident-weights.ts +135 -0
- package/src/model/glm52-streamed-experts.ts +453 -0
- package/src/model/glm52-streamed-metal.ts +610 -0
- package/src/model/glm52-weights.ts +503 -0
- package/src/model/glm52.ts +1153 -0
- package/src/model/support.ts +8 -1
- package/src/native/expert_io.c +523 -0
- package/src/native-pack.ts +6 -4
- package/src/responses.ts +62 -49
- package/src/serve/batch-scheduler.ts +95 -16
- package/src/serve/generation-gateway.ts +41 -16
- package/src/server.ts +362 -81
- package/src/spec/glm52-mtp-source.ts +392 -0
- package/src/spec/serve-loop.ts +99 -18
- package/src/spec/source.ts +31 -6
- package/src/ssd-cache.ts +6 -2
- package/src/tool-call.ts +30 -2
- package/src/train/job.ts +0 -10
- package/src/train/lora-params.ts +17 -6
- package/src/train/trainer.ts +59 -26
- package/src/web/app.html +13 -2
- package/src/web/app.js +30 -3
- package/src/web/src/status.ts +51 -3
- package/src/web-tools.ts +54 -2
- package/src/weights.ts +82 -19
package/src/media-fetch.ts
CHANGED
|
@@ -13,10 +13,9 @@
|
|
|
13
13
|
// apply.
|
|
14
14
|
//
|
|
15
15
|
// DNS names are resolved and every resolved address is checked before the
|
|
16
|
-
// fetch. The
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
// bun's fetch doesn't expose).
|
|
16
|
+
// fetch. The request then dials one of those checked IP literals while
|
|
17
|
+
// preserving the logical Host header and HTTPS SNI, so the transport cannot
|
|
18
|
+
// re-resolve to a private address between validation and connect.
|
|
20
19
|
//
|
|
21
20
|
// This module must stay free of mlx imports so tests/media-fetch.test.ts
|
|
22
21
|
// runs model-free.
|
|
@@ -26,6 +25,8 @@ import { lookup } from "node:dns/promises";
|
|
|
26
25
|
export interface MediaFetchPolicy {
|
|
27
26
|
/** Permit private/loopback/link-local destinations (LAN hosts). */
|
|
28
27
|
allowPrivate: boolean;
|
|
28
|
+
/** Optional caller-specific explanation when private destinations fail. */
|
|
29
|
+
privateDestinationHint?: string;
|
|
29
30
|
/** Wall-clock budget for the whole fetch — every redirect hop plus the
|
|
30
31
|
* body read share one AbortSignal.timeout. */
|
|
31
32
|
timeoutMs: number;
|
|
@@ -35,6 +36,39 @@ export interface MediaFetchPolicy {
|
|
|
35
36
|
maxRedirects: number;
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
export type RestrictedFetch = (
|
|
40
|
+
input: string | URL,
|
|
41
|
+
init?: BunFetchRequestInit,
|
|
42
|
+
) => Promise<Response>;
|
|
43
|
+
|
|
44
|
+
export type ResolveHost = (
|
|
45
|
+
hostname: string,
|
|
46
|
+
) => Promise<ReadonlyArray<{ address: string }>>;
|
|
47
|
+
|
|
48
|
+
export interface RestrictedFetchDependencies {
|
|
49
|
+
fetch: RestrictedFetch;
|
|
50
|
+
resolve: ResolveHost;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface RestrictedFetchOptions {
|
|
54
|
+
/** Optional caller cancellation, combined with the policy timeout. */
|
|
55
|
+
signal?: AbortSignal;
|
|
56
|
+
/** Request headers applied on every validated redirect hop. */
|
|
57
|
+
headers?: Record<string, string>;
|
|
58
|
+
/** Test seam for deterministic DNS/redirect/body behavior. */
|
|
59
|
+
dependencies?: Partial<RestrictedFetchDependencies>;
|
|
60
|
+
/** Media callers reject non-2xx before reading; web_fetch returns the body. */
|
|
61
|
+
rejectHttpErrors?: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface RestrictedFetchResult {
|
|
65
|
+
ok: boolean;
|
|
66
|
+
status: number;
|
|
67
|
+
finalUrl: string;
|
|
68
|
+
contentType: string;
|
|
69
|
+
bytes: Uint8Array;
|
|
70
|
+
}
|
|
71
|
+
|
|
38
72
|
export function defaultMediaFetchPolicy(): MediaFetchPolicy {
|
|
39
73
|
return {
|
|
40
74
|
allowPrivate: process.env.MLX_BUN_ALLOW_PRIVATE_MEDIA === "1",
|
|
@@ -50,6 +84,10 @@ const PRIVATE_HINT =
|
|
|
50
84
|
"private/loopback hosts are blocked by default; start the server with " +
|
|
51
85
|
"--allow-private-media (or MLX_BUN_ALLOW_PRIVATE_MEDIA=1) to fetch from LAN hosts";
|
|
52
86
|
|
|
87
|
+
function privateHint(policy: MediaFetchPolicy): string {
|
|
88
|
+
return policy.privateDestinationHint ?? PRIVATE_HINT;
|
|
89
|
+
}
|
|
90
|
+
|
|
53
91
|
function parseIpv4(s: string): number[] | null {
|
|
54
92
|
const parts = s.split(".");
|
|
55
93
|
if (parts.length !== 4) return null;
|
|
@@ -155,19 +193,22 @@ export function checkMediaUrl(u: URL, policy: MediaFetchPolicy): string | null {
|
|
|
155
193
|
if (policy.allowPrivate) return null;
|
|
156
194
|
const lower = host.toLowerCase();
|
|
157
195
|
if (lower === "localhost" || lower.endsWith(".localhost"))
|
|
158
|
-
return `destination "${host}" — ${
|
|
196
|
+
return `destination "${host}" — ${privateHint(policy)}`;
|
|
159
197
|
// WHATWG URL parsing already canonicalized numeric IPv4 forms
|
|
160
198
|
// (http://2130706433/, http://0x7f000001/ → 127.0.0.1).
|
|
161
199
|
if (parseIpv4(lower) || parseIpv6(lower)) {
|
|
162
|
-
if (isBlockedAddress(lower))
|
|
200
|
+
if (isBlockedAddress(lower))
|
|
201
|
+
return `destination "${host}" — ${privateHint(policy)}`;
|
|
163
202
|
}
|
|
164
203
|
return null;
|
|
165
204
|
}
|
|
166
205
|
|
|
167
|
-
async function assertResolvesPublic(
|
|
206
|
+
export async function assertResolvesPublic(
|
|
168
207
|
hostname: string,
|
|
169
208
|
kind: string,
|
|
170
209
|
policy: MediaFetchPolicy,
|
|
210
|
+
resolve: ResolveHost = async (host) =>
|
|
211
|
+
await lookup(host, { all: true, verbatim: true }),
|
|
171
212
|
): Promise<void> {
|
|
172
213
|
if (policy.allowPrivate) return;
|
|
173
214
|
const host =
|
|
@@ -177,52 +218,105 @@ async function assertResolvesPublic(
|
|
|
177
218
|
if (parseIpv4(host) || parseIpv6(host)) return; // literal — checked sync
|
|
178
219
|
let addrs: { address: string }[];
|
|
179
220
|
try {
|
|
180
|
-
addrs = await
|
|
221
|
+
addrs = [...await resolve(host)];
|
|
181
222
|
} catch {
|
|
182
223
|
throw new Error(`${kind} fetch failed: could not resolve host "${host}"`);
|
|
183
224
|
}
|
|
184
225
|
for (const a of addrs) {
|
|
185
226
|
if (isBlockedAddress(a.address))
|
|
186
227
|
throw new Error(
|
|
187
|
-
`${kind} url rejected: host "${host}" resolves to ${a.address} — ${
|
|
228
|
+
`${kind} url rejected: host "${host}" resolves to ${a.address} — ${privateHint(policy)}`,
|
|
188
229
|
);
|
|
189
230
|
}
|
|
190
231
|
}
|
|
191
232
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
)
|
|
203
|
-
if (
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
return Uint8Array.from(Buffer.from(body, "base64"));
|
|
233
|
+
async function resolveFetchAddresses(
|
|
234
|
+
hostname: string,
|
|
235
|
+
kind: string,
|
|
236
|
+
policy: MediaFetchPolicy,
|
|
237
|
+
resolve: ResolveHost,
|
|
238
|
+
): Promise<string[]> {
|
|
239
|
+
const host =
|
|
240
|
+
hostname.startsWith("[") && hostname.endsWith("]")
|
|
241
|
+
? hostname.slice(1, -1)
|
|
242
|
+
: hostname;
|
|
243
|
+
if (parseIpv4(host) || parseIpv6(host)) return [host];
|
|
244
|
+
if (policy.allowPrivate) return [];
|
|
245
|
+
let addrs: { address: string }[];
|
|
246
|
+
try {
|
|
247
|
+
addrs = [...await resolve(host)];
|
|
248
|
+
} catch {
|
|
249
|
+
throw new Error(`${kind} fetch failed: could not resolve host "${host}"`);
|
|
210
250
|
}
|
|
251
|
+
if (addrs.length === 0)
|
|
252
|
+
throw new Error(`${kind} fetch failed: host "${host}" resolved to no addresses`);
|
|
253
|
+
for (const a of addrs) {
|
|
254
|
+
if (isBlockedAddress(a.address))
|
|
255
|
+
throw new Error(
|
|
256
|
+
`${kind} url rejected: host "${host}" resolves to ${a.address} — ${privateHint(policy)}`,
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
return addrs.map((a) => a.address);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Fetch an http(s) response through the shared public-destination policy.
|
|
264
|
+
* Every redirect is handled manually and revalidated, caller cancellation
|
|
265
|
+
* shares one signal with the wall-clock timeout, and the body is streamed
|
|
266
|
+
* into a byte-capped buffer.
|
|
267
|
+
*/
|
|
268
|
+
export async function fetchRestrictedHttpBytes(
|
|
269
|
+
url: string,
|
|
270
|
+
kind: string,
|
|
271
|
+
policy: MediaFetchPolicy,
|
|
272
|
+
options: RestrictedFetchOptions = {},
|
|
273
|
+
): Promise<RestrictedFetchResult> {
|
|
211
274
|
let current: URL;
|
|
212
275
|
try {
|
|
213
276
|
current = new URL(url);
|
|
214
277
|
} catch {
|
|
215
278
|
throw new Error(`unsupported ${kind} url scheme: ${url.slice(0, 16)}`);
|
|
216
279
|
}
|
|
280
|
+
|
|
281
|
+
const fetchImpl = options.dependencies?.fetch ??
|
|
282
|
+
((input: string | URL, init?: BunFetchRequestInit) => fetch(input, init));
|
|
283
|
+
const resolve = options.dependencies?.resolve ??
|
|
284
|
+
(async (host: string) => await lookup(host, { all: true, verbatim: true }));
|
|
285
|
+
const timeout = AbortSignal.timeout(policy.timeoutMs);
|
|
286
|
+
const signal = options.signal
|
|
287
|
+
? AbortSignal.any([options.signal, timeout])
|
|
288
|
+
: timeout;
|
|
217
289
|
const capMb = Math.round(policy.maxBytes / (1024 * 1024));
|
|
218
|
-
const signal = AbortSignal.timeout(policy.timeoutMs);
|
|
219
290
|
let overCap = false;
|
|
291
|
+
|
|
220
292
|
try {
|
|
221
293
|
for (let hop = 0; ; hop++) {
|
|
222
294
|
const reason = checkMediaUrl(current, policy);
|
|
223
295
|
if (reason) throw new Error(`${kind} url rejected: ${reason}`);
|
|
224
|
-
await
|
|
225
|
-
|
|
296
|
+
const addresses = await resolveFetchAddresses(
|
|
297
|
+
current.hostname,
|
|
298
|
+
kind,
|
|
299
|
+
policy,
|
|
300
|
+
resolve,
|
|
301
|
+
);
|
|
302
|
+
const logicalHostname = current.hostname.replace(/^\[|\]$/g, "");
|
|
303
|
+
const connectUrl = new URL(current);
|
|
304
|
+
if (!policy.allowPrivate && addresses.length > 0) {
|
|
305
|
+
const address = addresses[0]!;
|
|
306
|
+
connectUrl.hostname = address.includes(":") ? `[${address}]` : address;
|
|
307
|
+
}
|
|
308
|
+
const headers = new Headers(options.headers);
|
|
309
|
+
if (connectUrl.hostname !== current.hostname && !headers.has("host"))
|
|
310
|
+
headers.set("host", current.host);
|
|
311
|
+
const init: BunFetchRequestInit = {
|
|
312
|
+
redirect: "manual",
|
|
313
|
+
signal,
|
|
314
|
+
headers,
|
|
315
|
+
};
|
|
316
|
+
if (current.protocol === "https:" && connectUrl.hostname !== current.hostname) {
|
|
317
|
+
init.tls = { serverName: logicalHostname };
|
|
318
|
+
}
|
|
319
|
+
const res = await fetchImpl(connectUrl, init);
|
|
226
320
|
if (res.status >= 300 && res.status < 400) {
|
|
227
321
|
const loc = res.headers.get("location");
|
|
228
322
|
if (!loc)
|
|
@@ -239,46 +333,94 @@ export async function fetchMediaBytes(
|
|
|
239
333
|
}
|
|
240
334
|
continue;
|
|
241
335
|
}
|
|
242
|
-
|
|
336
|
+
|
|
337
|
+
if (options.rejectHttpErrors && !res.ok) {
|
|
338
|
+
try {
|
|
339
|
+
await res.body?.cancel();
|
|
340
|
+
} catch {}
|
|
341
|
+
throw new Error(`${kind} fetch failed: ${res.status} ${current}`);
|
|
342
|
+
}
|
|
343
|
+
|
|
243
344
|
const declared = Number(res.headers.get("content-length") ?? "0");
|
|
244
|
-
if (declared > policy.maxBytes) {
|
|
345
|
+
if (Number.isFinite(declared) && declared > policy.maxBytes) {
|
|
245
346
|
overCap = true;
|
|
246
347
|
try {
|
|
247
348
|
await res.body?.cancel();
|
|
248
349
|
} catch {}
|
|
249
350
|
throw new Error(`${kind} response exceeds the ${capMb} MB limit`);
|
|
250
351
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
for (;;) {
|
|
257
|
-
const { done, value } = await reader.read();
|
|
258
|
-
if (done) break;
|
|
259
|
-
total += value.byteLength;
|
|
260
|
-
if (total > policy.maxBytes) {
|
|
352
|
+
|
|
353
|
+
let bytes: Uint8Array;
|
|
354
|
+
if (!res.body) {
|
|
355
|
+
bytes = new Uint8Array(await res.arrayBuffer());
|
|
356
|
+
if (bytes.byteLength > policy.maxBytes) {
|
|
261
357
|
overCap = true;
|
|
262
|
-
try {
|
|
263
|
-
await reader.cancel();
|
|
264
|
-
} catch {}
|
|
265
358
|
throw new Error(`${kind} response exceeds the ${capMb} MB limit`);
|
|
266
359
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
360
|
+
} else {
|
|
361
|
+
const reader = res.body.getReader();
|
|
362
|
+
const chunks: Uint8Array[] = [];
|
|
363
|
+
let total = 0;
|
|
364
|
+
for (;;) {
|
|
365
|
+
const { done, value } = await reader.read();
|
|
366
|
+
if (done) break;
|
|
367
|
+
total += value.byteLength;
|
|
368
|
+
if (total > policy.maxBytes) {
|
|
369
|
+
overCap = true;
|
|
370
|
+
try {
|
|
371
|
+
await reader.cancel();
|
|
372
|
+
} catch {}
|
|
373
|
+
throw new Error(`${kind} response exceeds the ${capMb} MB limit`);
|
|
374
|
+
}
|
|
375
|
+
chunks.push(value);
|
|
376
|
+
}
|
|
377
|
+
bytes = new Uint8Array(total);
|
|
378
|
+
let offset = 0;
|
|
379
|
+
for (const chunk of chunks) {
|
|
380
|
+
bytes.set(chunk, offset);
|
|
381
|
+
offset += chunk.byteLength;
|
|
382
|
+
}
|
|
274
383
|
}
|
|
275
|
-
|
|
384
|
+
|
|
385
|
+
return {
|
|
386
|
+
ok: res.ok,
|
|
387
|
+
status: res.status,
|
|
388
|
+
finalUrl: current.href,
|
|
389
|
+
contentType: res.headers.get("content-type") ?? "",
|
|
390
|
+
bytes,
|
|
391
|
+
};
|
|
276
392
|
}
|
|
277
|
-
} catch (
|
|
278
|
-
if (
|
|
393
|
+
} catch (error) {
|
|
394
|
+
if (timeout.aborted && !options.signal?.aborted && !overCap)
|
|
279
395
|
throw new Error(
|
|
280
396
|
`${kind} fetch timed out after ${policy.timeoutMs} ms: ${url.slice(0, 200)}`,
|
|
281
397
|
);
|
|
282
|
-
throw
|
|
398
|
+
throw error;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** Fetch image/audio bytes for an OpenAI-style image_url/audio_url part.
|
|
403
|
+
* data: URLs decode locally (no policy — the bytes came in the request);
|
|
404
|
+
* http(s) URLs go through the destination policy above, with every
|
|
405
|
+
* redirect hop re-validated, one wall-clock timeout, and a streaming
|
|
406
|
+
* size cap. Throws Error with a client-presentable message — callers
|
|
407
|
+
* surface it as a 400 (the prompt-build catch in server.ts). */
|
|
408
|
+
export async function fetchMediaBytes(
|
|
409
|
+
url: string,
|
|
410
|
+
kind: "image" | "audio",
|
|
411
|
+
policy: MediaFetchPolicy = defaultMediaFetchPolicy(),
|
|
412
|
+
): Promise<Uint8Array> {
|
|
413
|
+
if (url.startsWith("data:")) {
|
|
414
|
+
const comma = url.indexOf(",");
|
|
415
|
+
if (comma === -1) throw new Error("malformed data: URL");
|
|
416
|
+
const meta = url.slice(0, comma);
|
|
417
|
+
const body = url.slice(comma + 1);
|
|
418
|
+
if (!meta.includes("base64")) throw new Error("data: URL must be base64");
|
|
419
|
+
return Uint8Array.from(Buffer.from(body, "base64"));
|
|
283
420
|
}
|
|
421
|
+
return (
|
|
422
|
+
await fetchRestrictedHttpBytes(url, kind, policy, {
|
|
423
|
+
rejectHttpErrors: true,
|
|
424
|
+
})
|
|
425
|
+
).bytes;
|
|
284
426
|
}
|
package/src/mlx/array.ts
CHANGED
|
@@ -292,6 +292,49 @@ export class MlxArray {
|
|
|
292
292
|
return out;
|
|
293
293
|
}
|
|
294
294
|
|
|
295
|
+
/** Read an f32/f16/bf16 array numerically as float32 without enqueueing a
|
|
296
|
+
* GPU cast. This evaluates the ORIGINAL array, then expands 16-bit values
|
|
297
|
+
* on the host. Use this for small readbacks after later GPU work has
|
|
298
|
+
* already been dispatched: toFloat32() would append astype(float32) behind
|
|
299
|
+
* that work and turn the readback into an accidental full-step barrier. */
|
|
300
|
+
toFloat32Host(): Float32Array {
|
|
301
|
+
this.eval();
|
|
302
|
+
const dt = this.dtype;
|
|
303
|
+
if (dt === Dtype.float32) {
|
|
304
|
+
const p = C.mlx_array_data_float32(this.handle);
|
|
305
|
+
return new Float32Array(toArrayBuffer(p!, 0, this.size * 4)).slice();
|
|
306
|
+
}
|
|
307
|
+
if (dt === Dtype.bfloat16) {
|
|
308
|
+
const p = C.mlx_array_data_bfloat16(this.handle);
|
|
309
|
+
const src = new Uint16Array(toArrayBuffer(p!, 0, this.size * 2));
|
|
310
|
+
const words = new Uint32Array(src.length);
|
|
311
|
+
for (let i = 0; i < src.length; i++) words[i] = src[i]! << 16;
|
|
312
|
+
return new Float32Array(words.buffer);
|
|
313
|
+
}
|
|
314
|
+
if (dt === Dtype.float16) {
|
|
315
|
+
const p = C.mlx_array_data_float16(this.handle);
|
|
316
|
+
const src = new Uint16Array(toArrayBuffer(p!, 0, this.size * 2));
|
|
317
|
+
const out = new Float32Array(src.length);
|
|
318
|
+
for (let i = 0; i < src.length; i++) {
|
|
319
|
+
const bits = src[i]!;
|
|
320
|
+
const sign = bits & 0x8000;
|
|
321
|
+
const exponent = (bits >>> 10) & 0x1f;
|
|
322
|
+
const fraction = bits & 0x03ff;
|
|
323
|
+
let value: number;
|
|
324
|
+
if (exponent === 0) {
|
|
325
|
+
value = fraction === 0 ? 0 : fraction * 2 ** -24;
|
|
326
|
+
} else if (exponent === 0x1f) {
|
|
327
|
+
value = fraction === 0 ? Infinity : NaN;
|
|
328
|
+
} else {
|
|
329
|
+
value = (1 + fraction / 1024) * 2 ** (exponent - 15);
|
|
330
|
+
}
|
|
331
|
+
out[i] = sign ? -value : value;
|
|
332
|
+
}
|
|
333
|
+
return out;
|
|
334
|
+
}
|
|
335
|
+
throw new Error(`toFloat32Host: unsupported dtype ${this.dtypeName}`);
|
|
336
|
+
}
|
|
337
|
+
|
|
295
338
|
/** Basic slice: start/stop per dimension (stride 1). */
|
|
296
339
|
slice(start: number[], stop: number[], stream: MlxHandle = gpuStream): MlxArray {
|
|
297
340
|
const s = new Int32Array(start);
|
package/src/mlx/autograd.ts
CHANGED
|
@@ -15,6 +15,23 @@ import { JSCallback, ptr, read } from "bun:ffi";
|
|
|
15
15
|
import { C, takeMlxError } from "./ffi";
|
|
16
16
|
import { MlxArray } from "./array";
|
|
17
17
|
|
|
18
|
+
type CallbackHandle = Pick<JSCallback, "ptr" | "close">;
|
|
19
|
+
type NativeCall<F extends (...args: any[]) => any> =
|
|
20
|
+
(...args: Parameters<F>) => ReturnType<F>;
|
|
21
|
+
|
|
22
|
+
/** Constructor-only dependency seam for deterministic native failure tests. */
|
|
23
|
+
export interface ValueAndGradConstructorHooks {
|
|
24
|
+
callback?: (
|
|
25
|
+
callback: ConstructorParameters<typeof JSCallback>[0],
|
|
26
|
+
definition: ConstructorParameters<typeof JSCallback>[1],
|
|
27
|
+
) => CallbackHandle;
|
|
28
|
+
closureNew?: NativeCall<typeof C.mlx_closure_new_func_payload>;
|
|
29
|
+
transformedNew?: NativeCall<typeof C.mlx_closure_value_and_grad_new>;
|
|
30
|
+
transform?: NativeCall<typeof C.mlx_value_and_grad>;
|
|
31
|
+
transformedFree?: NativeCall<typeof C.mlx_closure_value_and_grad_free>;
|
|
32
|
+
closureFree?: NativeCall<typeof C.mlx_closure_free>;
|
|
33
|
+
}
|
|
34
|
+
|
|
18
35
|
/** A reusable value-and-grad over a JS-built loss graph.
|
|
19
36
|
*
|
|
20
37
|
* `loss(primals)` receives the input arrays in argument order and must
|
|
@@ -28,18 +45,30 @@ import { MlxArray } from "./array";
|
|
|
28
45
|
* are owned by mlx — do NOT dispose them; the returned loss IS disposed
|
|
29
46
|
* of by mlx's graph. */
|
|
30
47
|
export class ValueAndGrad {
|
|
31
|
-
readonly #cb:
|
|
48
|
+
readonly #cb: CallbackHandle;
|
|
32
49
|
readonly #closure: bigint;
|
|
33
50
|
readonly #vag: bigint;
|
|
34
51
|
readonly #argIdx: Int32Array;
|
|
52
|
+
readonly #freeClosure: NativeCall<typeof C.mlx_closure_free>;
|
|
53
|
+
readonly #freeVag: NativeCall<typeof C.mlx_closure_value_and_grad_free>;
|
|
35
54
|
#closureError: string | null = null;
|
|
36
55
|
#disposed = false;
|
|
37
56
|
|
|
38
|
-
constructor(
|
|
57
|
+
constructor(
|
|
58
|
+
loss: (primals: MlxArray[]) => MlxArray,
|
|
59
|
+
argIdx: number[],
|
|
60
|
+
hooks: ValueAndGradConstructorHooks = {},
|
|
61
|
+
) {
|
|
39
62
|
if (argIdx.length === 0) throw new Error("ValueAndGrad: argIdx must be non-empty");
|
|
40
63
|
this.#argIdx = new Int32Array(argIdx);
|
|
64
|
+
const makeCallback = hooks.callback ?? ((callback, definition) => new JSCallback(callback, definition));
|
|
65
|
+
const closureNew = hooks.closureNew ?? C.mlx_closure_new_func_payload;
|
|
66
|
+
const transformedNew = hooks.transformedNew ?? C.mlx_closure_value_and_grad_new;
|
|
67
|
+
const transform = hooks.transform ?? C.mlx_value_and_grad;
|
|
68
|
+
this.#freeClosure = hooks.closureFree ?? C.mlx_closure_free;
|
|
69
|
+
this.#freeVag = hooks.transformedFree ?? C.mlx_closure_value_and_grad_free;
|
|
41
70
|
|
|
42
|
-
|
|
71
|
+
const cb = makeCallback(
|
|
43
72
|
(outPtr: number, inVec: bigint, _payload: number): number => {
|
|
44
73
|
try {
|
|
45
74
|
// Read the primals out of the input vector_array. mlx_vector_array_get
|
|
@@ -73,16 +102,24 @@ export class ValueAndGrad {
|
|
|
73
102
|
{ args: ["ptr", "u64", "ptr"], returns: "i32" },
|
|
74
103
|
);
|
|
75
104
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
105
|
+
let closure: bigint | null = null;
|
|
106
|
+
let vagSlot: BigUint64Array | null = null;
|
|
107
|
+
try {
|
|
108
|
+
closure = closureNew(cb.ptr as never, null, null);
|
|
109
|
+
vagSlot = new BigUint64Array([transformedNew()]);
|
|
110
|
+
if (transform(ptr(vagSlot), closure, ptr(this.#argIdx), BigInt(this.#argIdx.length)) !== 0)
|
|
111
|
+
throw new Error("mlx_value_and_grad failed");
|
|
112
|
+
this.#cb = cb;
|
|
113
|
+
this.#closure = closure;
|
|
114
|
+
this.#vag = read.u64(ptr(vagSlot), 0);
|
|
115
|
+
} catch (e) {
|
|
116
|
+
// mlx-c out-params are initialized live handles. On failure the slot
|
|
117
|
+
// still owns whichever handle it contains and must be freed explicitly.
|
|
118
|
+
if (vagSlot) this.#freeVag(read.u64(ptr(vagSlot), 0));
|
|
119
|
+
if (closure !== null) this.#freeClosure(closure);
|
|
120
|
+
cb.close();
|
|
121
|
+
throw e;
|
|
84
122
|
}
|
|
85
|
-
this.#vag = read.u64(ptr(vagSlot), 0);
|
|
86
123
|
}
|
|
87
124
|
|
|
88
125
|
/** Run the loss + backward on `primals` (in argument order). Returns the
|
|
@@ -136,8 +173,8 @@ export class ValueAndGrad {
|
|
|
136
173
|
dispose(): void {
|
|
137
174
|
if (this.#disposed) return;
|
|
138
175
|
this.#disposed = true;
|
|
139
|
-
|
|
140
|
-
|
|
176
|
+
this.#freeVag(this.#vag);
|
|
177
|
+
this.#freeClosure(this.#closure);
|
|
141
178
|
this.#cb.close();
|
|
142
179
|
}
|
|
143
180
|
|
package/src/mlx/checkpoint.ts
CHANGED
|
@@ -15,6 +15,22 @@ import { JSCallback, ptr, read } from "bun:ffi";
|
|
|
15
15
|
import { C, takeMlxError } from "./ffi";
|
|
16
16
|
import { MlxArray } from "./array";
|
|
17
17
|
|
|
18
|
+
type CallbackHandle = Pick<JSCallback, "ptr" | "close">;
|
|
19
|
+
type NativeCall<F extends (...args: any[]) => any> =
|
|
20
|
+
(...args: Parameters<F>) => ReturnType<F>;
|
|
21
|
+
|
|
22
|
+
/** Constructor-only dependency seam for deterministic native failure tests. */
|
|
23
|
+
export interface CheckpointConstructorHooks {
|
|
24
|
+
callback?: (
|
|
25
|
+
callback: ConstructorParameters<typeof JSCallback>[0],
|
|
26
|
+
definition: ConstructorParameters<typeof JSCallback>[1],
|
|
27
|
+
) => CallbackHandle;
|
|
28
|
+
closureNew?: NativeCall<typeof C.mlx_closure_new_func_payload>;
|
|
29
|
+
transformedNew?: NativeCall<typeof C.mlx_closure_new>;
|
|
30
|
+
transform?: NativeCall<typeof C.mlx_checkpoint>;
|
|
31
|
+
closureFree?: NativeCall<typeof C.mlx_closure_free>;
|
|
32
|
+
}
|
|
33
|
+
|
|
18
34
|
/** A reusable checkpointed wrapper over a JS-built subgraph.
|
|
19
35
|
*
|
|
20
36
|
* `fn(inputs)` receives the input arrays (in order) and returns one or more
|
|
@@ -24,14 +40,24 @@ import { MlxArray } from "./array";
|
|
|
24
40
|
* AND to any outer-trace leaves captured by `fn` (e.g. swapped-in LoRA
|
|
25
41
|
* primals), exactly as if `fn` had been inlined. */
|
|
26
42
|
export class Checkpoint {
|
|
27
|
-
readonly #cb:
|
|
43
|
+
readonly #cb: CallbackHandle;
|
|
28
44
|
readonly #closure: bigint;
|
|
29
45
|
readonly #ckpt: bigint;
|
|
46
|
+
readonly #freeClosure: NativeCall<typeof C.mlx_closure_free>;
|
|
30
47
|
#closureError: string | null = null;
|
|
31
48
|
#disposed = false;
|
|
32
49
|
|
|
33
|
-
constructor(
|
|
34
|
-
|
|
50
|
+
constructor(
|
|
51
|
+
fn: (inputs: MlxArray[]) => MlxArray[],
|
|
52
|
+
hooks: CheckpointConstructorHooks = {},
|
|
53
|
+
) {
|
|
54
|
+
const makeCallback = hooks.callback ?? ((callback, definition) => new JSCallback(callback, definition));
|
|
55
|
+
const closureNew = hooks.closureNew ?? C.mlx_closure_new_func_payload;
|
|
56
|
+
const transformedNew = hooks.transformedNew ?? C.mlx_closure_new;
|
|
57
|
+
const transform = hooks.transform ?? C.mlx_checkpoint;
|
|
58
|
+
this.#freeClosure = hooks.closureFree ?? C.mlx_closure_free;
|
|
59
|
+
|
|
60
|
+
const cb = makeCallback(
|
|
35
61
|
(outPtr: number, inVec: bigint, _payload: number): number => {
|
|
36
62
|
try {
|
|
37
63
|
const n = Number(C.mlx_vector_array_size(inVec));
|
|
@@ -58,15 +84,22 @@ export class Checkpoint {
|
|
|
58
84
|
{ args: ["ptr", "u64", "ptr"], returns: "i32" },
|
|
59
85
|
);
|
|
60
86
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
87
|
+
let closure: bigint | null = null;
|
|
88
|
+
let slot: BigUint64Array | null = null;
|
|
89
|
+
try {
|
|
90
|
+
closure = closureNew(cb.ptr as never, null, null);
|
|
91
|
+
slot = new BigUint64Array([transformedNew()]);
|
|
92
|
+
if (transform(ptr(slot), closure) !== 0)
|
|
93
|
+
throw new Error(`mlx_checkpoint failed: ${takeMlxError() ?? ""}`);
|
|
94
|
+
this.#cb = cb;
|
|
95
|
+
this.#closure = closure;
|
|
96
|
+
this.#ckpt = read.u64(ptr(slot), 0);
|
|
97
|
+
} catch (e) {
|
|
98
|
+
if (slot) this.#freeClosure(read.u64(ptr(slot), 0));
|
|
99
|
+
if (closure !== null) this.#freeClosure(closure);
|
|
100
|
+
cb.close();
|
|
101
|
+
throw e;
|
|
68
102
|
}
|
|
69
|
-
this.#ckpt = read.u64(ptr(slot), 0);
|
|
70
103
|
}
|
|
71
104
|
|
|
72
105
|
/** Apply the checkpointed fn. Returns its outputs as owned MlxArrays. */
|
|
@@ -105,8 +138,8 @@ export class Checkpoint {
|
|
|
105
138
|
dispose(): void {
|
|
106
139
|
if (this.#disposed) return;
|
|
107
140
|
this.#disposed = true;
|
|
108
|
-
|
|
109
|
-
|
|
141
|
+
this.#freeClosure(this.#ckpt);
|
|
142
|
+
this.#freeClosure(this.#closure);
|
|
110
143
|
this.#cb.close();
|
|
111
144
|
}
|
|
112
145
|
|