mlx-bun 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +362 -0
- package/THIRD_PARTY_LICENSES.md +41 -0
- package/bin/mlx-bun.mjs +32 -0
- package/docs/curve-designer.html +303 -0
- package/docs/design/adapters-end-to-end.md +201 -0
- package/docs/design/compat-cli-surface-design.md +227 -0
- package/docs/design/docs-reorg-plan.md +152 -0
- package/docs/design/hlg-sampling.md +340 -0
- package/docs/design/memory-system.md +339 -0
- package/docs/design/optimization_plan.md +344 -0
- package/docs/design/parallel-slots.md +515 -0
- package/docs/design/runtime-isolation.md +247 -0
- package/docs/design/segmented-backward-training.md +508 -0
- package/docs/investigations/chunk-finetune-experiment.md +84 -0
- package/docs/investigations/curve-bisector-routing.md +123 -0
- package/docs/investigations/curve-runs/bisector-route.json +98 -0
- package/docs/investigations/curve-runs/bisector-trace.json +1393 -0
- package/docs/investigations/curve-runs/curve-terrain.json +1179 -0
- package/docs/investigations/curve-runs/curve-terrain.svg +42 -0
- package/docs/investigations/curve-terrain.html +105 -0
- package/docs/investigations/expert-offload-single-user-moe.md +292 -0
- package/docs/investigations/hlg-ab.html +34 -0
- package/docs/investigations/hlg-explorer.html +238 -0
- package/docs/investigations/hlg-figs/cliff.svg +1 -0
- package/docs/investigations/hlg-figs/cross.svg +1 -0
- package/docs/investigations/hlg-figs/edges.svg +1 -0
- package/docs/investigations/hlg-figs/k20-grids.svg +1 -0
- package/docs/investigations/hlg-figs/k20-peaks.svg +1 -0
- package/docs/investigations/hlg-figs/lw-curve.svg +1 -0
- package/docs/investigations/hlg-figs/terrain.svg +1 -0
- package/docs/investigations/hlg-report.html +262 -0
- package/docs/investigations/hlg-runs/hlg-ab.json +149 -0
- package/docs/investigations/hlg-runs/hlg-blind-prompts.md +87 -0
- package/docs/investigations/hlg-runs/hlg-blind.json +107 -0
- package/docs/investigations/hlg-runs/hlg-cliff.json +197 -0
- package/docs/investigations/hlg-runs/hlg-cross.json +505 -0
- package/docs/investigations/hlg-runs/hlg-edge.json +388 -0
- package/docs/investigations/hlg-runs/hlg-eval1.json +25 -0
- package/docs/investigations/hlg-runs/hlg-frontier.json +86 -0
- package/docs/investigations/hlg-runs/hlg-grid.json +242 -0
- package/docs/investigations/hlg-runs/hlg-grid2.json +343 -0
- package/docs/investigations/hlg-runs/hlg-lw-coarse.json +110 -0
- package/docs/investigations/hlg-runs/hlg-map.json +383 -0
- package/docs/investigations/hlg-runs/run0-original.log +159 -0
- package/docs/investigations/hlg-runs/runA-recommended.log +131 -0
- package/docs/investigations/hlg-runs/runAA-cliff-cpm5.log +26 -0
- package/docs/investigations/hlg-runs/runAB-grid2-e4b.log +18 -0
- package/docs/investigations/hlg-runs/runAC-peak-e4b-partial.log +25 -0
- package/docs/investigations/hlg-runs/runAD-ab-samples.log +8 -0
- package/docs/investigations/hlg-runs/runAE-edge-e4b.log +51 -0
- package/docs/investigations/hlg-runs/runAF-logit-calibration.log +10 -0
- package/docs/investigations/hlg-runs/runAG-eval-config2.log +16 -0
- package/docs/investigations/hlg-runs/runB-gain-variants.log +192 -0
- package/docs/investigations/hlg-runs/runC-pivots.log +162 -0
- package/docs/investigations/hlg-runs/runD-oetf.log +132 -0
- package/docs/investigations/hlg-runs/runE-eotf.log +162 -0
- package/docs/investigations/hlg-runs/runF-shaper-working.log +86 -0
- package/docs/investigations/hlg-runs/runG-shaper-sweep.log +101 -0
- package/docs/investigations/hlg-runs/runH-wsweep.log +176 -0
- package/docs/investigations/hlg-runs/runI-asweep.log +176 -0
- package/docs/investigations/hlg-runs/runJ-knobmap.log +49 -0
- package/docs/investigations/hlg-runs/runK-frontier.log +19 -0
- package/docs/investigations/hlg-runs/runL-widemap.log +65 -0
- package/docs/investigations/hlg-runs/runM-frontier-emb.log +15 -0
- package/docs/investigations/hlg-runs/runN-verify.log +116 -0
- package/docs/investigations/hlg-runs/runO-frontier-seed5000.log +15 -0
- package/docs/investigations/hlg-runs/runP-grid-AxSm.log +15 -0
- package/docs/investigations/hlg-runs/runQ-grid-tgW.log +15 -0
- package/docs/investigations/hlg-runs/runR-blind-transcript.log +79 -0
- package/docs/investigations/hlg-runs/runS-cross-coherence.log +33 -0
- package/docs/investigations/hlg-runs/runT-cross-cpm5.log +33 -0
- package/docs/investigations/hlg-runs/runU-cross-12b.log +33 -0
- package/docs/investigations/hlg-runs/runV-lw-e4b.log +22 -0
- package/docs/investigations/hlg-runs/runW-lw-12b.log +22 -0
- package/docs/investigations/hlg-runs/runX-lw-cpm5.log +22 -0
- package/docs/investigations/hlg-runs/runY-cliff-e4b.log +26 -0
- package/docs/investigations/hlg-runs/runZ-cliff-12b.log +26 -0
- package/docs/investigations/hlg-sampling-investigation.md +970 -0
- package/docs/investigations/lab-build-journal.md +157 -0
- package/docs/investigations/pi-builtin-investigation.md +308 -0
- package/docs/investigations/starter-model-port-handoff.md +323 -0
- package/docs/planning/IDEAS.md +26 -0
- package/docs/planning/PRODUCT_ROADMAP.md +426 -0
- package/docs/planning/ResearchTopics.md +119 -0
- package/docs/planning/chunk-task-roadmap.md +77 -0
- package/docs/planning/journal.md +155 -0
- package/docs/reference/distribution.md +121 -0
- package/docs/reference/embedding.md +168 -0
- package/docs/reference/library-api.md +153 -0
- package/docs/reference/server-api.md +224 -0
- package/docs/reference/server-config.md +243 -0
- package/docs/segmented-backward-handoff.md +148 -0
- package/package.json +68 -0
- package/src/anthropic.ts +491 -0
- package/src/chat-template.ts +186 -0
- package/src/cli.ts +1075 -0
- package/src/config.ts +229 -0
- package/src/curve-sampler.ts +122 -0
- package/src/dataset/generators.ts +1087 -0
- package/src/dataset/index.ts +12 -0
- package/src/dataset/job.ts +81 -0
- package/src/dataset/llm.ts +71 -0
- package/src/dataset/registry.ts +528 -0
- package/src/download.ts +283 -0
- package/src/eval/README.md +86 -0
- package/src/eval/kl-prompts.ts +29 -0
- package/src/eval/kl.ts +294 -0
- package/src/eval/quality-db.ts +92 -0
- package/src/eval/runner.ts +102 -0
- package/src/eval/score.ts +49 -0
- package/src/eval/tasks/bfcl.ts +500 -0
- package/src/eval/tasks/gsm8k.ts +85 -0
- package/src/eval/tasks/hashhop.ts +109 -0
- package/src/eval/tasks/humaneval.ts +252 -0
- package/src/eval/tasks/ifeval.ts +354 -0
- package/src/eval/tasks/mmlu.ts +145 -0
- package/src/evaldb.ts +199 -0
- package/src/expert-offload-build.ts +77 -0
- package/src/expert-offload.ts +61 -0
- package/src/expert-trace.ts +67 -0
- package/src/fit.ts +254 -0
- package/src/generate.ts +451 -0
- package/src/harness-pi.ts +177 -0
- package/src/hf-push.ts +628 -0
- package/src/index.ts +22 -0
- package/src/jobs/db.ts +139 -0
- package/src/jobs/index.ts +29 -0
- package/src/jobs/job-entry.ts +106 -0
- package/src/jobs/runner.ts +304 -0
- package/src/jobs/sse.ts +114 -0
- package/src/jobs/types.ts +58 -0
- package/src/kv-store.ts +185 -0
- package/src/lora.ts +314 -0
- package/src/md-modules.d.ts +7 -0
- package/src/mlx/array.ts +237 -0
- package/src/mlx/autograd.ts +267 -0
- package/src/mlx/checkpoint.ts +116 -0
- package/src/mlx/compile.ts +158 -0
- package/src/mlx/custom-vjp.ts +140 -0
- package/src/mlx/ffi.ts +433 -0
- package/src/mlx/metal-kernel.ts +130 -0
- package/src/mlx/ops.ts +707 -0
- package/src/mmap.ts +90 -0
- package/src/model/batched-mask.ts +204 -0
- package/src/model/batched-rotating.ts +300 -0
- package/src/model/compiled-decode.ts +681 -0
- package/src/model/factory.ts +27 -0
- package/src/model/fingerprint.ts +33 -0
- package/src/model/flash-attention.ts +674 -0
- package/src/model/fused-decode-kernel.ts +305 -0
- package/src/model/fused-geglu-kernel.ts +87 -0
- package/src/model/gemma4-base.ts +1616 -0
- package/src/model/gemma4.ts +1012 -0
- package/src/model/generated/gemma4-12b.ts +526 -0
- package/src/model/generated/gemma4-26b.ts +500 -0
- package/src/model/generated/gemma4-e4b.ts +757 -0
- package/src/model/generated/index.ts +18 -0
- package/src/model/minicpm5.ts +278 -0
- package/src/model/qwen3-delta.ts +224 -0
- package/src/model/qwen3_5.ts +429 -0
- package/src/model/support.ts +46 -0
- package/src/native-pack.ts +152 -0
- package/src/pi-provider.ts +95 -0
- package/src/pi-terminal.ts +264 -0
- package/src/pi-web.ts +924 -0
- package/src/preflight.ts +123 -0
- package/src/prompt-cache.ts +113 -0
- package/src/quantize/allocator.ts +627 -0
- package/src/quantize/calibration.ts +180 -0
- package/src/quantize/config-writer.ts +132 -0
- package/src/quantize/index.ts +57 -0
- package/src/quantize/job.ts +136 -0
- package/src/quantize/quantizer.ts +462 -0
- package/src/quantize/safetensors-writer.ts +171 -0
- package/src/quantize/sensitivity.ts +234 -0
- package/src/registry.ts +336 -0
- package/src/responses.ts +774 -0
- package/src/safetensors.ts +158 -0
- package/src/sampler.ts +517 -0
- package/src/serve/batch-scheduler.ts +362 -0
- package/src/serve/generation-gateway.ts +160 -0
- package/src/server.ts +1891 -0
- package/src/spec/drafter.ts +261 -0
- package/src/spec/generate.ts +221 -0
- package/src/status-page.html +497 -0
- package/src/tokenizer.ts +38 -0
- package/src/tool-call.ts +242 -0
- package/src/train/dataset.ts +461 -0
- package/src/train/export.ts +32 -0
- package/src/train/forward.ts +229 -0
- package/src/train/index.ts +43 -0
- package/src/train/job.ts +148 -0
- package/src/train/lora-params.ts +194 -0
- package/src/train/loss.ts +342 -0
- package/src/train/merge.ts +195 -0
- package/src/train/optimizer.ts +176 -0
- package/src/train/rank.ts +101 -0
- package/src/train/segmented.ts +520 -0
- package/src/train/trainer.ts +600 -0
- package/src/tui.ts +161 -0
- package/src/vision/embedder.ts +168 -0
- package/src/vision/preprocess.ts +247 -0
- package/src/vision/prompt.ts +158 -0
- package/src/vision/siglip.ts +599 -0
- package/src/web/app.html +2471 -0
- package/src/web/skills/web-research/SKILL.md +44 -0
- package/src/web/skills.ts +59 -0
- package/src/web-tools.ts +426 -0
- package/src/weights.ts +91 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: web-research
|
|
3
|
+
description: Use when the user asks you to research something, do a deep dive, fact-check a claim, compare options, or answer a question that needs current, accurate, real-world information ("what's the latest on…", prices, releases, news, specs). Gives a rigorous workflow for searching the web, reading primary sources, cross-checking claims, and answering with citations.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Web Research
|
|
7
|
+
|
|
8
|
+
A workflow for answering questions that need current, accurate, real-world
|
|
9
|
+
information, using the `web_search` and `web_fetch` tools. The goal is a
|
|
10
|
+
trustworthy, well-sourced answer — not a guess dressed up as one.
|
|
11
|
+
|
|
12
|
+
## When to use this
|
|
13
|
+
|
|
14
|
+
- The answer depends on current events, recent releases, prices, versions,
|
|
15
|
+
or anything that changes over time.
|
|
16
|
+
- The user wants a thorough or authoritative answer, or asks for sources.
|
|
17
|
+
- A specific claim needs verifying, or you would otherwise be speculating.
|
|
18
|
+
|
|
19
|
+
## Workflow
|
|
20
|
+
|
|
21
|
+
1. **Plan.** Restate the question in your own words and list the 2–4 concrete
|
|
22
|
+
things you need to find out. This keeps the searches focused.
|
|
23
|
+
2. **Search broadly.** Call `web_search` with a few focused queries — one idea
|
|
24
|
+
per query — rather than a single vague one. Vary the wording if the first
|
|
25
|
+
results are weak.
|
|
26
|
+
3. **Read the best sources.** Use `web_fetch` to actually read the most
|
|
27
|
+
relevant results. Prefer primary sources (official docs, the original
|
|
28
|
+
announcement, the paper, the vendor's own page) over blogs and aggregators.
|
|
29
|
+
Read at least two or three independent sources.
|
|
30
|
+
4. **Cross-check.** Confirm each key fact appears in more than one independent
|
|
31
|
+
source. Where sources disagree, surface the disagreement instead of picking
|
|
32
|
+
one silently.
|
|
33
|
+
5. **Answer.** Lead with a direct answer, then the supporting detail. Cite
|
|
34
|
+
every nontrivial claim with a Markdown link to a page you actually fetched.
|
|
35
|
+
Separate what's well-supported from what's uncertain.
|
|
36
|
+
|
|
37
|
+
## Quality rules
|
|
38
|
+
|
|
39
|
+
- Check publication dates; for fast-moving topics, prefer recent sources.
|
|
40
|
+
- Never cite a page you didn't fetch, and never invent quotes, numbers, or URLs.
|
|
41
|
+
- Quote sparingly and exactly.
|
|
42
|
+
- If the sources are thin or conflicting, say so plainly — don't overstate
|
|
43
|
+
confidence to sound complete.
|
|
44
|
+
- If a search returns nothing useful, try different terms before giving up.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// mlx-bun bundled skills — curated pi skills the web chat ships with.
|
|
2
|
+
//
|
|
3
|
+
// Why materialize to disk? pi loads skills by reading SKILL.md from a
|
|
4
|
+
// filesystem path, but mlx-bun also ships as a single `bun build --compile`
|
|
5
|
+
// binary where source files don't exist on disk. So each SKILL.md is
|
|
6
|
+
// embedded into the bundle as a text import (the same trick app.html uses)
|
|
7
|
+
// and written to ~/.mlx-bun/skills/<name>/SKILL.md at startup. One source of
|
|
8
|
+
// truth that resolves identically when run from source and from the binary —
|
|
9
|
+
// no build-script changes, no sidecar files.
|
|
10
|
+
//
|
|
11
|
+
// These load via DefaultResourceLoader.additionalSkillPaths even though the
|
|
12
|
+
// web session keeps noSkills:true: that flag only disables auto-discovery of
|
|
13
|
+
// the user's personal ~/.pi and project .pi skills, so the chat stays
|
|
14
|
+
// isolated while still getting our curated set (resource-loader.js:277).
|
|
15
|
+
//
|
|
16
|
+
// To add a skill: drop a folder under ./skills/<name>/SKILL.md, add a text
|
|
17
|
+
// import for it, and append it to BUNDLED_SKILLS.
|
|
18
|
+
|
|
19
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
20
|
+
import { homedir } from "node:os";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
import webResearchMd from "./skills/web-research/SKILL.md" with { type: "text" };
|
|
23
|
+
|
|
24
|
+
interface BundledSkill {
|
|
25
|
+
/** Directory name under the skills root; should match the SKILL.md `name`. */
|
|
26
|
+
name: string;
|
|
27
|
+
/** Raw SKILL.md contents, embedded at build time. */
|
|
28
|
+
md: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const BUNDLED_SKILLS: readonly BundledSkill[] = [
|
|
32
|
+
{ name: "web-research", md: webResearchMd },
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
/** Where the bundled skills are written; also pi's per-session scratch root. */
|
|
36
|
+
export function bundledSkillsRoot(): string {
|
|
37
|
+
return join(homedir(), ".mlx-bun", "skills");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Memoize: the embedded content can't change within a process run. */
|
|
41
|
+
let materialized = false;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Write the embedded skills to disk (overwriting, so updates propagate) and
|
|
45
|
+
* return the root directory to pass as a single additionalSkillPaths entry.
|
|
46
|
+
* pi scans it for per-skill SKILL.md files. Runs its disk writes once per
|
|
47
|
+
* process even when called per browser connection.
|
|
48
|
+
*/
|
|
49
|
+
export function materializeBundledSkills(): string {
|
|
50
|
+
const root = bundledSkillsRoot();
|
|
51
|
+
if (materialized) return root;
|
|
52
|
+
for (const skill of BUNDLED_SKILLS) {
|
|
53
|
+
const dir = join(root, skill.name);
|
|
54
|
+
mkdirSync(dir, { recursive: true });
|
|
55
|
+
writeFileSync(join(dir, "SKILL.md"), skill.md);
|
|
56
|
+
}
|
|
57
|
+
materialized = true;
|
|
58
|
+
return root;
|
|
59
|
+
}
|
package/src/web-tools.ts
ADDED
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
// mlx-bun web-tools — outward-facing capabilities for the web chat agent:
|
|
2
|
+
// web_search, web_fetch, and weather. These are pi custom tools (defineTool)
|
|
3
|
+
// registered alongside the built-in coding tools so the local model can pull
|
|
4
|
+
// in current, real-world information instead of only inspecting the machine.
|
|
5
|
+
//
|
|
6
|
+
// Design notes:
|
|
7
|
+
// - Keyless by default. weather uses Open-Meteo (no key, no anti-bot),
|
|
8
|
+
// web_fetch is a plain HTTP GET, and web_search defaults to DuckDuckGo's
|
|
9
|
+
// keyless HTML endpoint. Setting TAVILY_API_KEY or BRAVE_API_KEY upgrades
|
|
10
|
+
// web_search to that provider's JSON API (more reliable, ranked results).
|
|
11
|
+
// - Non-mutating: none of these touch the user's filesystem, so the web
|
|
12
|
+
// chat's approval gate auto-allows them (they're not in GATED_TOOLS).
|
|
13
|
+
// They DO make outbound network requests — that's the one thing that
|
|
14
|
+
// leaves the machine, and only when the user asks for live info.
|
|
15
|
+
// - execute() never throws on a network/upstream failure: it returns the
|
|
16
|
+
// error as tool text so the model can read it and adapt (retry, tell the
|
|
17
|
+
// user, fall back) rather than the turn aborting.
|
|
18
|
+
//
|
|
19
|
+
// The pure helpers (htmlToText, wmoCodeToText, parseDuckDuckGoHtml, the
|
|
20
|
+
// formatters) are exported and unit-tested without touching the network.
|
|
21
|
+
|
|
22
|
+
import { defineTool, type ToolDefinition } from "@earendil-works/pi-coding-agent";
|
|
23
|
+
import { Type } from "typebox";
|
|
24
|
+
|
|
25
|
+
/** Tool names, exported so pi-web can add them to its tool allowlist. */
|
|
26
|
+
export const WEB_TOOL_NAMES = ["web_search", "web_fetch", "weather"] as const;
|
|
27
|
+
|
|
28
|
+
/** Browser-ish UA: some endpoints (DuckDuckGo) serve a challenge to blanks. */
|
|
29
|
+
const USER_AGENT =
|
|
30
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
|
|
31
|
+
"(KHTML, like Gecko) Chrome/124.0 Safari/537.36";
|
|
32
|
+
|
|
33
|
+
const FETCH_TIMEOUT_MS = 20_000;
|
|
34
|
+
/** Cap on text returned to the model from a fetched page. */
|
|
35
|
+
const MAX_FETCH_CHARS = 12_000;
|
|
36
|
+
const DEFAULT_SEARCH_RESULTS = 5;
|
|
37
|
+
const MAX_SEARCH_RESULTS = 10;
|
|
38
|
+
|
|
39
|
+
// ---- low-level fetch helper ------------------------------------------
|
|
40
|
+
|
|
41
|
+
/** Combine the tool's abort signal with a wall-clock timeout. */
|
|
42
|
+
function withTimeout(signal: AbortSignal | undefined, ms: number): AbortSignal {
|
|
43
|
+
const timeout = AbortSignal.timeout(ms);
|
|
44
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface FetchResult {
|
|
48
|
+
ok: boolean;
|
|
49
|
+
status: number;
|
|
50
|
+
finalUrl: string;
|
|
51
|
+
contentType: string;
|
|
52
|
+
text: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** GET/POST a URL and read the body as text, with timeout + abort wiring. */
|
|
56
|
+
async function httpText(
|
|
57
|
+
url: string,
|
|
58
|
+
opts: { method?: string; headers?: Record<string, string>; body?: string; signal?: AbortSignal; timeoutMs?: number },
|
|
59
|
+
): Promise<FetchResult> {
|
|
60
|
+
const res = await fetch(url, {
|
|
61
|
+
method: opts.method ?? "GET",
|
|
62
|
+
headers: { "user-agent": USER_AGENT, ...opts.headers },
|
|
63
|
+
body: opts.body,
|
|
64
|
+
redirect: "follow",
|
|
65
|
+
signal: withTimeout(opts.signal, opts.timeoutMs ?? FETCH_TIMEOUT_MS),
|
|
66
|
+
});
|
|
67
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
68
|
+
const text = await res.text();
|
|
69
|
+
return { ok: res.ok, status: res.status, finalUrl: res.url || url, contentType, text };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Normalize anything thrown by fetch (timeout, DNS, abort) into a message. */
|
|
73
|
+
function describeFetchError(err: unknown): string {
|
|
74
|
+
if (err instanceof Error) {
|
|
75
|
+
if (err.name === "TimeoutError") return "request timed out";
|
|
76
|
+
if (err.name === "AbortError") return "request was cancelled";
|
|
77
|
+
return err.message;
|
|
78
|
+
}
|
|
79
|
+
return String(err);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// ---- pure helpers (unit-tested) --------------------------------------
|
|
83
|
+
|
|
84
|
+
const NAMED_ENTITIES: Record<string, string> = {
|
|
85
|
+
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", "#39": "'", "#34": '"',
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/** Decode the handful of HTML entities that show up in scraped text. */
|
|
89
|
+
export function decodeEntities(s: string): string {
|
|
90
|
+
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (whole, body: string) => {
|
|
91
|
+
if (body[0] === "#") {
|
|
92
|
+
const code = body[1] === "x" || body[1] === "X" ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
93
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : whole;
|
|
94
|
+
}
|
|
95
|
+
return NAMED_ENTITIES[body] ?? whole;
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Strip all HTML tags from a fragment and decode entities (single-line). */
|
|
100
|
+
export function stripTags(html: string): string {
|
|
101
|
+
return decodeEntities(html.replace(/<[^>]*>/g, "")).replace(/\s+/g, " ").trim();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Extract the <title> of an HTML document, if any. */
|
|
105
|
+
export function extractTitle(html: string): string | undefined {
|
|
106
|
+
const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
107
|
+
return m ? stripTags(m[1] ?? "") : undefined;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Reduce an HTML document to readable plain text: drop script/style/head,
|
|
112
|
+
* turn block-level boundaries into newlines, strip remaining tags, decode
|
|
113
|
+
* entities, and collapse runaway whitespace.
|
|
114
|
+
*/
|
|
115
|
+
export function htmlToText(html: string): string {
|
|
116
|
+
let s = html
|
|
117
|
+
.replace(/<!--[\s\S]*?-->/g, "")
|
|
118
|
+
.replace(/<(script|style|noscript|template|svg)[^>]*>[\s\S]*?<\/\1>/gi, "")
|
|
119
|
+
.replace(/<head[^>]*>[\s\S]*?<\/head>/gi, "");
|
|
120
|
+
// Block-level close/break tags become newlines so structure survives.
|
|
121
|
+
s = s.replace(/<\/(p|div|section|article|li|ul|ol|tr|h[1-6]|header|footer|nav|blockquote)>/gi, "\n");
|
|
122
|
+
s = s.replace(/<br\s*\/?>/gi, "\n");
|
|
123
|
+
s = s.replace(/<[^>]+>/g, "");
|
|
124
|
+
s = decodeEntities(s);
|
|
125
|
+
// Collapse spaces/tabs, trim each line, squeeze blank-line runs.
|
|
126
|
+
s = s.replace(/[ \t\f\v]+/g, " ");
|
|
127
|
+
s = s.split("\n").map((line) => line.trim()).join("\n");
|
|
128
|
+
s = s.replace(/\n{3,}/g, "\n\n");
|
|
129
|
+
return s.trim();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Clamp text to a max length, appending a clear truncation marker. */
|
|
133
|
+
export function truncate(text: string, max: number): string {
|
|
134
|
+
if (text.length <= max) return text;
|
|
135
|
+
return text.slice(0, max) + `\n\n…[truncated ${text.length - max} more characters]`;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** WMO weather interpretation codes (Open-Meteo) → human description. */
|
|
139
|
+
export function wmoCodeToText(code: number): string {
|
|
140
|
+
const map: Record<number, string> = {
|
|
141
|
+
0: "clear sky", 1: "mainly clear", 2: "partly cloudy", 3: "overcast",
|
|
142
|
+
45: "fog", 48: "depositing rime fog",
|
|
143
|
+
51: "light drizzle", 53: "moderate drizzle", 55: "dense drizzle",
|
|
144
|
+
56: "light freezing drizzle", 57: "dense freezing drizzle",
|
|
145
|
+
61: "slight rain", 63: "moderate rain", 65: "heavy rain",
|
|
146
|
+
66: "light freezing rain", 67: "heavy freezing rain",
|
|
147
|
+
71: "slight snowfall", 73: "moderate snowfall", 75: "heavy snowfall",
|
|
148
|
+
77: "snow grains",
|
|
149
|
+
80: "slight rain showers", 81: "moderate rain showers", 82: "violent rain showers",
|
|
150
|
+
85: "slight snow showers", 86: "heavy snow showers",
|
|
151
|
+
95: "thunderstorm", 96: "thunderstorm with slight hail", 99: "thunderstorm with heavy hail",
|
|
152
|
+
};
|
|
153
|
+
return map[code] ?? `unknown (code ${code})`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface SearchResult {
|
|
157
|
+
title: string;
|
|
158
|
+
url: string;
|
|
159
|
+
snippet: string;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Resolve a DuckDuckGo redirect href (//duckduckgo.com/l/?uddg=…) to the real URL. */
|
|
163
|
+
export function resolveDdgHref(href: string): string {
|
|
164
|
+
const decoded = decodeEntities(href);
|
|
165
|
+
const m = decoded.match(/[?&]uddg=([^&]+)/);
|
|
166
|
+
if (m && m[1]) {
|
|
167
|
+
try {
|
|
168
|
+
return decodeURIComponent(m[1]);
|
|
169
|
+
} catch {
|
|
170
|
+
return decoded;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return decoded.startsWith("//") ? "https:" + decoded : decoded;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Parse the result list out of html.duckduckgo.com/html/ output. Pairs each
|
|
178
|
+
* result anchor (title + href) with the following snippet anchor by order.
|
|
179
|
+
* Returns [] when the page carries no results (e.g. a challenge/empty page).
|
|
180
|
+
*/
|
|
181
|
+
export function parseDuckDuckGoHtml(html: string, limit: number): SearchResult[] {
|
|
182
|
+
const results: SearchResult[] = [];
|
|
183
|
+
const linkRe = /<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
184
|
+
const snippetRe = /<a[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
185
|
+
const snippets: string[] = [];
|
|
186
|
+
let sm: RegExpExecArray | null;
|
|
187
|
+
while ((sm = snippetRe.exec(html)) !== null) snippets.push(stripTags(sm[1] ?? ""));
|
|
188
|
+
let lm: RegExpExecArray | null;
|
|
189
|
+
let i = 0;
|
|
190
|
+
while ((lm = linkRe.exec(html)) !== null && results.length < limit) {
|
|
191
|
+
const url = resolveDdgHref(lm[1] ?? "");
|
|
192
|
+
const title = stripTags(lm[2] ?? "");
|
|
193
|
+
if (!title || !url) {
|
|
194
|
+
i++;
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
results.push({ title, url, snippet: snippets[i] ?? "" });
|
|
198
|
+
i++;
|
|
199
|
+
}
|
|
200
|
+
return results;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Render a result list as numbered Markdown for the model. */
|
|
204
|
+
export function formatSearchResults(query: string, results: SearchResult[], provider: string): string {
|
|
205
|
+
if (results.length === 0) {
|
|
206
|
+
return `No results found for "${query}" (via ${provider}).`;
|
|
207
|
+
}
|
|
208
|
+
const lines = results.map((r, idx) => {
|
|
209
|
+
const head = `${idx + 1}. ${r.title}\n ${r.url}`;
|
|
210
|
+
return r.snippet ? `${head}\n ${r.snippet}` : head;
|
|
211
|
+
});
|
|
212
|
+
return `Search results for "${query}" (via ${provider}):\n\n${lines.join("\n\n")}`;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// ---- weather: Open-Meteo geocode + forecast --------------------------
|
|
216
|
+
|
|
217
|
+
interface GeoPlace {
|
|
218
|
+
name: string;
|
|
219
|
+
latitude: number;
|
|
220
|
+
longitude: number;
|
|
221
|
+
country?: string;
|
|
222
|
+
admin1?: string;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Format an Open-Meteo forecast response into a compact human summary. */
|
|
226
|
+
export function formatWeather(place: GeoPlace, data: unknown, fahrenheit: boolean): string {
|
|
227
|
+
const d = data as {
|
|
228
|
+
current?: {
|
|
229
|
+
temperature_2m?: number;
|
|
230
|
+
apparent_temperature?: number;
|
|
231
|
+
relative_humidity_2m?: number;
|
|
232
|
+
wind_speed_10m?: number;
|
|
233
|
+
weather_code?: number;
|
|
234
|
+
};
|
|
235
|
+
daily?: { time?: string[]; weather_code?: number[]; temperature_2m_max?: number[]; temperature_2m_min?: number[]; precipitation_probability_max?: number[] };
|
|
236
|
+
};
|
|
237
|
+
const tUnit = fahrenheit ? "°F" : "°C";
|
|
238
|
+
const wUnit = fahrenheit ? "mph" : "km/h";
|
|
239
|
+
const where = [place.name, place.admin1, place.country].filter(Boolean).join(", ");
|
|
240
|
+
const lines: string[] = [`Weather for ${where}:`];
|
|
241
|
+
|
|
242
|
+
const cur = d.current;
|
|
243
|
+
if (cur?.temperature_2m != null) {
|
|
244
|
+
const desc = cur.weather_code != null ? `, ${wmoCodeToText(cur.weather_code)}` : "";
|
|
245
|
+
const parts = [`${Math.round(cur.temperature_2m)}${tUnit}${desc}`];
|
|
246
|
+
if (cur.apparent_temperature != null) parts.push(`feels like ${Math.round(cur.apparent_temperature)}${tUnit}`);
|
|
247
|
+
if (cur.relative_humidity_2m != null) parts.push(`humidity ${Math.round(cur.relative_humidity_2m)}%`);
|
|
248
|
+
if (cur.wind_speed_10m != null) parts.push(`wind ${Math.round(cur.wind_speed_10m)} ${wUnit}`);
|
|
249
|
+
lines.push(`Now: ${parts.join(", ")}.`);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const daily = d.daily;
|
|
253
|
+
if (daily?.time?.length) {
|
|
254
|
+
lines.push("");
|
|
255
|
+
lines.push("Forecast:");
|
|
256
|
+
for (let i = 0; i < daily.time.length; i++) {
|
|
257
|
+
const code = daily.weather_code?.[i];
|
|
258
|
+
const hi = daily.temperature_2m_max?.[i];
|
|
259
|
+
const lo = daily.temperature_2m_min?.[i];
|
|
260
|
+
const pop = daily.precipitation_probability_max?.[i];
|
|
261
|
+
const desc = code != null ? wmoCodeToText(code) : "";
|
|
262
|
+
const range = hi != null && lo != null ? `${Math.round(lo)}–${Math.round(hi)}${tUnit}` : "";
|
|
263
|
+
const rain = pop != null ? `, precip ${pop}%` : "";
|
|
264
|
+
lines.push(`- ${daily.time[i]}: ${[range, desc].filter(Boolean).join(", ")}${rain}`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return lines.join("\n");
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async function geocode(name: string, signal: AbortSignal | undefined): Promise<GeoPlace | undefined> {
|
|
271
|
+
const url = `https://geocoding-api.open-meteo.com/v1/search?name=${encodeURIComponent(name)}&count=1&language=en&format=json`;
|
|
272
|
+
const res = await httpText(url, { signal });
|
|
273
|
+
if (!res.ok) return undefined;
|
|
274
|
+
const json = JSON.parse(res.text) as { results?: GeoPlace[] };
|
|
275
|
+
return json.results?.[0];
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// ---- web_search providers --------------------------------------------
|
|
279
|
+
|
|
280
|
+
async function searchTavily(query: string, count: number, signal: AbortSignal | undefined, apiKey: string): Promise<SearchResult[]> {
|
|
281
|
+
const res = await httpText("https://api.tavily.com/search", {
|
|
282
|
+
method: "POST",
|
|
283
|
+
headers: { "content-type": "application/json" },
|
|
284
|
+
body: JSON.stringify({ api_key: apiKey, query, max_results: count, search_depth: "basic" }),
|
|
285
|
+
signal,
|
|
286
|
+
});
|
|
287
|
+
if (!res.ok) throw new Error(`Tavily returned HTTP ${res.status}`);
|
|
288
|
+
const json = JSON.parse(res.text) as { results?: Array<{ title?: string; url?: string; content?: string }> };
|
|
289
|
+
return (json.results ?? []).slice(0, count).map((r) => ({ title: r.title ?? r.url ?? "", url: r.url ?? "", snippet: r.content ?? "" }));
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
async function searchBrave(query: string, count: number, signal: AbortSignal | undefined, apiKey: string): Promise<SearchResult[]> {
|
|
293
|
+
const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=${count}`;
|
|
294
|
+
const res = await httpText(url, { headers: { "x-subscription-token": apiKey, accept: "application/json" }, signal });
|
|
295
|
+
if (!res.ok) throw new Error(`Brave returned HTTP ${res.status}`);
|
|
296
|
+
const json = JSON.parse(res.text) as { web?: { results?: Array<{ title?: string; url?: string; description?: string }> } };
|
|
297
|
+
return (json.web?.results ?? []).slice(0, count).map((r) => ({ title: r.title ?? r.url ?? "", url: r.url ?? "", snippet: stripTags(r.description ?? "") }));
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function searchDuckDuckGo(query: string, count: number, signal: AbortSignal | undefined): Promise<SearchResult[]> {
|
|
301
|
+
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
302
|
+
const res = await httpText(url, { headers: { accept: "text/html" }, signal });
|
|
303
|
+
if (!res.ok) throw new Error(`DuckDuckGo returned HTTP ${res.status}`);
|
|
304
|
+
return parseDuckDuckGoHtml(res.text, count);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** Pick the search backend from env; returns the provider label too. */
|
|
308
|
+
function resolveSearchProvider(): { name: string; run: (q: string, n: number, s: AbortSignal | undefined) => Promise<SearchResult[]> } {
|
|
309
|
+
const tavily = process.env.TAVILY_API_KEY;
|
|
310
|
+
if (tavily) return { name: "Tavily", run: (q, n, s) => searchTavily(q, n, s, tavily) };
|
|
311
|
+
const brave = process.env.BRAVE_API_KEY;
|
|
312
|
+
if (brave) return { name: "Brave", run: (q, n, s) => searchBrave(q, n, s, brave) };
|
|
313
|
+
return { name: "DuckDuckGo", run: searchDuckDuckGo };
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// ---- tool definitions ------------------------------------------------
|
|
317
|
+
|
|
318
|
+
function textResult(text: string): { content: [{ type: "text"; text: string }]; details: Record<string, never> } {
|
|
319
|
+
return { content: [{ type: "text", text }], details: {} };
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const webSearchTool = defineTool({
|
|
323
|
+
name: "web_search",
|
|
324
|
+
label: "Web Search",
|
|
325
|
+
description:
|
|
326
|
+
"Search the web for current, real-world information (news, facts, docs, anything that changes over time or that you don't already know). " +
|
|
327
|
+
"Returns a ranked list of result titles, URLs, and snippets. Follow up with web_fetch to read a result in full.",
|
|
328
|
+
parameters: Type.Object({
|
|
329
|
+
query: Type.String({ description: "The search query." }),
|
|
330
|
+
count: Type.Optional(Type.Number({ description: `Number of results to return (default ${DEFAULT_SEARCH_RESULTS}, max ${MAX_SEARCH_RESULTS}).` })),
|
|
331
|
+
}),
|
|
332
|
+
execute: async (_id, params, signal) => {
|
|
333
|
+
const query = params.query.trim();
|
|
334
|
+
if (!query) return textResult("web_search needs a non-empty query.");
|
|
335
|
+
const count = Math.max(1, Math.min(MAX_SEARCH_RESULTS, Math.round(params.count ?? DEFAULT_SEARCH_RESULTS)));
|
|
336
|
+
const provider = resolveSearchProvider();
|
|
337
|
+
try {
|
|
338
|
+
const results = await provider.run(query, count, signal);
|
|
339
|
+
if (results.length === 0 && provider.name === "DuckDuckGo") {
|
|
340
|
+
return textResult(
|
|
341
|
+
`No results for "${query}". DuckDuckGo (the keyless default) may be rate-limiting; ` +
|
|
342
|
+
`set TAVILY_API_KEY or BRAVE_API_KEY for a reliable search backend.`,
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
return textResult(formatSearchResults(query, results, provider.name));
|
|
346
|
+
} catch (err) {
|
|
347
|
+
return textResult(`Web search failed (${provider.name}): ${describeFetchError(err)}`);
|
|
348
|
+
}
|
|
349
|
+
},
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
const webFetchTool = defineTool({
|
|
353
|
+
name: "web_fetch",
|
|
354
|
+
label: "Fetch URL",
|
|
355
|
+
description:
|
|
356
|
+
"Fetch a URL and return its contents as readable text (HTML pages are stripped to plain text). " +
|
|
357
|
+
"Use this to read a web page, article, or API/JSON endpoint — e.g. after web_search, or when the user gives you a link.",
|
|
358
|
+
parameters: Type.Object({
|
|
359
|
+
url: Type.String({ description: "The absolute http(s) URL to fetch." }),
|
|
360
|
+
}),
|
|
361
|
+
execute: async (_id, params, signal) => {
|
|
362
|
+
let url = params.url.trim();
|
|
363
|
+
if (!/^https?:\/\//i.test(url)) url = "https://" + url;
|
|
364
|
+
try {
|
|
365
|
+
const res = await httpText(url, { headers: { accept: "text/html,application/json,text/plain,*/*" }, signal });
|
|
366
|
+
const header = `Fetched ${res.finalUrl} (HTTP ${res.status})`;
|
|
367
|
+
if (!res.ok) return textResult(`${header}\n\n${truncate(htmlToText(res.text), 2_000)}`);
|
|
368
|
+
const isHtml = /text\/html|application\/xhtml/i.test(res.contentType);
|
|
369
|
+
if (isHtml) {
|
|
370
|
+
const title = extractTitle(res.text);
|
|
371
|
+
const body = truncate(htmlToText(res.text), MAX_FETCH_CHARS);
|
|
372
|
+
return textResult(`${header}${title ? `\nTitle: ${title}` : ""}\n\n${body}`);
|
|
373
|
+
}
|
|
374
|
+
const isTextual = /text\/|application\/(json|xml|javascript|x-ndjson)|\+json|\+xml/i.test(res.contentType) || res.contentType === "";
|
|
375
|
+
if (!isTextual) {
|
|
376
|
+
return textResult(`${header}\n\nContent-Type ${res.contentType || "unknown"} is not text; not displaying ${res.text.length} bytes of binary data.`);
|
|
377
|
+
}
|
|
378
|
+
return textResult(`${header}\n\n${truncate(res.text, MAX_FETCH_CHARS)}`);
|
|
379
|
+
} catch (err) {
|
|
380
|
+
return textResult(`Failed to fetch ${url}: ${describeFetchError(err)}`);
|
|
381
|
+
}
|
|
382
|
+
},
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
const weatherTool = defineTool({
|
|
386
|
+
name: "weather",
|
|
387
|
+
label: "Weather",
|
|
388
|
+
description:
|
|
389
|
+
"Get current weather conditions and a short forecast for a place. Accepts a city/place name (e.g. 'Austin', 'Paris, France', 'Tokyo'). " +
|
|
390
|
+
"No API key required.",
|
|
391
|
+
parameters: Type.Object({
|
|
392
|
+
location: Type.String({ description: "City or place name to look up." }),
|
|
393
|
+
unit: Type.Optional(Type.String({ description: "Temperature unit: 'celsius' or 'fahrenheit' (default celsius)." })),
|
|
394
|
+
}),
|
|
395
|
+
execute: async (_id, params, signal) => {
|
|
396
|
+
const location = params.location.trim();
|
|
397
|
+
if (!location) return textResult("weather needs a location.");
|
|
398
|
+
const fahrenheit = params.unit === "fahrenheit";
|
|
399
|
+
try {
|
|
400
|
+
const place = await geocode(location, signal);
|
|
401
|
+
if (!place) return textResult(`Couldn't find a place named "${location}".`);
|
|
402
|
+
const params2 = new URLSearchParams({
|
|
403
|
+
latitude: String(place.latitude),
|
|
404
|
+
longitude: String(place.longitude),
|
|
405
|
+
current: "temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,weather_code,wind_speed_10m",
|
|
406
|
+
daily: "weather_code,temperature_2m_max,temperature_2m_min,precipitation_probability_max",
|
|
407
|
+
timezone: "auto",
|
|
408
|
+
forecast_days: "3",
|
|
409
|
+
});
|
|
410
|
+
if (fahrenheit) {
|
|
411
|
+
params2.set("temperature_unit", "fahrenheit");
|
|
412
|
+
params2.set("wind_speed_unit", "mph");
|
|
413
|
+
}
|
|
414
|
+
const res = await httpText(`https://api.open-meteo.com/v1/forecast?${params2.toString()}`, { signal });
|
|
415
|
+
if (!res.ok) return textResult(`Weather lookup failed: Open-Meteo returned HTTP ${res.status}.`);
|
|
416
|
+
return textResult(formatWeather(place, JSON.parse(res.text), fahrenheit));
|
|
417
|
+
} catch (err) {
|
|
418
|
+
return textResult(`Weather lookup failed: ${describeFetchError(err)}`);
|
|
419
|
+
}
|
|
420
|
+
},
|
|
421
|
+
});
|
|
422
|
+
|
|
423
|
+
/** All web tools, ready to pass to createAgentSession({ customTools }). */
|
|
424
|
+
export function createWebTools(): ToolDefinition[] {
|
|
425
|
+
return [webSearchTool, webFetchTool, weatherTool];
|
|
426
|
+
}
|
package/src/weights.ts
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// Weights: model directory → lazy mlx arrays.
|
|
2
|
+
//
|
|
3
|
+
// Tensor data goes through mlx's native safetensors loader
|
|
4
|
+
// (mlx_load_safetensors): opening is milliseconds, and each tensor's bytes
|
|
5
|
+
// are read into an mlx-owned (page-aligned, Metal-visible) buffer only
|
|
6
|
+
// when first evaluated — the same lazy semantics as Python's mx.load.
|
|
7
|
+
//
|
|
8
|
+
// Why not wrap our own mmap pointers? Verified in Phase 1: GPU ops on
|
|
9
|
+
// externally-wrapped buffers read garbage unless the pointer is
|
|
10
|
+
// page-aligned, and safetensors tensor offsets are arbitrary (not even
|
|
11
|
+
// element-aligned). CPU-stream ops on wrapped pointers are correct, but
|
|
12
|
+
// the weights' consumers are GPU kernels. See PLAN.md Phase 1 findings.
|
|
13
|
+
//
|
|
14
|
+
// The JS-side parser (ShardedSafetensors) stays as the metadata source:
|
|
15
|
+
// names, shapes, dtypes, byte sizes — for the registry, fit reports, and
|
|
16
|
+
// validation — without touching tensor bytes.
|
|
17
|
+
|
|
18
|
+
import { ptr, read } from "bun:ffi";
|
|
19
|
+
import { ShardedSafetensors, type TensorInfo } from "./safetensors";
|
|
20
|
+
import { MlxArray, cpuStream } from "./mlx/array";
|
|
21
|
+
import { C } from "./mlx/ffi";
|
|
22
|
+
|
|
23
|
+
const cstr = (s: string) => Buffer.from(s + "\0", "utf8");
|
|
24
|
+
|
|
25
|
+
export class Weights {
|
|
26
|
+
readonly shards: ShardedSafetensors;
|
|
27
|
+
/** shard filename → native mlx map handle (string → lazy array). */
|
|
28
|
+
readonly #maps = new Map<string, bigint>();
|
|
29
|
+
readonly #arrays = new Map<string, MlxArray>();
|
|
30
|
+
|
|
31
|
+
private constructor(shards: ShardedSafetensors) {
|
|
32
|
+
this.shards = shards;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
static async open(modelDir: string): Promise<Weights> {
|
|
36
|
+
const self = new Weights(await ShardedSafetensors.open(modelDir));
|
|
37
|
+
for (const [file, sf] of self.shards.files) {
|
|
38
|
+
// out-param slots read back via read.u64, not [0] (DFG stale-read
|
|
39
|
+
// bug — see outArray in mlx/ffi.ts). Cold path, but the rule is
|
|
40
|
+
// uniform: native wrote it, read.* reads it.
|
|
41
|
+
const arrMap = new BigUint64Array([C.mlx_map_string_to_array_new()]);
|
|
42
|
+
const metaMap = new BigUint64Array([C.mlx_map_string_to_string_new()]);
|
|
43
|
+
const arrMapPtr = ptr(arrMap);
|
|
44
|
+
const metaMapPtr = ptr(metaMap);
|
|
45
|
+
const status = C.mlx_load_safetensors(
|
|
46
|
+
arrMapPtr, metaMapPtr, ptr(cstr(sf.path)), cpuStream,
|
|
47
|
+
);
|
|
48
|
+
C.mlx_map_string_to_string_free(read.u64(metaMapPtr, 0));
|
|
49
|
+
if (status !== 0) throw new Error(`mlx_load_safetensors(${sf.path}) failed`);
|
|
50
|
+
self.#maps.set(file, read.u64(arrMapPtr, 0));
|
|
51
|
+
}
|
|
52
|
+
return self;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
get tensorNames(): string[] {
|
|
56
|
+
return this.shards.tensorNames;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
info(name: string): TensorInfo {
|
|
60
|
+
return this.shards.info(name);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
has(name: string): boolean {
|
|
64
|
+
return this.shards.tensorToFile.has(name);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Lazy mlx array for a tensor; cached per name. */
|
|
68
|
+
tensor(name: string): MlxArray {
|
|
69
|
+
let arr = this.#arrays.get(name);
|
|
70
|
+
if (!arr) {
|
|
71
|
+
const sf = this.shards.tensorToFile.get(name);
|
|
72
|
+
if (!sf) throw new Error(`no tensor named ${name}`);
|
|
73
|
+
const mapHandle = [...this.#maps.entries()]
|
|
74
|
+
.find(([file]) => this.shards.files.get(file) === sf)![1];
|
|
75
|
+
const slot = new BigUint64Array([C.mlx_array_new()]);
|
|
76
|
+
const slotPtr = ptr(slot);
|
|
77
|
+
if (C.mlx_map_string_to_array_get(slotPtr, mapHandle, ptr(cstr(name))) !== 0)
|
|
78
|
+
throw new Error(`tensor ${name} missing from native map`);
|
|
79
|
+
arr = new MlxArray(read.u64(slotPtr, 0));
|
|
80
|
+
this.#arrays.set(name, arr);
|
|
81
|
+
}
|
|
82
|
+
return arr;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
dispose(): void {
|
|
86
|
+
for (const arr of this.#arrays.values()) arr.dispose();
|
|
87
|
+
this.#arrays.clear();
|
|
88
|
+
for (const map of this.#maps.values()) C.mlx_map_string_to_array_free(map);
|
|
89
|
+
this.#maps.clear();
|
|
90
|
+
}
|
|
91
|
+
}
|