mlx-bun 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +11 -2
  2. package/docs/dag/training-inference-map.html +575 -0
  3. package/docs/design/memory-system.md +193 -295
  4. package/docs/design/orpo-dynamic-lambda.md +179 -0
  5. package/docs/design/orpo-future-enhancements.md +42 -0
  6. package/docs/design/orpo-training.md +13 -0
  7. package/docs/design/parity-tier-dag.md +143 -0
  8. package/docs/design/w4a16-compute-precision-spike.md +232 -0
  9. package/docs/investigations/orpo-base-uf-experiment-and-directions.md +120 -0
  10. package/docs/investigations/orpo-flash-cce-pin-leak.md +167 -0
  11. package/docs/investigations/orpo-uf-testing-handoff.md +126 -0
  12. package/docs/investigations/steel-flash-cce-handoff.md +58 -0
  13. package/docs/investigations/trainer-validation-experiment.md +173 -0
  14. package/docs/reference/memory.md +77 -0
  15. package/docs/reference/orpo-quickstart.md +44 -5
  16. package/docs/reference/training.md +9 -5
  17. package/package.json +1 -1
  18. package/src/cli.ts +651 -18
  19. package/src/eval/runner.ts +76 -6
  20. package/src/eval/tasks/bfcl.ts +49 -15
  21. package/src/eval/tasks/gsm8k.ts +21 -9
  22. package/src/eval/tasks/hashhop.ts +33 -4
  23. package/src/eval/tasks/humaneval.ts +30 -8
  24. package/src/eval/tasks/ifeval.ts +52 -18
  25. package/src/eval/tasks/mmlu.ts +85 -74
  26. package/src/jobs/runner.ts +6 -0
  27. package/src/memory/chunk.ts +25 -0
  28. package/src/memory/cluster.ts +24 -0
  29. package/src/memory/db.ts +114 -0
  30. package/src/memory/ingest.ts +22 -0
  31. package/src/memory/pipeline.ts +63 -0
  32. package/src/memory/prompts.ts +23 -0
  33. package/src/memory/schedule.ts +163 -0
  34. package/src/memory/synthesize.ts +29 -0
  35. package/src/memory/tools.ts +399 -0
  36. package/src/memory/vault.ts +694 -0
  37. package/src/memory/wikify.ts +23 -0
  38. package/src/mlx/metal-kernel.ts +13 -0
  39. package/src/model/flash-attention.ts +5 -3
  40. package/src/pi-provider.ts +29 -13
  41. package/src/pi-session.ts +81 -0
  42. package/src/pi-terminal.ts +13 -24
  43. package/src/pi-web.ts +246 -166
  44. package/src/quantize/job.ts +17 -2
  45. package/src/quantize/quantizer.ts +26 -2
  46. package/src/server.ts +213 -44
  47. package/src/tokenizer.ts +6 -2
  48. package/src/train/dataset.ts +11 -3
  49. package/src/train/flash-cce.ts +18 -5
  50. package/src/train/job.ts +41 -4
  51. package/src/train/loss.ts +10 -4
  52. package/src/train/trainer.ts +62 -3
  53. package/src/train/watch.ts +542 -0
  54. package/src/web/app.html +264 -14
  55. package/src/web/skills/memory/SKILL.md +97 -0
  56. package/src/web/skills/web-research/SKILL.md +3 -1
  57. package/src/web/skills.ts +46 -8
@@ -0,0 +1,163 @@
1
+ // mlx-bun memory — nightly scheduling via launchd. ✅ REAL (the pipeline it
2
+ // runs is the M1 stub, but install/remove/status here are fully functional).
3
+ //
4
+ // mlx-bun is macOS/Apple-silicon-only by identity, so launchd is the native
5
+ // choice over crontab: it survives reboot, runs jobs missed while asleep on
6
+ // next wake, and gives clean install/uninstall. `memory schedule` writes
7
+ // ~/Library/LaunchAgents/com.mlx-bun.memory.plist (a StartCalendarInterval that
8
+ // runs `mlx-bun memory synthesize`) and loads it; `unschedule` unloads + removes
9
+ // it. Today that job is a no-op (synthesis is stubbed) — harmless; when M1 lands
10
+ // the same schedule starts producing articles with no reconfiguration.
11
+ //
12
+ // See docs/design/memory-system.md → "Scheduling (launchd)".
13
+
14
+ import { spawn } from "node:child_process";
15
+ import { mkdir, rm, writeFile } from "node:fs/promises";
16
+ import { existsSync } from "node:fs";
17
+ import { homedir } from "node:os";
18
+ import { join } from "node:path";
19
+
20
+ export const LAUNCHD_LABEL = "com.mlx-bun.memory";
21
+
22
+ export function plistPath(): string {
23
+ return join(homedir(), "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
24
+ }
25
+
26
+ function logsDir(): string {
27
+ return join(homedir(), ".mlx-bun", "logs");
28
+ }
29
+
30
+ /**
31
+ * Resolve how to invoke mlx-bun from launchd's minimal environment. Prefer the
32
+ * installed binary on PATH; fall back to the current dev invocation
33
+ * (`<bun> <cli.ts>`). Wrapped in a login shell at call time so PATH/bun resolve.
34
+ */
35
+ function mlxBunCommand(): string {
36
+ const onPath = Bun.which("mlx-bun");
37
+ if (onPath) return shQuote(onPath);
38
+ // Dev: process.execPath is bun, argv[1] is .../src/cli.ts.
39
+ return `${shQuote(process.execPath)} ${shQuote(process.argv[1] ?? "")}`.trim();
40
+ }
41
+
42
+ function shQuote(s: string): string {
43
+ return `'${s.replace(/'/g, "'\\''")}'`;
44
+ }
45
+
46
+ function xmlEscape(s: string): string {
47
+ return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
48
+ }
49
+
50
+ export interface ScheduleOptions {
51
+ /** Local wall-clock time, "HH:MM" 24h. Default 03:00. */
52
+ at?: string;
53
+ }
54
+
55
+ /** Parse "HH:MM" → {hour, minute}; defaults to 03:00 on bad/empty input. */
56
+ export function parseAt(at?: string): { hour: number; minute: number } {
57
+ const m = /^(\d{1,2}):(\d{2})$/.exec((at ?? "").trim());
58
+ if (!m) return { hour: 3, minute: 0 };
59
+ const hour = Math.min(23, Math.max(0, Number(m[1])));
60
+ const minute = Math.min(59, Math.max(0, Number(m[2])));
61
+ return { hour, minute };
62
+ }
63
+
64
+ /** Pure: build the launchd plist XML for the nightly synthesis job. */
65
+ export function buildPlist(opts: ScheduleOptions = {}): string {
66
+ const { hour, minute } = parseAt(opts.at);
67
+ // Login shell so the user's PATH (bun, mlx-bun) resolves under launchd.
68
+ const cmd = `exec ${mlxBunCommand()} memory synthesize`;
69
+ const out = join(logsDir(), "memory-synthesis.out.log");
70
+ const err = join(logsDir(), "memory-synthesis.err.log");
71
+ return `<?xml version="1.0" encoding="UTF-8"?>
72
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
73
+ <plist version="1.0">
74
+ <dict>
75
+ <key>Label</key>
76
+ <string>${LAUNCHD_LABEL}</string>
77
+
78
+ <key>ProgramArguments</key>
79
+ <array>
80
+ <string>/bin/zsh</string>
81
+ <string>-lc</string>
82
+ <string>${xmlEscape(cmd)}</string>
83
+ </array>
84
+
85
+ <!-- Local wall-clock; if asleep at the time, launchd runs on next wake. -->
86
+ <key>StartCalendarInterval</key>
87
+ <dict>
88
+ <key>Hour</key>
89
+ <integer>${hour}</integer>
90
+ <key>Minute</key>
91
+ <integer>${minute}</integer>
92
+ </dict>
93
+
94
+ <key>RunAtLoad</key>
95
+ <false/>
96
+
97
+ <key>StandardOutPath</key>
98
+ <string>${xmlEscape(out)}</string>
99
+ <key>StandardErrorPath</key>
100
+ <string>${xmlEscape(err)}</string>
101
+
102
+ <key>ProcessType</key>
103
+ <string>Background</string>
104
+ </dict>
105
+ </plist>
106
+ `;
107
+ }
108
+
109
+ function launchctl(args: string[]): Promise<boolean> {
110
+ return new Promise((resolve) => {
111
+ const proc = spawn("launchctl", args, { stdio: "ignore" });
112
+ proc.on("exit", (code) => resolve(code === 0));
113
+ proc.on("error", () => resolve(false));
114
+ });
115
+ }
116
+
117
+ export interface InstallResult {
118
+ plistPath: string;
119
+ at: { hour: number; minute: number };
120
+ loaded: boolean;
121
+ }
122
+
123
+ /** Write the plist and (re)load it into launchd. */
124
+ export async function installSchedule(opts: ScheduleOptions = {}): Promise<InstallResult> {
125
+ const path = plistPath();
126
+ await mkdir(join(homedir(), "Library", "LaunchAgents"), { recursive: true });
127
+ await mkdir(logsDir(), { recursive: true });
128
+ await writeFile(path, buildPlist(opts));
129
+ // Reload: unload first (ignore failure if not loaded), then load -w.
130
+ await launchctl(["unload", path]);
131
+ const loaded = await launchctl(["load", "-w", path]);
132
+ return { plistPath: path, at: parseAt(opts.at), loaded };
133
+ }
134
+
135
+ /** Unload and remove the launchd agent. Returns true if a plist was removed. */
136
+ export async function removeSchedule(): Promise<boolean> {
137
+ const path = plistPath();
138
+ if (!existsSync(path)) return false;
139
+ await launchctl(["unload", "-w", path]);
140
+ await rm(path, { force: true });
141
+ return true;
142
+ }
143
+
144
+ export interface ScheduleStatus {
145
+ installed: boolean;
146
+ plistPath: string;
147
+ /** Whether launchctl currently lists the agent (best-effort). */
148
+ loaded: boolean;
149
+ }
150
+
151
+ export async function scheduleStatus(): Promise<ScheduleStatus> {
152
+ const path = plistPath();
153
+ const installed = existsSync(path);
154
+ let loaded = false;
155
+ if (installed) {
156
+ loaded = await new Promise<boolean>((resolve) => {
157
+ const proc = spawn("launchctl", ["list", LAUNCHD_LABEL], { stdio: "ignore" });
158
+ proc.on("exit", (code) => resolve(code === 0));
159
+ proc.on("error", () => resolve(false));
160
+ });
161
+ }
162
+ return { installed, plistPath: path, loaded };
163
+ }
@@ -0,0 +1,29 @@
1
+ // mlx-bun memory — synthesis stage 4: SYNTHESIZE. ⚠️ M1 STUB.
2
+ //
3
+ // For each bucket touched this run, integrate its new chunks into the bucket's
4
+ // article (local-model call) — create-or-update, conservatively. The hard,
5
+ // proven gate is inherited from lucien: preserve prose, ≥70% word-count floor,
6
+ // every original conv: citation survives, integrate rather than overwrite. A
7
+ // weak local pass is gated to NO-OP rather than allowed to corrupt the vault.
8
+ // Mirrors lucien's synthesize-dispatch.ts / synthesize-update.ts.
9
+ //
10
+ // See docs/design/memory-system.md → "Synthesis runs on the local model".
11
+
12
+ import type { SynthesisEvent } from "./pipeline";
13
+
14
+ export interface SynthesizeResult {
15
+ articlesCreated: number;
16
+ articlesUpdated: number;
17
+ skippedByGate: number;
18
+ }
19
+
20
+ /** Bucket → article create/update + safety gate (local model). STUB. */
21
+ export async function synthesizeBuckets(
22
+ _model: unknown,
23
+ _onEvent?: (e: SynthesisEvent) => void,
24
+ ): Promise<SynthesizeResult> {
25
+ // TODO(M1): per touched bucket, feed unsynthesized chunks + current article
26
+ // through the synthesize prompt; apply the conservative gate; write the
27
+ // article; record (bucket, chunk_id) in synthesized_bucket_chunks.
28
+ throw new Error("memory.synthesize is not implemented yet (M1)");
29
+ }
@@ -0,0 +1,399 @@
1
+ // mlx-bun memory — the read tools the local assistant uses to consult your wiki.
2
+ //
3
+ // These are pi custom tools (defineTool), registered alongside the coding +
4
+ // web tools in both front doors (pi-terminal.ts, pi-web.ts). They're read-only,
5
+ // so they auto-allow (not in GATED_TOOLS) — consulting your own memory never
6
+ // needs an approval gate. They read ~/.mlx-bun/wiki via src/memory/vault.ts.
7
+ //
8
+ // Mirrors lucien's read surface: search (discovery) → focused read (toc /
9
+ // section / links) → full read / list / status. Knowing *when* to consult is
10
+ // driven by these tool descriptions and the bundled memory skill (pi surfaces
11
+ // its description and loads the body on demand); the system prompt carries only
12
+ // a one-line presence hint (memoryIndexHint), not a title dump.
13
+
14
+ import { defineTool, type ToolDefinition } from "@earendil-works/pi-coding-agent";
15
+ import { Type } from "typebox";
16
+ import { scheduleStatus } from "./schedule";
17
+ import {
18
+ extractSection,
19
+ getArticleLinks,
20
+ listArticles,
21
+ listReferenceDocs,
22
+ parseToc,
23
+ readArticle,
24
+ searchArticles,
25
+ vaultRoot,
26
+ vaultStatus,
27
+ type SearchHit,
28
+ } from "./vault";
29
+
30
+ /** Tool names, exported so the front doors can add them to the allowlist. */
31
+ export const MEMORY_TOOL_NAMES = [
32
+ "memory_search",
33
+ "memory_read",
34
+ "memory_toc",
35
+ "memory_section",
36
+ "memory_links",
37
+ "memory_list",
38
+ "memory_status",
39
+ ] as const;
40
+
41
+ export const REFERENCE_TOOL_NAMES = [
42
+ "reference_search",
43
+ "reference_read",
44
+ "reference_list",
45
+ ] as const;
46
+
47
+ const MAX_SEARCH_HITS = 30;
48
+
49
+ function textResult(text: string): { content: [{ type: "text"; text: string }]; details: Record<string, never> } {
50
+ return { content: [{ type: "text", text }], details: {} };
51
+ }
52
+
53
+ export async function isMemoryEnabled(): Promise<boolean> {
54
+ try {
55
+ return (await vaultStatus()).exists;
56
+ } catch {
57
+ return false;
58
+ }
59
+ }
60
+
61
+ function isReferenceId(article: string): boolean {
62
+ return article.trim().startsWith("Reference/");
63
+ }
64
+
65
+ async function memoryUnavailableMessage(): Promise<string | null> {
66
+ const st = await vaultStatus();
67
+ if (st.exists) return null;
68
+ return `Memory is not set up yet at ${st.root}. Run \`mlx-bun memory init\` to create the local Markdown wiki, or ask the user before setting it up.`;
69
+ }
70
+
71
+ /** Compact, model-friendly rendering of search hits grouped by relevance. */
72
+ function formatSearch(
73
+ query: string,
74
+ summaries: { article: string; occurrences: number; matched_terms?: string[] }[],
75
+ hits: SearchHit[],
76
+ opts: { corpusLabel?: string; readTool?: string } = {},
77
+ ): string {
78
+ const corpus = opts.corpusLabel ?? "personal-memory articles";
79
+ const readTool = opts.readTool ?? "memory_read";
80
+ if (summaries.length === 0) return `No ${corpus} match "${query}".`;
81
+ const top = summaries.slice(0, 12);
82
+ const lines = [`${corpus[0]!.toUpperCase()}${corpus.slice(1)} matching "${query}" (most relevant first):`, ""];
83
+ for (const s of top) {
84
+ const terms = s.matched_terms && s.matched_terms.length ? ` [${s.matched_terms.join(", ")}]` : "";
85
+ lines.push(`- ${s.article} (${s.occurrences} hit${s.occurrences === 1 ? "" : "s"})${terms}`);
86
+ }
87
+ if (summaries.length > top.length) lines.push(` …and ${summaries.length - top.length} more.`);
88
+ if (hits.length) {
89
+ lines.push("", "Sample lines:");
90
+ for (const h of hits.slice(0, 8)) {
91
+ const where = h.anchor ? `${h.article}#${h.anchor}` : h.article;
92
+ lines.push(`- ${where}:${h.line} ${h.excerpt}`);
93
+ }
94
+ }
95
+ lines.push("", `Read a full result with ${readTool} (e.g. ${readTool} article="${top[0]!.article}").`);
96
+ return lines.join("\n");
97
+ }
98
+
99
+ const memorySearchTool = defineTool({
100
+ name: "memory_search",
101
+ label: "Search Memory",
102
+ description:
103
+ "Search the user's local personal-memory articles for user-specific continuity: prior conversations, named personal projects, people, preferences, decisions, or history. " +
104
+ "Use when the answer depends on missing personal context; do not use for weather, current public facts, generic web research, or ordinary coding/file tasks when current files are sufficient. " +
105
+ "Returns ranked article matches and sample lines; follow up with memory_read to read a relevant article in full.",
106
+ parameters: Type.Object({
107
+ query: Type.String({ description: "Word or phrase to search for across the user's memory articles." }),
108
+ limit: Type.Optional(Type.Number({ description: `Max sample line-hits to return (default ${MAX_SEARCH_HITS}).` })),
109
+ }),
110
+ execute: async (_id, params) => {
111
+ const query = params.query.trim();
112
+ if (!query) return textResult("memory_search needs a non-empty query.");
113
+ const limit = Math.max(1, Math.min(MAX_SEARCH_HITS, Math.round(params.limit ?? MAX_SEARCH_HITS)));
114
+ try {
115
+ const unavailable = await memoryUnavailableMessage();
116
+ if (unavailable) return textResult(unavailable);
117
+ const { summaries, hits } = await searchArticles(vaultRoot(), query, { limit, scope: "articles" });
118
+ return textResult(formatSearch(query, summaries, hits));
119
+ } catch (err) {
120
+ return textResult(`memory_search failed: ${err instanceof Error ? err.message : String(err)}`);
121
+ }
122
+ },
123
+ });
124
+
125
+ const memoryReadTool = defineTool({
126
+ name: "memory_read",
127
+ label: "Read Memory",
128
+ description:
129
+ "Read one full Markdown article from the user's personal memory. " +
130
+ "Use after memory_search when user-specific prior context is needed. " +
131
+ "Pass the article stem exactly as returned by memory_search or memory_list (underscored, e.g. \"Archie_Project\").",
132
+ parameters: Type.Object({
133
+ article: Type.String({ description: "Article stem to read (underscored filename, no .md, e.g. \"Archie_Project\")." }),
134
+ }),
135
+ execute: async (_id, params) => {
136
+ const article = params.article.trim();
137
+ if (!article) return textResult("memory_read needs an article name.");
138
+ try {
139
+ const unavailable = await memoryUnavailableMessage();
140
+ if (unavailable) return textResult(unavailable);
141
+ if (isReferenceId(article)) return textResult("memory_read is for personal-memory articles. Use reference_read for Reference/ documents.");
142
+ const { content } = await readArticle(vaultRoot(), article);
143
+ return textResult(content);
144
+ } catch (err) {
145
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
146
+ }
147
+ },
148
+ });
149
+
150
+ const memoryTocTool = defineTool({
151
+ name: "memory_toc",
152
+ label: "Memory TOC",
153
+ description:
154
+ "List the Markdown headings in one memory article, with anchors. Use before memory_section when the article is long or you need a specific part.",
155
+ parameters: Type.Object({
156
+ article: Type.String({ description: "Article stem (underscored filename, no .md, e.g. \"Archie_Project\")." }),
157
+ }),
158
+ execute: async (_id, params) => {
159
+ const article = params.article.trim();
160
+ if (!article) return textResult("memory_toc needs an article name.");
161
+ try {
162
+ const unavailable = await memoryUnavailableMessage();
163
+ if (unavailable) return textResult(unavailable);
164
+ if (isReferenceId(article)) return textResult("memory_toc is for personal-memory articles. Use reference_read for Reference/ documents.");
165
+ const { content } = await readArticle(vaultRoot(), article);
166
+ const toc = parseToc(content);
167
+ if (toc.length === 0) return textResult(`${article} has no Markdown headings.`);
168
+ return textResult(
169
+ `Headings in ${article}:\n\n` +
170
+ toc.map((h) => `${" ".repeat(Math.max(0, h.depth - 1))}- ${h.title} (#${h.anchor})`).join("\n"),
171
+ );
172
+ } catch (err) {
173
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
174
+ }
175
+ },
176
+ });
177
+
178
+ const memorySectionTool = defineTool({
179
+ name: "memory_section",
180
+ label: "Read Memory Section",
181
+ description:
182
+ "Read one section from a memory article by heading anchor. Use memory_toc or memory_search first to find anchors; this is cheaper than reading a long article in full.",
183
+ parameters: Type.Object({
184
+ article: Type.String({ description: "Article stem (underscored filename, no .md, e.g. \"Archie_Project\")." }),
185
+ anchor: Type.String({ description: "Heading anchor, e.g. \"origin\" or \"current-design\"." }),
186
+ }),
187
+ execute: async (_id, params) => {
188
+ const article = params.article.trim();
189
+ const anchor = params.anchor.trim().replace(/^#/, "");
190
+ if (!article) return textResult("memory_section needs an article name.");
191
+ if (!anchor) return textResult("memory_section needs a heading anchor.");
192
+ try {
193
+ const unavailable = await memoryUnavailableMessage();
194
+ if (unavailable) return textResult(unavailable);
195
+ if (isReferenceId(article)) return textResult("memory_section is for personal-memory articles. Use reference_read for Reference/ documents.");
196
+ const { content } = await readArticle(vaultRoot(), article);
197
+ const section = extractSection(content, anchor);
198
+ if (!section) return textResult(`No section #${anchor} found in ${article}. Try memory_toc article=\"${article}\".`);
199
+ return textResult(section);
200
+ } catch (err) {
201
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
202
+ }
203
+ },
204
+ });
205
+
206
+ const memoryLinksTool = defineTool({
207
+ name: "memory_links",
208
+ label: "Memory Links",
209
+ description:
210
+ "Show resolved outgoing and incoming wikilinks for one memory article. Use when nearby concepts/backlinks might matter to the answer.",
211
+ parameters: Type.Object({
212
+ article: Type.String({ description: "Article stem (underscored filename, no .md, e.g. \"Archie_Project\")." }),
213
+ }),
214
+ execute: async (_id, params) => {
215
+ const article = params.article.trim();
216
+ if (!article) return textResult("memory_links needs an article name.");
217
+ try {
218
+ const unavailable = await memoryUnavailableMessage();
219
+ if (unavailable) return textResult(unavailable);
220
+ if (isReferenceId(article)) return textResult("memory_links is for personal-memory articles; Reference/ documents do not participate in the personal-memory link graph.");
221
+ const { outbound, inbound } = await getArticleLinks(vaultRoot(), article);
222
+ const lines = [`Links for ${article}:`, "", "Outbound:"];
223
+ lines.push(outbound.length ? outbound.map((s) => `- ${s}`).join("\n") : "- (none)");
224
+ lines.push("", "Inbound:");
225
+ lines.push(inbound.length ? inbound.map((s) => `- ${s}`).join("\n") : "- (none)");
226
+ return textResult(lines.join("\n"));
227
+ } catch (err) {
228
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
229
+ }
230
+ },
231
+ });
232
+
233
+ const memoryListTool = defineTool({
234
+ name: "memory_list",
235
+ label: "List Memory",
236
+ description:
237
+ "List the titles of the user's personal-memory articles. Use as an overview, or as a fallback when memory_search doesn't surface what you expected.",
238
+ parameters: Type.Object({}),
239
+ execute: async () => {
240
+ try {
241
+ const unavailable = await memoryUnavailableMessage();
242
+ if (unavailable) return textResult(unavailable);
243
+ const articles = await listArticles(vaultRoot());
244
+ if (articles.length === 0) return textResult("The user's memory has no personal articles yet.");
245
+ return textResult(
246
+ `${articles.length} personal-memory article(s):\n\n` +
247
+ articles.map((s) => `- ${s}`).join("\n"),
248
+ );
249
+ } catch (err) {
250
+ return textResult(`memory_list failed: ${err instanceof Error ? err.message : String(err)}`);
251
+ }
252
+ },
253
+ });
254
+
255
+ const memoryStatusTool = defineTool({
256
+ name: "memory_status",
257
+ label: "Memory Status",
258
+ description:
259
+ "Show read-only status for the user's memory: vault path, setup state, article count, git state, and nightly schedule state. Use when the user asks whether memory is on or where it lives.",
260
+ parameters: Type.Object({}),
261
+ execute: async () => {
262
+ try {
263
+ const st = await vaultStatus();
264
+ const sched = await scheduleStatus();
265
+ const recent = st.recentArticles.length
266
+ ? st.recentArticles.slice(0, 5).map((r) => ` - ${r.article} (${new Date(r.mtimeMs).toISOString()})`).join("\n")
267
+ : " - (none)";
268
+ return textResult([
269
+ "Memory status:",
270
+ `- vault: ${st.root}`,
271
+ `- setup: ${st.exists ? "yes" : "no"}`,
272
+ `- articles: ${st.articleCount}`,
273
+ `- read-only references: ${st.referenceCount}`,
274
+ `- git: ${st.isGitRepo ? "tracked" : "not tracked"}`,
275
+ `- last synthesis: not available yet (M1 synthesis is stubbed)`,
276
+ `- synthesis: M1 stub (manual/nightly writes are not implemented yet)`,
277
+ `- nightly: ${sched.installed ? (sched.loaded ? "scheduled" : "installed but not loaded") : "not scheduled"}`,
278
+ sched.installed ? `- launchd plist: ${sched.plistPath}` : "- setup command: mlx-bun memory init",
279
+ "- recent changed articles:",
280
+ recent,
281
+ ].join("\n"));
282
+ } catch (err) {
283
+ return textResult(`memory_status failed: ${err instanceof Error ? err.message : String(err)}`);
284
+ }
285
+ },
286
+ });
287
+
288
+ // ---- reference docs (separate from personal memory) -------------------
289
+
290
+ const referenceSearchTool = defineTool({
291
+ name: "reference_search",
292
+ label: "Search mlx-bun Reference",
293
+ description:
294
+ "Search read-only mlx-bun reference documents that are symlinked into the memory vault. " +
295
+ "Use for mlx-bun documentation questions when repository files are not enough. This is project reference, not personal memory.",
296
+ parameters: Type.Object({
297
+ query: Type.String({ description: "Word or phrase to search for across mlx-bun reference documents." }),
298
+ limit: Type.Optional(Type.Number({ description: `Max sample line-hits to return (default ${MAX_SEARCH_HITS}).` })),
299
+ }),
300
+ execute: async (_id, params) => {
301
+ const query = params.query.trim();
302
+ if (!query) return textResult("reference_search needs a non-empty query.");
303
+ const limit = Math.max(1, Math.min(MAX_SEARCH_HITS, Math.round(params.limit ?? MAX_SEARCH_HITS)));
304
+ try {
305
+ const unavailable = await memoryUnavailableMessage();
306
+ if (unavailable) return textResult(unavailable);
307
+ const { summaries, hits } = await searchArticles(vaultRoot(), query, { limit, scope: "reference" });
308
+ return textResult(formatSearch(query, summaries, hits, { corpusLabel: "read-only reference documents", readTool: "reference_read" }));
309
+ } catch (err) {
310
+ return textResult(`reference_search failed: ${err instanceof Error ? err.message : String(err)}`);
311
+ }
312
+ },
313
+ });
314
+
315
+ const referenceReadTool = defineTool({
316
+ name: "reference_read",
317
+ label: "Read mlx-bun Reference",
318
+ description:
319
+ "Read one read-only mlx-bun reference document from the vault's Reference/ namespace. " +
320
+ "Pass the id exactly as returned by reference_search or reference_list, e.g. \"Reference/mlx-bun_Server_API\".",
321
+ parameters: Type.Object({
322
+ article: Type.String({ description: "Reference doc id, e.g. \"Reference/mlx-bun_Server_API\"." }),
323
+ }),
324
+ execute: async (_id, params) => {
325
+ const article = params.article.trim();
326
+ if (!article) return textResult("reference_read needs a Reference/... document id.");
327
+ try {
328
+ const unavailable = await memoryUnavailableMessage();
329
+ if (unavailable) return textResult(unavailable);
330
+ const id = article.startsWith("Reference/") ? article : `Reference/${article}`;
331
+ const { content } = await readArticle(vaultRoot(), id);
332
+ return textResult(content);
333
+ } catch (err) {
334
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try reference_search to find the right reference document.`);
335
+ }
336
+ },
337
+ });
338
+
339
+ const referenceListTool = defineTool({
340
+ name: "reference_list",
341
+ label: "List mlx-bun References",
342
+ description: "List read-only mlx-bun reference documents available under the vault's Reference/ namespace.",
343
+ parameters: Type.Object({}),
344
+ execute: async () => {
345
+ try {
346
+ const unavailable = await memoryUnavailableMessage();
347
+ if (unavailable) return textResult(unavailable);
348
+ const refs = await listReferenceDocs(vaultRoot());
349
+ if (refs.length === 0) return textResult("No read-only reference documents are linked in the memory vault.");
350
+ return textResult(`${refs.length} read-only reference document(s):\n\n` + refs.map((s) => `- ${s}`).join("\n"));
351
+ } catch (err) {
352
+ return textResult(`reference_list failed: ${err instanceof Error ? err.message : String(err)}`);
353
+ }
354
+ },
355
+ });
356
+
357
+ export function createMemoryTools(): ToolDefinition[] {
358
+ return [
359
+ memorySearchTool,
360
+ memoryReadTool,
361
+ memoryTocTool,
362
+ memorySectionTool,
363
+ memoryLinksTool,
364
+ memoryListTool,
365
+ memoryStatusTool,
366
+ ];
367
+ }
368
+
369
+ export function createReferenceTools(): ToolDefinition[] {
370
+ return [referenceSearchTool, referenceReadTool, referenceListTool];
371
+ }
372
+
373
+ /**
374
+ * One gated line for the system prompt: states that memory is ON for this user
375
+ * and points at the `memory` skill. NOT a title dump and NOT a how-to. pi
376
+ * already auto-injects every skill's name+description+location
377
+ * (formatSkillsForPrompt) and loads the SKILL.md body on demand, so the skill
378
+ * owns the detailed workflow. This hint is intentionally soft: memory is a
379
+ * scoped retrieval capability for user-specific continuity, not a default first
380
+ * step for every prompt. Returns "" when memory is not turned on (vault absent)
381
+ * so we never advertise memory the user hasn't enabled. Runs once at session
382
+ * construction.
383
+ */
384
+ export async function memoryIndexHint(): Promise<string> {
385
+ let st: Awaited<ReturnType<typeof vaultStatus>>;
386
+ try {
387
+ st = await vaultStatus();
388
+ } catch {
389
+ return "";
390
+ }
391
+ if (!st.exists) return ""; // memory not turned on → don't advertise it
392
+ const articleCount = st.articleCount > 0 ? `${st.articleCount} article${st.articleCount === 1 ? "" : "s"}` : "no user articles yet";
393
+ const refCount = st.referenceCount > 0 ? ` plus ${st.referenceCount} read-only mlx-bun reference doc${st.referenceCount === 1 ? "" : "s"}` : "";
394
+ return (
395
+ `\n\nThe user's personal memory is on (${articleCount}${refCount}). Use it only when the answer depends on ` +
396
+ `user-specific continuity or missing personal context; do not use it for weather, current public facts, ` +
397
+ `generic web research, or ordinary coding/file tasks when the current files are enough. See the \`memory\` skill for the memory_* workflow.`
398
+ );
399
+ }