mlx-bun 0.0.8 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +112 -20
  2. package/docs/design/batching-v2-plan.md +515 -0
  3. package/docs/design/bucketing-stage.md +196 -0
  4. package/docs/design/compat-cli-surface-design.md +11 -11
  5. package/docs/design/docs-reorg-plan.md +1 -1
  6. package/docs/design/dreaming-nightly-pipeline.md +291 -0
  7. package/docs/design/dspark-speculative-decoding.md +106 -0
  8. package/docs/design/generic-model-support.md +381 -0
  9. package/docs/design/memory-inference-path.md +422 -0
  10. package/docs/design/memory-synthesis.md +287 -0
  11. package/docs/design/minicpm5-decode-megakernel.md +484 -0
  12. package/docs/design/mlx-lm-tool-parity-plan.md +535 -0
  13. package/docs/design/orpo-dynamic-lambda.md +41 -0
  14. package/docs/design/orpo-training.md +35 -5
  15. package/docs/design/parallel-slots.md +12 -1
  16. package/docs/design/parity-tier-dag.md +13 -5
  17. package/docs/design/spec-decode-larger-targets.md +229 -0
  18. package/docs/design/tauri-desktop-app.md +2 -2
  19. package/docs/design/the-dreaming-handoff.md +141 -0
  20. package/docs/design/the-dreaming-master-plan.md +446 -0
  21. package/docs/design/write-pipeline-entity.md +468 -0
  22. package/docs/investigations/curve-bisector-routing.md +1 -1
  23. package/docs/investigations/decode-roofline-lookagain.md +262 -0
  24. package/docs/investigations/dspark-handoff.md +58 -0
  25. package/docs/investigations/kernel-perf-review-2026-07.md +92 -0
  26. package/docs/planning/PRODUCT_ROADMAP.md +2 -2
  27. package/docs/planning/curve-sampler-research-plan.md +331 -0
  28. package/docs/planning/memory-docs-and-dag-plan.md +185 -0
  29. package/docs/planning/pi-integration-review.md +178 -0
  30. package/docs/planning/release-notes-v0.0.9.md +122 -0
  31. package/docs/planning/web-ui-pass-plan.md +213 -0
  32. package/docs/planning/website-readme-pass-plan.md +224 -0
  33. package/docs/reference/cli.md +255 -0
  34. package/docs/reference/embedding.md +7 -2
  35. package/docs/reference/library-api.md +22 -3
  36. package/docs/reference/memory.md +5 -5
  37. package/docs/reference/models.md +137 -0
  38. package/docs/reference/orpo-quickstart.md +6 -0
  39. package/docs/reference/server-api.md +187 -9
  40. package/docs/reference/server-config.md +40 -9
  41. package/docs/reference/training.md +25 -11
  42. package/package.json +1 -1
  43. package/src/anthropic.ts +22 -0
  44. package/src/assets/curve-designer.html +1 -1
  45. package/src/chat-template.ts +25 -7
  46. package/src/cli.ts +1029 -86
  47. package/src/config.ts +4 -1
  48. package/src/download.ts +99 -8
  49. package/src/embed.ts +77 -0
  50. package/src/eval/perplexity.ts +164 -0
  51. package/src/eval/runner.ts +15 -6
  52. package/src/eval/tasks/bfcl.ts +1 -1
  53. package/src/generate.ts +141 -18
  54. package/src/harness-pi.ts +43 -17
  55. package/src/index.ts +2 -0
  56. package/src/lora.ts +2 -2
  57. package/src/memory/article.ts +558 -0
  58. package/src/memory/chunk-validation.ts +150 -0
  59. package/src/memory/chunk.ts +305 -12
  60. package/src/memory/cluster.ts +344 -17
  61. package/src/memory/coverage.ts +134 -0
  62. package/src/memory/crosslink.ts +0 -0
  63. package/src/memory/db.ts +184 -45
  64. package/src/memory/entity.ts +203 -0
  65. package/src/memory/gate.ts +187 -0
  66. package/src/memory/history.ts +152 -0
  67. package/src/memory/ingest.ts +324 -11
  68. package/src/memory/model.ts +321 -0
  69. package/src/memory/normalize.ts +671 -0
  70. package/src/memory/parse.ts +22 -0
  71. package/src/memory/pipeline.ts +242 -30
  72. package/src/memory/prompts.ts +42 -0
  73. package/src/memory/query.ts +334 -0
  74. package/src/memory/reconcile.ts +734 -0
  75. package/src/memory/redirect.ts +108 -0
  76. package/src/memory/reindex.ts +325 -0
  77. package/src/memory/resolve.ts +307 -0
  78. package/src/memory/route.ts +272 -0
  79. package/src/memory/schedule.ts +6 -4
  80. package/src/memory/stages.ts +774 -0
  81. package/src/memory/synthesize.ts +1629 -20
  82. package/src/memory/tools.ts +446 -96
  83. package/src/memory/vault.ts +462 -35
  84. package/src/memory/wikify.ts +1057 -16
  85. package/src/mlx/ffi.ts +3 -0
  86. package/src/mlx/ops.ts +46 -0
  87. package/src/model/compiled-decode.ts +43 -8
  88. package/src/model/factory.ts +29 -8
  89. package/src/model/flash-attention.ts +4 -2
  90. package/src/model/fused-decode-kernel.ts +14 -7
  91. package/src/model/fused-mlp-kernel.ts +169 -0
  92. package/src/model/fused-swiglu-kernel.ts +240 -0
  93. package/src/model/gemma4-base.ts +62 -6
  94. package/src/model/gemma4.ts +33 -2
  95. package/src/model/minicpm5.ts +106 -8
  96. package/src/model/qwen3.ts +276 -0
  97. package/src/model/steel-linear-kernel.ts +96 -0
  98. package/src/model/support.ts +38 -8
  99. package/src/model/universal/archs.ts +332 -0
  100. package/src/model/universal/dense.ts +576 -0
  101. package/src/model/universal/modules.ts +163 -0
  102. package/src/model/universal/rope.ts +366 -0
  103. package/src/pi-provider.ts +1 -1
  104. package/src/pi-terminal.ts +29 -8
  105. package/src/pi-web.ts +19 -2
  106. package/src/quantize/config-writer.ts +1 -1
  107. package/src/registry.ts +215 -7
  108. package/src/responses.ts +23 -0
  109. package/src/sampler.ts +220 -9
  110. package/src/serve/batch-scheduler.ts +272 -80
  111. package/src/serve/generation-gateway.ts +86 -18
  112. package/src/server.ts +612 -49
  113. package/src/spec/drafter.ts +81 -49
  114. package/src/spec/dspark/data-dflash.ts +156 -0
  115. package/src/spec/dspark/data.ts +201 -0
  116. package/src/spec/dspark/generate-dflash.ts +193 -0
  117. package/src/spec/dspark/generate.ts +290 -0
  118. package/src/spec/dspark/loss.ts +213 -0
  119. package/src/spec/dspark/module-dflash.ts +361 -0
  120. package/src/spec/dspark/module.ts +540 -0
  121. package/src/spec/dspark/sample.ts +112 -0
  122. package/src/spec/generate.ts +40 -19
  123. package/src/status-page.html +1 -1
  124. package/src/tokenizer.ts +7 -0
  125. package/src/train/flash-cce.ts +70 -15
  126. package/src/train/fuse.ts +243 -0
  127. package/src/train/index.ts +3 -0
  128. package/src/train/job.ts +8 -0
  129. package/src/train/loss.ts +289 -30
  130. package/src/train/prefix-shared.ts +33 -5
  131. package/src/train/segmented.ts +67 -11
  132. package/src/train/trainer.ts +92 -28
  133. package/src/web/skills/memory/SKILL.md +40 -17
@@ -3,39 +3,70 @@
3
3
  // These are pi custom tools (defineTool), registered alongside the coding +
4
4
  // web tools in both front doors (pi-terminal.ts, pi-web.ts). They're read-only,
5
5
  // so they auto-allow (not in GATED_TOOLS) — consulting your own memory never
6
- // needs an approval gate. They read ~/.mlx-bun/wiki via src/memory/vault.ts.
6
+ // needs an approval gate. They read ~/.mlx-bun/wiki via src/memory/vault.ts +
7
+ // the deterministic navigation index in src/memory/query.ts.
7
8
  //
8
- // Mirrors lucien's read surface: search (discovery) → focused read (toc /
9
- // section / links) → full read / list / status. Knowing *when* to consult is
10
- // driven by these tool descriptions and the bundled memory skill (pi surfaces
11
- // its description and loads the body on demand); the system prompt carries only
12
- // a one-line presence hint (memoryIndexHint), not a title dump.
9
+ // The read surface is a deliberate FIND → READ path, never a vector search and
10
+ // never an infobox-field filter (the infobox is CONTENT the model reads, not a
11
+ // query target — there is NO memory_infobox_query):
12
+ //
13
+ // memory_resolve (name/alias → the article) ─┐ FIND the article
14
+ // memory_category (category/type/series → members) ─┘ deterministically
15
+ // memory_read (TOC + lead by default) ─┐ READ it small:
16
+ // memory_section (one section's body — THE DEFAULT) ─┘ TOC → one section
17
+ // memory_links / memory_infobox (follow the graph / read the infobox)
18
+ // memory_search (LAST resort: substring fallback when FIND failed)
19
+ //
20
+ // Silent-colleague contract (ported from lucien's LUCIEN_INSTRUCTIONS): look the
21
+ // article up BEFORE answering; each article is the consolidated current position,
22
+ // so read it rather than reconstructing from raw history; then speak as a
23
+ // continuation, never "per the wiki…". Knowing *when* to consult is driven by
24
+ // these tool descriptions + the bundled memory skill (pi surfaces its description
25
+ // and loads the body on demand); the system prompt carries only a one-line
26
+ // presence hint (memoryIndexHint), not a title dump.
13
27
 
14
28
  import { defineTool, type ToolDefinition } from "@earendil-works/pi-coding-agent";
15
29
  import { Type } from "typebox";
30
+ import {
31
+ infoboxFieldValues,
32
+ parseInfobox,
33
+ parseSeriesBanner,
34
+ type Infobox,
35
+ } from "./article";
36
+ import {
37
+ articlesInCategory,
38
+ buildMemoryIndex,
39
+ resolveName,
40
+ type MemoryIndex,
41
+ } from "./query";
16
42
  import { scheduleStatus } from "./schedule";
17
43
  import {
18
44
  extractSection,
19
- getArticleLinks,
45
+ extractWikilinkTargets,
20
46
  listArticles,
21
47
  listReferenceDocs,
22
48
  parseToc,
23
49
  readArticle,
50
+ resolveWikilinkToStem,
24
51
  searchArticles,
25
52
  vaultRoot,
26
53
  vaultStatus,
27
54
  type SearchHit,
28
55
  } from "./vault";
29
56
 
30
- /** Tool names, exported so the front doors can add them to the allowlist. */
57
+ /** Tool names, exported so the front doors can add them to the allowlist. The
58
+ * order is the recommended call order: FIND (resolve/category) → READ (read/
59
+ * section) → follow (links/infobox) → reference utilities → search LAST. */
31
60
  export const MEMORY_TOOL_NAMES = [
32
- "memory_search",
61
+ "memory_resolve",
62
+ "memory_category",
33
63
  "memory_read",
34
- "memory_toc",
35
64
  "memory_section",
36
65
  "memory_links",
66
+ "memory_infobox",
37
67
  "memory_list",
38
68
  "memory_status",
69
+ "memory_search",
39
70
  ] as const;
40
71
 
41
72
  export const REFERENCE_TOOL_NAMES = [
@@ -45,6 +76,15 @@ export const REFERENCE_TOOL_NAMES = [
45
76
  ] as const;
46
77
 
47
78
  const MAX_SEARCH_HITS = 30;
79
+ /** memory_search is the demoted fallback: only the strongest few summaries. */
80
+ const MAX_MEMORY_SUMMARIES = 8;
81
+ /** A whole-article read above this many bytes degrades to TOC+lead+steer. */
82
+ const MAX_FULL_READ_BYTES = 8 * 1024;
83
+ /** memory_category page size (rows) and per-row lead budget. */
84
+ const MAX_CATEGORY_ROWS = 40;
85
+ const CATEGORY_LEAD_CHARS = 140;
86
+ /** memory_links: cap per origin group before "+N more". */
87
+ const MAX_LINKS_PER_GROUP = 25;
48
88
 
49
89
  function textResult(text: string): { content: [{ type: "text"; text: string }]; details: Record<string, never> } {
50
90
  return { content: [{ type: "text", text }], details: {} };
@@ -68,17 +108,34 @@ async function memoryUnavailableMessage(): Promise<string | null> {
68
108
  return `Memory is not set up yet at ${st.root}. Run \`mlx-bun memory init\` to create the local Markdown wiki, or ask the user before setting it up.`;
69
109
  }
70
110
 
111
+ /** Title form of a stem (underscores → spaces). */
112
+ function titleOf(stem: string): string {
113
+ return stem.replace(/_/g, " ");
114
+ }
115
+
116
+ /** Casefold + collapse whitespace/underscores (mirrors query.ts/reindex.ts). */
117
+ function normalizeAlias(s: string): string {
118
+ return s.trim().toLowerCase().replace(/_/g, " ").replace(/\s+/g, " ").trim();
119
+ }
120
+
121
+ /** Truncate to a budget, marking the cut so callers never silently lose text. */
122
+ function clip(text: string, max: number): string {
123
+ const t = text.trim();
124
+ return t.length <= max ? t : `${t.slice(0, max - 1).trimEnd()}…`;
125
+ }
126
+
71
127
  /** Compact, model-friendly rendering of search hits grouped by relevance. */
72
128
  function formatSearch(
73
129
  query: string,
74
130
  summaries: { article: string; occurrences: number; matched_terms?: string[] }[],
75
131
  hits: SearchHit[],
76
- opts: { corpusLabel?: string; readTool?: string } = {},
132
+ opts: { corpusLabel?: string; readTool?: string; maxSummaries?: number } = {},
77
133
  ): string {
78
134
  const corpus = opts.corpusLabel ?? "personal-memory articles";
79
135
  const readTool = opts.readTool ?? "memory_read";
136
+ const maxSummaries = opts.maxSummaries ?? 12;
80
137
  if (summaries.length === 0) return `No ${corpus} match "${query}".`;
81
- const top = summaries.slice(0, 12);
138
+ const top = summaries.slice(0, maxSummaries);
82
139
  const lines = [`${corpus[0]!.toUpperCase()}${corpus.slice(1)} matching "${query}" (most relevant first):`, ""];
83
140
  for (const s of top) {
84
141
  const terms = s.matched_terms && s.matched_terms.length ? ` [${s.matched_terms.join(", ")}]` : "";
@@ -92,85 +149,182 @@ function formatSearch(
92
149
  lines.push(`- ${where}:${h.line} ${h.excerpt}`);
93
150
  }
94
151
  }
95
- lines.push("", `Read a full result with ${readTool} (e.g. ${readTool} article="${top[0]!.article}").`);
152
+ lines.push("", `Read a result with ${readTool} (e.g. ${readTool} stem="${top[0]!.article}").`);
96
153
  return lines.join("\n");
97
154
  }
98
155
 
99
- const memorySearchTool = defineTool({
100
- name: "memory_search",
101
- label: "Search Memory",
156
+ // ---- index access -----------------------------------------------------
157
+
158
+ /** Build the (mtime-cached) navigation index over the live vault. */
159
+ function index(): MemoryIndex {
160
+ return buildMemoryIndex(vaultRoot());
161
+ }
162
+
163
+ function kindOf(idx: MemoryIndex, stem: string): string {
164
+ return idx.infoboxByStem.get(stem)?.entityKind ?? "thing";
165
+ }
166
+
167
+ function leadOf(idx: MemoryIndex, stem: string): string | null {
168
+ return idx.leadByStem.get(stem) ?? null;
169
+ }
170
+
171
+ // ---- FIND: resolve by name/alias --------------------------------------
172
+
173
+ const memoryResolveTool = defineTool({
174
+ name: "memory_resolve",
175
+ label: "Resolve Memory Article",
102
176
  description:
103
- "Search the user's local personal-memory articles for user-specific continuity: prior conversations, named personal projects, people, preferences, decisions, or history. " +
104
- "Use when the answer depends on missing personal context; do not use for weather, current public facts, generic web research, or ordinary coding/file tasks when current files are sufficient. " +
105
- "Returns ranked article matches and sample lines; follow up with memory_read to read a relevant article in full.",
177
+ "FIND the user's article for a name or alias — the first move when the user names a specific thing, person, project, or topic. " +
178
+ "Deterministic alias lookup (no model, no substring search): returns the matched article's stem, title, kind, and lead so you can then read it. " +
179
+ "On a miss it offers up to 3 near candidates. Look the article up before answering rather than guessing from raw history.",
106
180
  parameters: Type.Object({
107
- query: Type.String({ description: "Word or phrase to search for across the user's memory articles." }),
108
- limit: Type.Optional(Type.Number({ description: `Max sample line-hits to return (default ${MAX_SEARCH_HITS}).` })),
181
+ surface: Type.String({ description: "The name or alias as the user said it (a short form or the way they referred to the thing)." }),
109
182
  }),
110
183
  execute: async (_id, params) => {
111
- const query = params.query.trim();
112
- if (!query) return textResult("memory_search needs a non-empty query.");
113
- const limit = Math.max(1, Math.min(MAX_SEARCH_HITS, Math.round(params.limit ?? MAX_SEARCH_HITS)));
184
+ const surface = params.surface.trim();
185
+ if (!surface) return textResult("memory_resolve needs a name or alias.");
114
186
  try {
115
187
  const unavailable = await memoryUnavailableMessage();
116
188
  if (unavailable) return textResult(unavailable);
117
- const { summaries, hits } = await searchArticles(vaultRoot(), query, { limit, scope: "articles" });
118
- return textResult(formatSearch(query, summaries, hits));
189
+ const idx = index();
190
+ const stem = resolveName(idx, surface);
191
+ if (stem) {
192
+ const lead = leadOf(idx, stem);
193
+ return textResult(
194
+ [
195
+ `${titleOf(stem)} (stem: ${stem}, kind: ${kindOf(idx, stem)})`,
196
+ lead ? `\n${clip(lead, 360)}` : "",
197
+ `\n\nRead it with memory_read stem="${stem}" (TOC + lead), then memory_section for the part you need.`,
198
+ ].join(""),
199
+ );
200
+ }
201
+ // Miss: deterministic token-overlap candidates (no substring, no model).
202
+ const qTokens = new Set(normalizeAlias(surface).split(" ").filter(Boolean));
203
+ const scored = new Map<string, number>();
204
+ for (const [alias, st] of idx.aliasToStem) {
205
+ let shared = 0;
206
+ for (const tok of alias.split(" ")) if (qTokens.has(tok)) shared++;
207
+ if (shared > 0) scored.set(st, Math.max(scored.get(st) ?? 0, shared));
208
+ }
209
+ const candidates = [...scored.entries()]
210
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
211
+ .slice(0, 3)
212
+ .map(([st]) => st);
213
+ if (candidates.length === 0) {
214
+ return textResult(`No article resolves "${surface}". Try memory_category to find articles about a topic, or memory_search as a last resort.`);
215
+ }
216
+ return textResult(
217
+ `No exact match for "${surface}". Did you mean one of:\n` +
218
+ candidates.map((st) => `- ${titleOf(st)} (stem: ${st})`).join("\n") +
219
+ `\n\nResolve one with memory_read, or use memory_category for a topic.`,
220
+ );
119
221
  } catch (err) {
120
- return textResult(`memory_search failed: ${err instanceof Error ? err.message : String(err)}`);
222
+ return textResult(`memory_resolve failed: ${err instanceof Error ? err.message : String(err)}`);
121
223
  }
122
224
  },
123
225
  });
124
226
 
125
- const memoryReadTool = defineTool({
126
- name: "memory_read",
127
- label: "Read Memory",
227
+ // ---- FIND: articles about a category / type / series ------------------
228
+
229
+ const memoryCategoryTool = defineTool({
230
+ name: "memory_category",
231
+ label: "Articles in Category",
128
232
  description:
129
- "Read one full Markdown article from the user's personal memory. " +
130
- "Use after memory_search when user-specific prior context is needed. " +
131
- "Pass the article stem exactly as returned by memory_search or memory_list (underscored, e.g. \"Archie_Project\").",
233
+ "FIND the articles ABOUT a topic by membership — the \"articles about X\" finder. " +
234
+ "Pass exactly one of category, type, or series; returns the member articles as \"stem · lead\" rows. " +
235
+ "This is deterministic set membership (declared [[Category:…]] / infobox type / series banner), NOT a search and NOT an infobox-field filter. " +
236
+ "Use it to land on the right article, then memory_read its TOC and memory_section the one part you need.",
132
237
  parameters: Type.Object({
133
- article: Type.String({ description: "Article stem to read (underscored filename, no .md, e.g. \"Archie_Project\")." }),
238
+ category: Type.Optional(Type.String({ description: "Category name — the grouping declared as [[Category:<Name>]] near the top of member articles." })),
239
+ type: Type.Optional(Type.String({ description: "Infobox type label (the article's `type:` field value)." })),
240
+ series: Type.Optional(Type.String({ description: "Series name from a *Part of a series on [[X]]* banner." })),
134
241
  }),
135
242
  execute: async (_id, params) => {
136
- const article = params.article.trim();
137
- if (!article) return textResult("memory_read needs an article name.");
243
+ const category = params.category?.trim();
244
+ const type = params.type?.trim();
245
+ const series = params.series?.trim();
246
+ const picked = [category, type, series].filter((v) => v && v.length > 0);
247
+ if (picked.length !== 1) {
248
+ return textResult("memory_category needs exactly one of: category, type, or series.");
249
+ }
138
250
  try {
139
251
  const unavailable = await memoryUnavailableMessage();
140
252
  if (unavailable) return textResult(unavailable);
141
- if (isReferenceId(article)) return textResult("memory_read is for personal-memory articles. Use reference_read for Reference/ documents.");
142
- const { content } = await readArticle(vaultRoot(), article);
143
- return textResult(content);
253
+ const idx = index();
254
+ let stems: string[];
255
+ let label: string;
256
+ if (category) {
257
+ stems = articlesInCategory(idx, category);
258
+ label = `Category:${category}`;
259
+ } else if (type) {
260
+ stems = [...(idx.typeToStems.get(type) ?? [])];
261
+ label = `type:${type}`;
262
+ } else {
263
+ stems = [...(idx.seriesToStems.get(series!) ?? [])];
264
+ label = `series:${series}`;
265
+ }
266
+ if (stems.length === 0) return textResult(`No articles in ${label}.`);
267
+ const page = stems.slice(0, MAX_CATEGORY_ROWS);
268
+ const lines = [`Articles in ${label} (${stems.length}):`, ""];
269
+ for (const stem of page) {
270
+ const lead = leadOf(idx, stem);
271
+ lines.push(lead ? `- ${stem} · ${clip(lead, CATEGORY_LEAD_CHARS)}` : `- ${stem}`);
272
+ }
273
+ if (stems.length > page.length) lines.push(` …and ${stems.length - page.length} more.`);
274
+ lines.push("", "Open one with memory_read, then memory_section for the relevant part.");
275
+ return textResult(lines.join("\n"));
144
276
  } catch (err) {
145
- return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
277
+ return textResult(`memory_category failed: ${err instanceof Error ? err.message : String(err)}`);
146
278
  }
147
279
  },
148
280
  });
149
281
 
150
- const memoryTocTool = defineTool({
151
- name: "memory_toc",
152
- label: "Memory TOC",
282
+ // ---- READ: TOC + lead by default; full only on force -------------------
283
+
284
+ function tocAndLead(content: string, stem: string, idx: MemoryIndex, note?: string): string {
285
+ const lead = leadOf(idx, stem);
286
+ const toc = parseToc(content);
287
+ const lines = [`${titleOf(stem)} (stem: ${stem})`];
288
+ if (lead) lines.push("", clip(lead, 480));
289
+ if (note) lines.push("", note);
290
+ lines.push("", "Sections:");
291
+ if (toc.length === 0) lines.push("- (no headings)");
292
+ else for (const h of toc) lines.push(`${" ".repeat(Math.max(0, h.depth - 1))}- ${h.title} (#${h.anchor})`);
293
+ lines.push("", `Read the one relevant section with memory_section stem="${stem}" anchor="…". Whole article: memory_read stem="${stem}" force=true.`);
294
+ return lines.join("\n");
295
+ }
296
+
297
+ const memoryReadTool = defineTool({
298
+ name: "memory_read",
299
+ label: "Read Memory Article",
153
300
  description:
154
- "List the Markdown headings in one memory article, with anchors. Use before memory_section when the article is long or you need a specific part.",
301
+ "Open one article. BY DEFAULT returns its TOC + lead so you can pick the one section you need — small focused reads are the norm, never a whole-article dump. " +
302
+ "Pass force=true for the full prose+infobox, but only when you genuinely need the whole article (large articles still degrade to TOC+lead). " +
303
+ "The article is the user's consolidated current position on its topic; read it before answering rather than reconstructing from history. " +
304
+ "Pass the article stem (underscored, e.g. an \"<Article_Stem>\" filename without .md).",
155
305
  parameters: Type.Object({
156
- article: Type.String({ description: "Article stem (underscored filename, no .md, e.g. \"Archie_Project\")." }),
306
+ stem: Type.String({ description: "Article stem (underscored filename, no .md — e.g. \"<Article_Stem>\")." }),
307
+ force: Type.Optional(Type.Boolean({ description: "Return the FULL article instead of TOC+lead. Use sparingly." })),
157
308
  }),
158
309
  execute: async (_id, params) => {
159
- const article = params.article.trim();
160
- if (!article) return textResult("memory_toc needs an article name.");
310
+ const stem = params.stem.trim();
311
+ if (!stem) return textResult("memory_read needs an article stem.");
161
312
  try {
162
313
  const unavailable = await memoryUnavailableMessage();
163
314
  if (unavailable) return textResult(unavailable);
164
- if (isReferenceId(article)) return textResult("memory_toc is for personal-memory articles. Use reference_read for Reference/ documents.");
165
- const { content } = await readArticle(vaultRoot(), article);
166
- const toc = parseToc(content);
167
- if (toc.length === 0) return textResult(`${article} has no Markdown headings.`);
168
- return textResult(
169
- `Headings in ${article}:\n\n` +
170
- toc.map((h) => `${" ".repeat(Math.max(0, h.depth - 1))}- ${h.title} (#${h.anchor})`).join("\n"),
171
- );
315
+ if (isReferenceId(stem)) return textResult("memory_read is for personal-memory articles. Use reference_read for Reference/ documents.");
316
+ const { content } = await readArticle(vaultRoot(), stem);
317
+ const idx = index();
318
+ if (!params.force) return textResult(tocAndLead(content, stem, idx));
319
+ const bytes = Buffer.byteLength(content, "utf8");
320
+ if (bytes > MAX_FULL_READ_BYTES) {
321
+ return textResult(
322
+ tocAndLead(content, stem, idx, `(This article is large — ${(bytes / 1024).toFixed(1)} KB. Read just the section you need.)`),
323
+ );
324
+ }
325
+ return textResult(content);
172
326
  } catch (err) {
173
- return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
327
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_resolve or memory_category to find the right article.`);
174
328
  }
175
329
  },
176
330
  });
@@ -179,62 +333,205 @@ const memorySectionTool = defineTool({
179
333
  name: "memory_section",
180
334
  label: "Read Memory Section",
181
335
  description:
182
- "Read one section from a memory article by heading anchor. Use memory_toc or memory_search first to find anchors; this is cheaper than reading a long article in full.",
336
+ "Read ONE section of an article by its heading anchor. This is the DEFAULT read: find the article, scan its TOC with memory_read, then read just the relevant section here — much cheaper than the whole article. " +
337
+ "Get anchors from memory_read's TOC.",
183
338
  parameters: Type.Object({
184
- article: Type.String({ description: "Article stem (underscored filename, no .md, e.g. \"Archie_Project\")." }),
185
- anchor: Type.String({ description: "Heading anchor, e.g. \"origin\" or \"current-design\"." }),
339
+ stem: Type.String({ description: "Article stem (underscored, e.g. \"<Article_Stem>\")." }),
340
+ anchor: Type.String({ description: "Heading anchor (the slug of a section heading from memory_read's TOC, e.g. \"<section-anchor>\")." }),
186
341
  }),
187
342
  execute: async (_id, params) => {
188
- const article = params.article.trim();
343
+ const stem = params.stem.trim();
189
344
  const anchor = params.anchor.trim().replace(/^#/, "");
190
- if (!article) return textResult("memory_section needs an article name.");
345
+ if (!stem) return textResult("memory_section needs an article stem.");
191
346
  if (!anchor) return textResult("memory_section needs a heading anchor.");
192
347
  try {
193
348
  const unavailable = await memoryUnavailableMessage();
194
349
  if (unavailable) return textResult(unavailable);
195
- if (isReferenceId(article)) return textResult("memory_section is for personal-memory articles. Use reference_read for Reference/ documents.");
196
- const { content } = await readArticle(vaultRoot(), article);
350
+ if (isReferenceId(stem)) return textResult("memory_section is for personal-memory articles. Use reference_read for Reference/ documents.");
351
+ const { content } = await readArticle(vaultRoot(), stem);
197
352
  const section = extractSection(content, anchor);
198
- if (!section) return textResult(`No section #${anchor} found in ${article}. Try memory_toc article=\"${article}\".`);
353
+ if (!section) return textResult(`No section #${anchor} in ${stem}. Use memory_read stem="${stem}" to see the TOC.`);
199
354
  return textResult(section);
200
355
  } catch (err) {
201
- return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
356
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_resolve or memory_category to find the right article.`);
202
357
  }
203
358
  },
204
359
  });
205
360
 
361
+ // ---- FOLLOW: links grouped by origin ----------------------------------
362
+
363
+ const WIKILINK_RE = /\[\[([^\]]+)\]\]/g;
364
+
365
+ /** Resolved outbound links grouped by where they appear in the article. Mirrors
366
+ * the reindex `via` taxonomy (infobox/series/seealso/prose); red links (no
367
+ * existing article) are dropped, matching getArticleLinks. */
368
+ function groupedOutbound(content: string, stem: string, stemSet: Set<string>, box: Infobox | null): {
369
+ infobox: string[];
370
+ series: string[];
371
+ seealso: string[];
372
+ prose: string[];
373
+ } {
374
+ const dedupe = (arr: string[]): string[] => [...new Set(arr)].sort((a, b) => a.localeCompare(b));
375
+ const resolve = (raw: string): string | null => {
376
+ const base = raw.split("|")[0]!.split("#")[0]!.trim();
377
+ const r = resolveWikilinkToStem(base, stemSet);
378
+ return r && r !== stem ? r : null;
379
+ };
380
+
381
+ const infobox: string[] = [];
382
+ if (box) {
383
+ for (const field of box.fields) {
384
+ for (const value of infoboxFieldValues(field)) {
385
+ WIKILINK_RE.lastIndex = 0;
386
+ const m = WIKILINK_RE.exec(value);
387
+ if (m) {
388
+ const r = resolve(m[1]!);
389
+ if (r) infobox.push(r);
390
+ }
391
+ }
392
+ }
393
+ }
394
+
395
+ const series: string[] = [];
396
+ const banner = parseSeriesBanner(content);
397
+ if (banner) {
398
+ const r = resolve(banner);
399
+ if (r) series.push(r);
400
+ }
401
+
402
+ const seealso: string[] = [];
403
+ const prose: string[] = [];
404
+ const lines = content.split(/\r?\n/);
405
+ let inFence = false;
406
+ let section: string | null = null;
407
+ for (const line of lines) {
408
+ if (/^```/.test(line)) {
409
+ inFence = !inFence;
410
+ continue;
411
+ }
412
+ if (inFence) continue;
413
+ const hm = /^(#{1,6})\s+(.+)$/.exec(line);
414
+ if (hm) {
415
+ if (hm[1]!.length >= 2) section = hm[2]!.trim().toLowerCase();
416
+ continue;
417
+ }
418
+ const t = line.trim();
419
+ if (/^\*part of a series on \[\[/i.test(t)) continue;
420
+ if (/^categories:/i.test(t)) continue;
421
+ if (/^\*?\s*main article:/i.test(t)) continue;
422
+ WIKILINK_RE.lastIndex = 0;
423
+ let m: RegExpExecArray | null;
424
+ while ((m = WIKILINK_RE.exec(line)) !== null) {
425
+ const base = m[1]!.split("|")[0]!.split("#")[0]!.trim();
426
+ if (!base || /^conv:/i.test(base) || /^Category:/i.test(base)) continue;
427
+ const r = resolve(base);
428
+ if (!r) continue;
429
+ if (section === "see also") seealso.push(r);
430
+ else prose.push(r);
431
+ }
432
+ }
433
+
434
+ return { infobox: dedupe(infobox), series: dedupe(series), seealso: dedupe(seealso), prose: dedupe(prose) };
435
+ }
436
+
206
437
  const memoryLinksTool = defineTool({
207
438
  name: "memory_links",
208
439
  label: "Memory Links",
209
440
  description:
210
- "Show resolved outgoing and incoming wikilinks for one memory article. Use when nearby concepts/backlinks might matter to the answer.",
441
+ "Follow the wiki graph: the article's outgoing [[links]] grouped by where they come from (infobox / series / see also / prose), plus inbound backlinks. " +
442
+ "Use to hop to a neighbouring article (e.g. from one entity's infobox [[wikilink]] to the linked entity's article) when a topic genuinely spans articles.",
211
443
  parameters: Type.Object({
212
- article: Type.String({ description: "Article stem (underscored filename, no .md, e.g. \"Archie_Project\")." }),
444
+ stem: Type.String({ description: "Article stem (underscored, e.g. \"<Article_Stem>\")." }),
213
445
  }),
214
446
  execute: async (_id, params) => {
215
- const article = params.article.trim();
216
- if (!article) return textResult("memory_links needs an article name.");
447
+ const stem = params.stem.trim();
448
+ if (!stem) return textResult("memory_links needs an article stem.");
217
449
  try {
218
450
  const unavailable = await memoryUnavailableMessage();
219
451
  if (unavailable) return textResult(unavailable);
220
- if (isReferenceId(article)) return textResult("memory_links is for personal-memory articles; Reference/ documents do not participate in the personal-memory link graph.");
221
- const { outbound, inbound } = await getArticleLinks(vaultRoot(), article);
222
- const lines = [`Links for ${article}:`, "", "Outbound:"];
223
- lines.push(outbound.length ? outbound.map((s) => `- ${s}`).join("\n") : "- (none)");
224
- lines.push("", "Inbound:");
225
- lines.push(inbound.length ? inbound.map((s) => `- ${s}`).join("\n") : "- (none)");
226
- return textResult(lines.join("\n"));
452
+ if (isReferenceId(stem)) return textResult("memory_links is for personal-memory articles; Reference/ documents are not in the link graph.");
453
+ const { content } = await readArticle(vaultRoot(), stem);
454
+ const stems = await listArticles(vaultRoot());
455
+ const stemSet = new Set(stems);
456
+ if (!stemSet.has(stem)) return textResult(`Article not found: ${stem}.`);
457
+ const box = parseInfobox(content);
458
+ const groups = groupedOutbound(content, stem, stemSet, box);
459
+
460
+ // Inbound: who points here (any origin).
461
+ const inbound: string[] = [];
462
+ for (const other of stems) {
463
+ if (other === stem) continue;
464
+ try {
465
+ const oc = (await readArticle(vaultRoot(), other)).content;
466
+ const og = groupedOutbound(oc, other, stemSet, parseInfobox(oc));
467
+ if ([...og.infobox, ...og.series, ...og.seealso, ...og.prose].includes(stem)) inbound.push(other);
468
+ } catch {
469
+ // best-effort
470
+ }
471
+ }
472
+
473
+ const renderGroup = (title: string, arr: string[]): string => {
474
+ if (arr.length === 0) return `${title}: (none)`;
475
+ const shown = arr.slice(0, MAX_LINKS_PER_GROUP);
476
+ const extra = arr.length > shown.length ? ` +${arr.length - shown.length} more` : "";
477
+ return `${title}: ${shown.join(", ")}${extra}`;
478
+ };
479
+ return textResult(
480
+ [
481
+ `Links for ${stem}:`,
482
+ "",
483
+ renderGroup("infobox", groups.infobox),
484
+ renderGroup("series", groups.series),
485
+ renderGroup("see also", groups.seealso),
486
+ renderGroup("prose", groups.prose),
487
+ renderGroup("inbound", inbound.sort((a, b) => a.localeCompare(b))),
488
+ ].join("\n"),
489
+ );
490
+ } catch (err) {
491
+ return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_resolve or memory_category to find the right article.`);
492
+ }
493
+ },
494
+ });
495
+
496
+ // ---- READ the infobox as content (NOT a query target) -----------------
497
+
498
+ const memoryInfoboxTool = defineTool({
499
+ name: "memory_infobox",
500
+ label: "Read Memory Infobox",
501
+ description:
502
+ "Read an article's infobox as key:value CONTENT — the structured facts (the `type:`, `kind:`, relationship, and world-fact fields). " +
503
+ "This surfaces facts to READ, not a filter: there is no way to query/sort articles by an infobox field; to find articles use memory_category or memory_resolve.",
504
+ parameters: Type.Object({
505
+ stem: Type.String({ description: "Article stem (underscored, e.g. \"<Article_Stem>\")." }),
506
+ }),
507
+ execute: async (_id, params) => {
508
+ const stem = params.stem.trim();
509
+ if (!stem) return textResult("memory_infobox needs an article stem.");
510
+ try {
511
+ const unavailable = await memoryUnavailableMessage();
512
+ if (unavailable) return textResult(unavailable);
513
+ if (isReferenceId(stem)) return textResult("memory_infobox is for personal-memory articles.");
514
+ const idx = index();
515
+ if (!idx.infoboxByStem.has(stem)) return textResult(`Article not found: ${stem}.`);
516
+ const box = idx.infoboxByStem.get(stem) ?? null;
517
+ if (!box || box.fields.length === 0) return textResult(`${stem} has no infobox.`);
518
+ return textResult(
519
+ `Infobox for ${stem} (facts to read, not a filter):\n\n` +
520
+ box.fields.map((f) => `${f.key}: ${f.value}`).join("\n"),
521
+ );
227
522
  } catch (err) {
228
- return textResult(`${err instanceof Error ? err.message : String(err)} — try memory_search to find the right article name.`);
523
+ return textResult(`memory_infobox failed: ${err instanceof Error ? err.message : String(err)}`);
229
524
  }
230
525
  },
231
526
  });
232
527
 
528
+ // ---- overview + status ------------------------------------------------
529
+
233
530
  const memoryListTool = defineTool({
234
531
  name: "memory_list",
235
532
  label: "List Memory",
236
533
  description:
237
- "List the titles of the user's personal-memory articles. Use as an overview, or as a fallback when memory_search doesn't surface what you expected.",
534
+ "List the stems of the user's personal-memory articles. An overview/fallback when memory_resolve and memory_category don't surface what you expected — not a first move.",
238
535
  parameters: Type.Object({}),
239
536
  execute: async () => {
240
537
  try {
@@ -243,8 +540,7 @@ const memoryListTool = defineTool({
243
540
  const articles = await listArticles(vaultRoot());
244
541
  if (articles.length === 0) return textResult("The user's memory has no personal articles yet.");
245
542
  return textResult(
246
- `${articles.length} personal-memory article(s):\n\n` +
247
- articles.map((s) => `- ${s}`).join("\n"),
543
+ `${articles.length} personal-memory article(s):\n\n` + articles.map((s) => `- ${s}`).join("\n"),
248
544
  );
249
545
  } catch (err) {
250
546
  return textResult(`memory_list failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -252,16 +548,37 @@ const memoryListTool = defineTool({
252
548
  },
253
549
  });
254
550
 
551
+ /** Last synthesis-pipeline write to the vault, from its git log (every
552
+ * pipeline write commits with a "memory: …" message — synthesize.ts,
553
+ * reconcile.ts, wikify.ts, stages.ts). null when the vault has no git repo
554
+ * or no synthesis commit yet. */
555
+ async function lastSynthesisCommit(root: string): Promise<{ date: string; subject: string } | null> {
556
+ try {
557
+ const proc = Bun.spawn(
558
+ ["git", "log", "-1", "--grep=^memory: ", "--format=%cI\t%s"],
559
+ { cwd: root, stdout: "pipe", stderr: "ignore" },
560
+ );
561
+ const out = (await new Response(proc.stdout).text()).trim();
562
+ if ((await proc.exited) !== 0 || !out) return null;
563
+ const tab = out.indexOf("\t");
564
+ if (tab === -1) return null;
565
+ return { date: out.slice(0, tab), subject: out.slice(tab + 1) };
566
+ } catch {
567
+ return null;
568
+ }
569
+ }
570
+
255
571
  const memoryStatusTool = defineTool({
256
572
  name: "memory_status",
257
573
  label: "Memory Status",
258
574
  description:
259
- "Show read-only status for the user's memory: vault path, setup state, article count, git state, and nightly schedule state. Use when the user asks whether memory is on or where it lives.",
575
+ "Show read-only status for the user's memory: vault path, setup state, article count, git state, last synthesis run, and nightly schedule state. Use when the user asks whether memory is on or where it lives.",
260
576
  parameters: Type.Object({}),
261
577
  execute: async () => {
262
578
  try {
263
579
  const st = await vaultStatus();
264
580
  const sched = await scheduleStatus();
581
+ const last = st.isGitRepo ? await lastSynthesisCommit(st.root) : null;
265
582
  const recent = st.recentArticles.length
266
583
  ? st.recentArticles.slice(0, 5).map((r) => ` - ${r.article} (${new Date(r.mtimeMs).toISOString()})`).join("\n")
267
584
  : " - (none)";
@@ -272,8 +589,8 @@ const memoryStatusTool = defineTool({
272
589
  `- articles: ${st.articleCount}`,
273
590
  `- read-only references: ${st.referenceCount}`,
274
591
  `- git: ${st.isGitRepo ? "tracked" : "not tracked"}`,
275
- `- last synthesis: not available yet (M1 synthesis is stubbed)`,
276
- `- synthesis: M1 stub (manual/nightly writes are not implemented yet)`,
592
+ `- last synthesis: ${last ? `${last.date} (${last.subject})` : "no synthesis run recorded yet"}`,
593
+ "- synthesis: available — `mlx-bun memory synthesize` runs the full local pipeline (conversations → articles); the nightly job runs it automatically when scheduled",
277
594
  `- nightly: ${sched.installed ? (sched.loaded ? "scheduled" : "installed but not loaded") : "not scheduled"}`,
278
595
  sched.installed ? `- launchd plist: ${sched.plistPath}` : "- setup command: mlx-bun memory init",
279
596
  "- recent changed articles:",
@@ -285,6 +602,33 @@ const memoryStatusTool = defineTool({
285
602
  },
286
603
  });
287
604
 
605
+ // ---- search: the demoted last-resort fallback -------------------------
606
+
607
+ const memorySearchTool = defineTool({
608
+ name: "memory_search",
609
+ label: "Search Memory (fallback)",
610
+ description:
611
+ "LAST-RESORT substring search across memory articles — use ONLY when name/category lookup (memory_resolve, memory_category) has failed across a couple of phrasings. " +
612
+ "It does not understand meaning; prefer the deterministic finders first. Returns the top few article matches; follow up with memory_read.",
613
+ parameters: Type.Object({
614
+ query: Type.String({ description: "Word or phrase to substring-match across memory articles." }),
615
+ limit: Type.Optional(Type.Number({ description: `Max sample line-hits to return (default ${MAX_SEARCH_HITS}).` })),
616
+ }),
617
+ execute: async (_id, params) => {
618
+ const query = params.query.trim();
619
+ if (!query) return textResult("memory_search needs a non-empty query.");
620
+ const limit = Math.max(1, Math.min(MAX_SEARCH_HITS, Math.round(params.limit ?? MAX_SEARCH_HITS)));
621
+ try {
622
+ const unavailable = await memoryUnavailableMessage();
623
+ if (unavailable) return textResult(unavailable);
624
+ const { summaries, hits } = await searchArticles(vaultRoot(), query, { limit, scope: "articles" });
625
+ return textResult(formatSearch(query, summaries, hits, { maxSummaries: MAX_MEMORY_SUMMARIES }));
626
+ } catch (err) {
627
+ return textResult(`memory_search failed: ${err instanceof Error ? err.message : String(err)}`);
628
+ }
629
+ },
630
+ });
631
+
288
632
  // ---- reference docs (separate from personal memory) -------------------
289
633
 
290
634
  const referenceSearchTool = defineTool({
@@ -356,13 +700,15 @@ const referenceListTool = defineTool({
356
700
 
357
701
  export function createMemoryTools(): ToolDefinition[] {
358
702
  return [
359
- memorySearchTool,
703
+ memoryResolveTool,
704
+ memoryCategoryTool,
360
705
  memoryReadTool,
361
- memoryTocTool,
362
706
  memorySectionTool,
363
707
  memoryLinksTool,
708
+ memoryInfoboxTool,
364
709
  memoryListTool,
365
710
  memoryStatusTool,
711
+ memorySearchTool,
366
712
  ];
367
713
  }
368
714
 
@@ -372,14 +718,14 @@ export function createReferenceTools(): ToolDefinition[] {
372
718
 
373
719
  /**
374
720
  * One gated line for the system prompt: states that memory is ON for this user
375
- * and points at the `memory` skill. NOT a title dump and NOT a how-to. pi
376
- * already auto-injects every skill's name+description+location
377
- * (formatSkillsForPrompt) and loads the SKILL.md body on demand, so the skill
378
- * owns the detailed workflow. This hint is intentionally soft: memory is a
379
- * scoped retrieval capability for user-specific continuity, not a default first
380
- * step for every prompt. Returns "" when memory is not turned on (vault absent)
381
- * so we never advertise memory the user hasn't enabled. Runs once at session
382
- * construction.
721
+ * and points at the `memory` skill, plus the FIND→READ order and the
722
+ * silent-colleague contract. NOT a title dump and NOT a how-to. pi already
723
+ * auto-injects every skill's name+description+location (formatSkillsForPrompt)
724
+ * and loads the SKILL.md body on demand, so the skill owns the detailed
725
+ * workflow. This hint is intentionally soft: memory is a scoped retrieval
726
+ * capability for user-specific continuity, not a default first step for every
727
+ * prompt. Returns "" when memory is not turned on (vault absent) so we never
728
+ * advertise memory the user hasn't enabled. Runs once at session construction.
383
729
  */
384
730
  export async function memoryIndexHint(): Promise<string> {
385
731
  let st: Awaited<ReturnType<typeof vaultStatus>>;
@@ -392,8 +738,12 @@ export async function memoryIndexHint(): Promise<string> {
392
738
  const articleCount = st.articleCount > 0 ? `${st.articleCount} article${st.articleCount === 1 ? "" : "s"}` : "no user articles yet";
393
739
  const refCount = st.referenceCount > 0 ? ` plus ${st.referenceCount} read-only mlx-bun reference doc${st.referenceCount === 1 ? "" : "s"}` : "";
394
740
  return (
395
- `\n\nThe user's personal memory is on (${articleCount}${refCount}). Use it only when the answer depends on ` +
396
- `user-specific continuity or missing personal context; do not use it for weather, current public facts, ` +
397
- `generic web research, or ordinary coding/file tasks when the current files are enough. See the \`memory\` skill for the memory_* workflow.`
741
+ `\n\nThe user's personal memory is on (${articleCount}${refCount}). When the answer depends on user-specific ` +
742
+ `continuity or missing personal context, look it up BEFORE answering: FIND the article (memory_resolve a name, ` +
743
+ `or memory_category for a topic), open its TOC with memory_read, then memory_section the one relevant part — ` +
744
+ `each article is the user's consolidated current position, so read it rather than reconstructing from history. ` +
745
+ `Whole-article reads and memory_search are last resorts. Use what you find silently, as a continuation — never ` +
746
+ `"per the wiki…". Do not use memory for weather, current public facts, generic web research, or ordinary ` +
747
+ `coding/file tasks when the current files are enough. See the \`memory\` skill for the workflow.`
398
748
  );
399
749
  }