mlx-bun 0.0.7 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +112 -20
  2. package/docs/design/batching-v2-plan.md +515 -0
  3. package/docs/design/bucketing-stage.md +196 -0
  4. package/docs/design/compat-cli-surface-design.md +11 -11
  5. package/docs/design/diffusion-gemma-port.md +602 -0
  6. package/docs/design/docs-reorg-plan.md +1 -1
  7. package/docs/design/dreaming-nightly-pipeline.md +291 -0
  8. package/docs/design/dspark-speculative-decoding.md +106 -0
  9. package/docs/design/generic-model-support.md +381 -0
  10. package/docs/design/memory-inference-path.md +422 -0
  11. package/docs/design/memory-synthesis.md +287 -0
  12. package/docs/design/minicpm5-decode-megakernel.md +484 -0
  13. package/docs/design/mlx-lm-tool-parity-plan.md +535 -0
  14. package/docs/design/orpo-dynamic-lambda.md +41 -0
  15. package/docs/design/orpo-training.md +35 -5
  16. package/docs/design/parallel-slots.md +12 -1
  17. package/docs/design/parity-tier-dag.md +13 -5
  18. package/docs/design/spec-decode-larger-targets.md +229 -0
  19. package/docs/design/tauri-desktop-app.md +2 -2
  20. package/docs/design/the-dreaming-handoff.md +141 -0
  21. package/docs/design/the-dreaming-master-plan.md +446 -0
  22. package/docs/design/write-pipeline-entity.md +468 -0
  23. package/docs/investigations/curve-bisector-routing.md +1 -1
  24. package/docs/investigations/decode-roofline-lookagain.md +262 -0
  25. package/docs/investigations/dspark-handoff.md +58 -0
  26. package/docs/investigations/kernel-perf-review-2026-07.md +92 -0
  27. package/docs/planning/PRODUCT_ROADMAP.md +2 -2
  28. package/docs/planning/curve-sampler-research-plan.md +331 -0
  29. package/docs/planning/memory-docs-and-dag-plan.md +185 -0
  30. package/docs/planning/pi-integration-review.md +178 -0
  31. package/docs/planning/release-notes-v0.0.9.md +122 -0
  32. package/docs/planning/web-ui-pass-plan.md +213 -0
  33. package/docs/planning/website-readme-pass-plan.md +224 -0
  34. package/docs/reference/cli.md +255 -0
  35. package/docs/reference/embedding.md +7 -2
  36. package/docs/reference/library-api.md +22 -3
  37. package/docs/reference/memory.md +5 -5
  38. package/docs/reference/models.md +137 -0
  39. package/docs/reference/orpo-quickstart.md +6 -0
  40. package/docs/reference/server-api.md +187 -9
  41. package/docs/reference/server-config.md +40 -9
  42. package/docs/reference/training.md +25 -11
  43. package/package.json +1 -1
  44. package/src/anthropic.ts +22 -0
  45. package/src/assets/curve-designer.html +1 -1
  46. package/src/chat-template.ts +25 -7
  47. package/src/cli.ts +1029 -86
  48. package/src/config.ts +54 -3
  49. package/src/diffusion/diffusion-generate.ts +436 -0
  50. package/src/download.ts +99 -8
  51. package/src/embed.ts +77 -0
  52. package/src/eval/perplexity.ts +164 -0
  53. package/src/eval/runner.ts +19 -7
  54. package/src/eval/tasks/bfcl.ts +1 -1
  55. package/src/generate.ts +196 -19
  56. package/src/harness-pi.ts +43 -17
  57. package/src/index.ts +2 -0
  58. package/src/lora.ts +2 -2
  59. package/src/memory/article.ts +558 -0
  60. package/src/memory/chunk-validation.ts +150 -0
  61. package/src/memory/chunk.ts +305 -12
  62. package/src/memory/cluster.ts +344 -17
  63. package/src/memory/coverage.ts +134 -0
  64. package/src/memory/crosslink.ts +0 -0
  65. package/src/memory/db.ts +184 -45
  66. package/src/memory/entity.ts +203 -0
  67. package/src/memory/gate.ts +187 -0
  68. package/src/memory/history.ts +152 -0
  69. package/src/memory/ingest.ts +324 -11
  70. package/src/memory/model.ts +321 -0
  71. package/src/memory/normalize.ts +671 -0
  72. package/src/memory/parse.ts +22 -0
  73. package/src/memory/pipeline.ts +242 -30
  74. package/src/memory/prompts.ts +42 -0
  75. package/src/memory/query.ts +334 -0
  76. package/src/memory/reconcile.ts +734 -0
  77. package/src/memory/redirect.ts +108 -0
  78. package/src/memory/reindex.ts +325 -0
  79. package/src/memory/resolve.ts +307 -0
  80. package/src/memory/route.ts +272 -0
  81. package/src/memory/schedule.ts +6 -4
  82. package/src/memory/stages.ts +774 -0
  83. package/src/memory/synthesize.ts +1629 -20
  84. package/src/memory/tools.ts +446 -96
  85. package/src/memory/vault.ts +462 -35
  86. package/src/memory/wikify.ts +1057 -16
  87. package/src/mlx/ffi.ts +11 -0
  88. package/src/mlx/ops.ts +113 -0
  89. package/src/model/compiled-decode.ts +43 -8
  90. package/src/model/diffusion-gemma.ts +1026 -0
  91. package/src/model/factory.ts +35 -8
  92. package/src/model/flash-attention.ts +4 -2
  93. package/src/model/fused-decode-kernel.ts +14 -7
  94. package/src/model/fused-mlp-kernel.ts +169 -0
  95. package/src/model/fused-swiglu-kernel.ts +240 -0
  96. package/src/model/gemma4-base.ts +62 -6
  97. package/src/model/gemma4.ts +33 -2
  98. package/src/model/minicpm5.ts +106 -8
  99. package/src/model/qwen3.ts +276 -0
  100. package/src/model/steel-linear-kernel.ts +96 -0
  101. package/src/model/support.ts +45 -7
  102. package/src/model/universal/archs.ts +332 -0
  103. package/src/model/universal/dense.ts +576 -0
  104. package/src/model/universal/modules.ts +163 -0
  105. package/src/model/universal/rope.ts +366 -0
  106. package/src/pi-provider.ts +1 -1
  107. package/src/pi-terminal.ts +29 -8
  108. package/src/pi-web.ts +19 -2
  109. package/src/quantize/config-writer.ts +1 -1
  110. package/src/registry.ts +215 -7
  111. package/src/responses.ts +23 -0
  112. package/src/sampler.ts +220 -9
  113. package/src/serve/batch-scheduler.ts +272 -80
  114. package/src/serve/generation-gateway.ts +90 -18
  115. package/src/server.ts +643 -50
  116. package/src/spec/drafter.ts +81 -49
  117. package/src/spec/dspark/data-dflash.ts +156 -0
  118. package/src/spec/dspark/data.ts +201 -0
  119. package/src/spec/dspark/generate-dflash.ts +193 -0
  120. package/src/spec/dspark/generate.ts +290 -0
  121. package/src/spec/dspark/loss.ts +213 -0
  122. package/src/spec/dspark/module-dflash.ts +361 -0
  123. package/src/spec/dspark/module.ts +540 -0
  124. package/src/spec/dspark/sample.ts +112 -0
  125. package/src/spec/generate.ts +40 -19
  126. package/src/status-page.html +1 -1
  127. package/src/tokenizer.ts +7 -0
  128. package/src/train/diffusion-lora.ts +216 -0
  129. package/src/train/flash-cce.ts +70 -15
  130. package/src/train/fuse.ts +243 -0
  131. package/src/train/index.ts +3 -0
  132. package/src/train/job.ts +8 -0
  133. package/src/train/loss.ts +289 -30
  134. package/src/train/prefix-shared.ts +33 -5
  135. package/src/train/segmented.ts +67 -11
  136. package/src/train/trainer.ts +92 -28
  137. package/src/vision/diffusion-vision.ts +620 -0
  138. package/src/web/skills/memory/SKILL.md +40 -17
@@ -1,29 +1,1638 @@
1
- // mlx-bun memory — synthesis stage 4: SYNTHESIZE. ⚠️ M1 STUB.
1
+ // mlx-bun memory — SYNTHESIZE / CREATE flow (P6-T3).
2
2
  //
3
- // For each bucket touched this run, integrate its new chunks into the bucket's
4
- // article (local-model call) — create-or-update, conservatively. The hard,
5
- // proven gate is inherited from lucien: preserve prose, ≥70% word-count floor,
6
- // every original conv: citation survives, integrate rather than overwrite. A
7
- // weak local pass is gated to NO-OP rather than allowed to corrupt the vault.
8
- // Mirrors lucien's synthesize-dispatch.ts / synthesize-update.ts.
3
+ // The write pipeline's cold-start branch: turn an entity's accumulated chunks
4
+ // into a fresh Wikipedia-shaped article. Entered ONLY when the P1-T5/P5-T4
5
+ // CREATE rule fired for the entity (≥3 routed chunks, or ≥1 substantive
6
+ // ownership/decision chunk). The complementary UPDATE branch (fold one chunk
7
+ // into an existing section) is P7's section-patch — not built here.
9
8
  //
10
- // See docs/design/memory-system.md → "Synthesis runs on the local model".
9
+ // The decomposition is the whole point (capacity-by-decomposition): we NEVER
10
+ // hold a whole article in the model's head. CREATE is
11
+ // sub-cluster → OUTLINE (a clean TOC of single-topic sections) → per-section
12
+ // DRAFT (each section drafted from empty, the SAME bounded op as a patch) →
13
+ // seed INFOBOX → assemble.
14
+ // Sharp section structure is the primary quality goal: the read path's
15
+ // TOC→section reads ride this outline.
16
+ //
17
+ // Citations are deterministic by construction (AI does judgment, CODE enforces
18
+ // invariants): we assign each source conversation a stable `[^N]` up front and
19
+ // generate the `## References` `[^N]:` definition lines ourselves in the fixed
20
+ // Lucien standard — `[^N]: `conv:HASH` (YYYY-MM-DD, source) — desc`, HASH = the
21
+ // first 8 lowercase hex of the conversation UUID with hyphens stripped. The
22
+ // model only PLACES the markers; it never writes a definition or invents a hash.
23
+ // This sidesteps the most common base-model defect (malformed/fabricated
24
+ // footnotes) entirely.
25
+ //
26
+ // Citation/wikilink rules are ported from lucien `scripts/synthesize.ts`
27
+ // (SYNTHESIS_PROMPT_BOOTSTRAP). Every CREATE output is run through the P6-T1
28
+ // gate (a weak pass → NO-OP, vault unchanged) and the P6-T2 NORMALIZE pass, then
29
+ // committed via `commitVault`; the integrated (chunk, section) edges are written
30
+ // to the `synthesized_chunk_sections` ledger.
31
+
32
+ import { mkdir, readFile, writeFile } from "node:fs/promises";
33
+ import { join } from "node:path";
34
+
35
+ import {
36
+ articleStructure,
37
+ fenceLabel,
38
+ isFenceLine,
39
+ parseInfobox,
40
+ repairFences,
41
+ type EntityKind,
42
+ } from "./article";
43
+ import { LEAD_ANCHOR } from "./cluster";
44
+ import type { MemoryStore } from "./db";
45
+ import { checkFootnoteIntegrity, gateEdit, type GateVerdict } from "./gate";
46
+ import {
47
+ buildContradictionClassifierPrompt,
48
+ classifyContradiction,
49
+ type ContradictionKind,
50
+ } from "./history";
51
+ import { callLocal, MAX_OUTPUT_TOKENS } from "./model";
52
+ import { hasStubMarker, normalizeArticle } from "./normalize";
53
+ import { parseLines } from "./parse";
54
+ import { loadMetaPolicy } from "./prompts";
55
+ import {
56
+ articlesDir,
57
+ commitVault,
58
+ extractSection,
59
+ listArticles,
60
+ parseToc,
61
+ resolveWikilinkToStem,
62
+ vaultRoot,
63
+ } from "./vault";
64
+
65
+ // ---- ported citation / wikilink rules (lucien SYNTHESIS_PROMPT_BOOTSTRAP) ----
66
+
67
+ /** Use ONLY content from the provided chunk(s); never copy names/values from these instructions. */
68
+ const NO_COPY_GUARD = `Use ONLY content from the provided chunk(s)/source notes below; never copy any name, value, or example from these instructions into the article.`;
69
+
70
+ /** Wikilink contract: exact stem, underscores, never the spaced phantom form. */
71
+ const WIKILINK_RULES = `WIKILINKS — link to OTHER ARTICLES below using the exact article stem, underscores not spaces: [[Exact_Stem]] or with display text [[Exact_Stem|display text]]. The spaced form [[Exact Stem]] resolves to nothing and creates a broken orphan link; it is forbidden. Only link to a stem that appears in OTHER ARTICLES.`;
72
+
73
+ /** Citation contract: place the GIVEN markers, never write defs or hashes. */
74
+ const CITATION_RULES = `CITATIONS — cite a substantive claim by placing one of the GIVEN footnote markers ([^1], [^2], …) immediately after it, e.g. "<claim>.[^1]". Two sources for one sentence: adjacent markers, no space — [^1][^2]. Reuse the same marker every time you cite that source. Do NOT write any footnote DEFINITION line, do NOT invent or guess a hash, do NOT emit a ## References section — the definitions are generated for you. NEVER fabricate a citation: an uncited true statement is fine, but only ever use a marker FROM THE LIST ABOVE — never a placeholder letter or any marker not in that list. FORBIDDEN — never emit HTML <sup>, #ref-/#cite- links, escaped [\\[1\\]] markers, or [[1]] wikilink markers; [^N] is the ONLY citation syntax.`;
75
+
76
+ /** Date contract (CREATE): every source note is tagged with its date. The model
77
+ * must treat the LATER-DATED note as the user's CURRENT position when notes
78
+ * conflict — chronology is a first-class signal, not just processing order. Kept
79
+ * schematic (no copyable concrete value) per the small-model parroting failure. */
80
+ const DATE_RESOLVE_RULES = `DATES — each source note below is tagged with its date in [YYYY-MM-DD] form (the same date that appears in its [^N] footnote). When two notes CONFLICT (a changed favourite, a reversed preference, an updated fact), the LATER-DATED note is the user's CURRENT position: resolve toward it. Write a superseded value as a past stage ("originally …, now …"), never as still-true.`;
81
+
82
+ // ---- conv:HASH + footnote scaffolding --------------------------------------
83
+
84
+ /** A filesystem-safe bare article stem from a free-form entity name: collapse
85
+ * whitespace to `_`, map path separators (a value containing `/`) and other
86
+ * filename-hostile characters to `-`, and strip leading dots, so a model-minted
87
+ * name like `<value>/<value>` yields a bare `<value>-<value>` stem (never a
88
+ * `articles/…` subpath). Stems with no separators are unchanged. The original
89
+ * surface survives as an `aliases:` entry, so name resolution still reaches the
90
+ * article. */
91
+ export function entityStem(entity: string): string {
92
+ const stem = entity
93
+ .trim()
94
+ .replace(/[/\\]+/g, "-")
95
+ .replace(/\s+/g, "_")
96
+ .replace(/[<>:"|?*]+/g, "")
97
+ .replace(/_+/g, "_")
98
+ .replace(/^\.+/, "")
99
+ .replace(/^[-_]+|[-_]+$/g, "");
100
+ return stem || "entity";
101
+ }
102
+
103
+ /** The fixed Lucien `conv:HASH`: the first 8 lowercase hex digits of the
104
+ * conversation UUID with hyphens stripped (`<8hex>-<4hex>-…` → `<8hex>`). An
105
+ * already-8-char hex id passes through unchanged. */
106
+ export function convHash(conv: string): string {
107
+ return conv.toLowerCase().replace(/[^0-9a-f]/g, "").slice(0, 8);
108
+ }
109
+
110
+ /** `YYYY-MM-DD` (UTC) from an epoch-millis timestamp; `undated` when missing. */
111
+ export function footnoteDate(ms: number | null | undefined): string {
112
+ if (ms == null || !Number.isFinite(ms)) return "undated";
113
+ return new Date(ms).toISOString().slice(0, 10);
114
+ }
115
+
116
+ /** The fixed Lucien footnote definition line for a source conversation as
117
+ * `[^n]: `conv:HASH` (YYYY-MM-DD, source)`. The single source of truth both
118
+ * CREATE (buildFootnoteMap) and PATCH (appendReferenceDef) generate defs from,
119
+ * so the model never writes — and cannot malform — a definition line. */
120
+ export function footnoteDefLine(n: number, conv: string, title: string, dateMs: number | null): string {
121
+ const source = (title ?? "").trim() || "conversation";
122
+ return `[^${n}]: \`conv:${convHash(conv)}\` (${footnoteDate(dateMs)}, ${source})`;
123
+ }
124
+
125
+ /** One source-conversation footnote: its stable per-article number, hash, and
126
+ * the generated `[^N]: `conv:HASH` (date, source)` definition line. */
127
+ export interface FootnoteEntry {
128
+ conv: string;
129
+ n: number;
130
+ hash: string;
131
+ source: string;
132
+ def: string;
133
+ }
134
+
135
+ /** Assign each distinct source conversation a stable `[^N]` (first-appearance
136
+ * order across the entity's chunks) and pre-generate its definition line in the
137
+ * fixed Lucien standard. The model places these markers; it never writes defs. */
138
+ export function buildFootnoteMap(chunks: SynthesisChunk[]): {
139
+ byConv: Map<string, FootnoteEntry>;
140
+ entries: FootnoteEntry[];
141
+ } {
142
+ const byConv = new Map<string, FootnoteEntry>();
143
+ const entries: FootnoteEntry[] = [];
144
+ for (const c of chunks) {
145
+ if (byConv.has(c.conv)) continue;
146
+ const n = entries.length + 1;
147
+ const hash = convHash(c.conv);
148
+ const source = (c.title ?? "").trim() || "conversation";
149
+ const def = footnoteDefLine(n, c.conv, c.title, c.dateMs);
150
+ const entry: FootnoteEntry = { conv: c.conv, n, hash, source, def };
151
+ byConv.set(c.conv, entry);
152
+ entries.push(entry);
153
+ }
154
+ return { byConv, entries };
155
+ }
156
+
157
+ // ---- chunk input -----------------------------------------------------------
158
+
159
+ /** One source chunk for synthesis: its id, conversation, reassembled text, and
160
+ * the metadata the footnote line needs. Text is NOT stored — the caller pulls
161
+ * it from `store.chunkText(id)`. */
162
+ export interface SynthesisChunk {
163
+ id: string;
164
+ conv: string;
165
+ label: string | null;
166
+ text: string;
167
+ title: string;
168
+ dateMs: number | null;
169
+ }
170
+
171
+ // ---- deterministic tokenizer (sub-clustering + see-also) -------------------
172
+
173
+ const TOK_STOP = new Set([
174
+ "the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with", "is",
175
+ "vs", "and", "its", "my", "your", "our", "their",
176
+ ]);
177
+
178
+ function tokens(s: string): Set<string> {
179
+ const out = new Set<string>();
180
+ for (const t of s.toLowerCase().split(/[^a-z0-9]+/)) {
181
+ if (t.length > 1 && !TOK_STOP.has(t)) out.add(t);
182
+ }
183
+ return out;
184
+ }
185
+
186
+ // ---- OUTLINE ---------------------------------------------------------------
187
+
188
+ const GENERIC_SECTION = new Set(["overview", "introduction", "summary", "details", "misc", "other"]);
189
+
190
+ /** Parse a model OUTLINE into clean single-topic section titles: strip bullet /
191
+ * heading / numbering markup, drop empties + bare generics, dedupe by anchor,
192
+ * cap at `max`. */
193
+ export function parseOutline(raw: string, max = 6): string[] {
194
+ const seen = new Set<string>();
195
+ const out: string[] = [];
196
+ for (const line of parseLines(raw)) {
197
+ const title = line
198
+ .replace(/^#{1,6}\s+/, "")
199
+ .replace(/^[-*+]\s+/, "")
200
+ .replace(/^\d+[.)]\s+/, "")
201
+ .replace(/[`*_]/g, "")
202
+ .trim();
203
+ if (!title) continue;
204
+ const key = title.toLowerCase();
205
+ if (seen.has(key)) continue;
206
+ seen.add(key);
207
+ out.push(title);
208
+ if (out.length >= max) break;
209
+ }
210
+ // An outline that is ONLY a generic single section is fine (→ stub); an empty
211
+ // outline falls back to one "Overview" so we always draft something.
212
+ if (out.length === 0) return ["Overview"];
213
+ return out;
214
+ }
215
+
216
+ /** Anchor (heading slug) for a section title — must match vault.ts slugifyHeading
217
+ * so the read path's TOC→section anchors line up. */
218
+ export function sectionAnchor(title: string): string {
219
+ return title
220
+ .trim()
221
+ .toLowerCase()
222
+ .replace(/[^a-z0-9]+/g, "-")
223
+ .replace(/^-+|-+$/g, "");
224
+ }
225
+
226
+ /**
227
+ * Assign each chunk to the single best-matching section by label/text token
228
+ * overlap (deterministic, GPU-free), so each section drafts only from its own
229
+ * sub-topic chunks and the (chunk → section) edge is recorded. A chunk with no
230
+ * overlap lands in the first section; sections that win no chunk are dropped by
231
+ * the caller so we never draft an empty section.
232
+ */
233
+ export function subClusterChunks(
234
+ chunks: SynthesisChunk[],
235
+ sections: string[],
236
+ ): Map<string, SynthesisChunk[]> {
237
+ const byAnchor = new Map<string, SynthesisChunk[]>();
238
+ const sectionToks = sections.map((t) => ({ title: t, anchor: sectionAnchor(t), toks: tokens(t) }));
239
+ for (const c of chunks) {
240
+ const ctoks = tokens(`${c.label ?? ""} ${c.text.slice(0, 400)}`);
241
+ let best = sectionToks[0]!;
242
+ let bestScore = -1;
243
+ for (const s of sectionToks) {
244
+ let score = 0;
245
+ for (const t of s.toks) if (ctoks.has(t)) score++;
246
+ if (score > bestScore) {
247
+ bestScore = score;
248
+ best = s;
249
+ }
250
+ }
251
+ const list = byAnchor.get(best.anchor) ?? [];
252
+ list.push(c);
253
+ byAnchor.set(best.anchor, list);
254
+ }
255
+ return byAnchor;
256
+ }
257
+
258
+ // ---- prompt builders -------------------------------------------------------
259
+
260
+ /** A `getOtherArticles`-style stem list (lucien synthesize.ts) so the model's
261
+ * [[wikilinks]] resolve to real articles. Capped to keep the prefill bounded. */
262
+ function otherArticlesBlock(stems: Iterable<string>, selfStem: string, cap = 200): string {
263
+ const list = [...stems].filter((s) => s !== selfStem).sort((a, b) => a.localeCompare(b)).slice(0, cap);
264
+ return list.length ? list.map((s) => `- ${s}`).join("\n") : "(none yet)";
265
+ }
266
+
267
+ /** Compact source block for the OUTLINE/INFOBOX/LEAD calls: each chunk DATE-TAGGED
268
+ * (so the model knows which note is most recent), its label + a short text excerpt
269
+ * (the model reads the gist, not the whole transcript). The `[YYYY-MM-DD]` tag is
270
+ * the same date carried in the chunk's `[^N]` footnote. */
271
+ function sourceDigest(chunks: SynthesisChunk[], perChunk = 600): string {
272
+ return chunks
273
+ .map(
274
+ (c, i) =>
275
+ `[${i + 1}] [${footnoteDate(c.dateMs)}] ${c.label ?? "(chunk)"}\n${c.text.slice(0, perChunk).trim()}`,
276
+ )
277
+ .join("\n\n");
278
+ }
279
+
280
+ function buildOutlinePrompt(entity: string, chunks: SynthesisChunk[], policy: string): string {
281
+ return (
282
+ `${policy}\n\n${NO_COPY_GUARD}\n\n` +
283
+ `You are outlining a personal-wiki article about "${entity}". Below are the source notes (chunk label + excerpt). ` +
284
+ `Propose a clean table of contents: 2 to 6 section headings, each a single sub-topic, in natural reading order. ` +
285
+ `Each heading is 2–5 words, Title Case, specific (not "Overview"/"Details"). Output ONE heading per line, nothing else, NONE if there is only one sub-topic.\n\n` +
286
+ `SOURCE NOTES:\n${sourceDigest(chunks)}\n`
287
+ );
288
+ }
289
+
290
+ function buildLeadPrompt(
291
+ entity: string,
292
+ chunks: SynthesisChunk[],
293
+ footnotes: Map<string, FootnoteEntry>,
294
+ otherArticles: string,
295
+ policy: string,
296
+ ): string {
297
+ return (
298
+ `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n${CITATION_RULES}\n\n${DATE_RESOLVE_RULES}\n\n` +
299
+ `${footnoteList(chunks, footnotes)}\n\n` +
300
+ `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
301
+ `Write the LEAD for a personal-wiki article about "${entity}": 2–4 sentences, present tense, naming **${entity}** in bold in the first sentence, describing what it is AND how the user relates to it (uses / owns / chose / considered). Write in the THIRD PERSON, Wikipedia-style — NEVER the first person ("I", "my"); refer to the user in the third person ("the user"). When the notes disagree, frame the user's CURRENT position from the LATEST-DATED note. Output ONLY the lead prose — no heading, no list, no References.\n\n` +
302
+ `SOURCE NOTES:\n${sourceDigest(chunks)}\n`
303
+ );
304
+ }
305
+
306
+ function buildInfoboxPrompt(
307
+ entity: string,
308
+ kind: EntityKind,
309
+ chunks: SynthesisChunk[],
310
+ otherArticles: string,
311
+ policy: string,
312
+ ): string {
313
+ return (
314
+ `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
315
+ `Produce the INFOBOX facts for a personal-wiki article about "${entity}". Output ONLY \`key: value\` lines, one per line, no fence, no prose. Rules:\n` +
316
+ `- snake_case keys.\n` +
317
+ `- A value that names another entity is a [[wikilink]] to a stem in OTHER ARTICLES (e.g. \`<entity_key>: [[Linked Entity]]\`).\n` +
318
+ `- Include \`type:\` (the emergent label — a short free-form noun phrase) when clear.\n` +
319
+ `- Include world-facts only if the notes support them; include relationship-facts (owned / acquired / used_for / chosen_over / opinion) the notes establish.\n` +
320
+ `- Do NOT emit \`kind:\` or \`aliases:\` — those are added for you.\n` +
321
+ `Emit nothing for a fact the notes do not support — never guess a spec.\n\n` +
322
+ `OTHER ARTICLES (link targets):\n${otherArticles}\n\n` +
323
+ `SOURCE NOTES:\n${sourceDigest(chunks)}\n`
324
+ );
325
+ }
326
+
327
+ /** The "Cite source X as [^n]" lines for the chunks in one draft call. */
328
+ function footnoteList(chunks: SynthesisChunk[], footnotes: Map<string, FootnoteEntry>): string {
329
+ const seen = new Set<number>();
330
+ const lines: string[] = [];
331
+ for (const c of chunks) {
332
+ const e = footnotes.get(c.conv);
333
+ if (!e || seen.has(e.n)) continue;
334
+ seen.add(e.n);
335
+ lines.push(`- [^${e.n}] = "${e.source}"`);
336
+ }
337
+ return lines.length
338
+ ? `Cite sources with these footnote markers (use ONLY these):\n${lines.join("\n")}`
339
+ : `(no citable sources for this section — leave it uncited)`;
340
+ }
341
+
342
+ function buildSectionPrompt(
343
+ entity: string,
344
+ sectionTitle: string,
345
+ chunks: SynthesisChunk[],
346
+ footnotes: Map<string, FootnoteEntry>,
347
+ otherArticles: string,
348
+ policy: string,
349
+ ): string {
350
+ const sources = chunks
351
+ .map((c) => {
352
+ const e = footnotes.get(c.conv);
353
+ const tag = e ? `[^${e.n}]` : "(uncited)";
354
+ return `SOURCE ${tag} [${footnoteDate(c.dateMs)}] — ${c.label ?? "chunk"}:\n${c.text.slice(0, 1400).trim()}`;
355
+ })
356
+ .join("\n\n---\n\n");
357
+ return (
358
+ `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n${CITATION_RULES}\n\n${DATE_RESOLVE_RULES}\n\n` +
359
+ `${footnoteList(chunks, footnotes)}\n\n` +
360
+ `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
361
+ `Draft ONLY the body of the "${sectionTitle}" section of the article about "${entity}". Factual prose in the THIRD PERSON, Wikipedia-style — NEVER the first person ("I", "my"); refer to the user in the third person ("the user"). These articles document what the user thinks, so they are NOT neutral: capture the user's positions and opinions accurately. Preserve specifics, and when two sources conflict fold the correction toward the user's LATEST-DATED position. Output ONLY this section's prose — do NOT repeat the "## ${sectionTitle}" heading, do NOT write other sections, do NOT write a References section.\n\n` +
362
+ `SOURCE MATERIAL:\n${sources}\n`
363
+ );
364
+ }
11
365
 
12
- import type { SynthesisEvent } from "./pipeline";
366
+ // ---- model-output sanitizers -----------------------------------------------
367
+
368
+ /**
369
+ * Meta / planning / refusal phrasings — the base model narrating ABOUT the task
370
+ * instead of writing the article body. A hit on any one rejects the draft (the
371
+ * real defect: a section body that literally read "The user is asking to draft
372
+ * the body prose for the <section name> section. The source material
373
+ * discusses: 1. …"). Tuned to NOT fire on legitimate personal-wiki prose, which
374
+ * IS about the user ("the user owned…", "the user weighed…").
375
+ */
376
+ const LEAK_PATTERNS: RegExp[] = [
377
+ /\bthe user (is asking|is requesting|wants me|asked me|requests|would like me|needs me|wanted me)\b/i,
378
+ /\bthe source material\b/i,
379
+ /\bthe source notes?\b/i,
380
+ /\bsource material discusses\b/i,
381
+ /\bbased on the (source|provided|notes|material)\b/i,
382
+ /\bdraft the body\b/i,
383
+ /\bbody prose for\b/i,
384
+ /\bfor the [^.\n]{0,40} section\b/i,
385
+ /\bi need to\b/i,
386
+ /\bi (will|shall) (draft|write|now|create|provide|outline|summarize)\b/i,
387
+ /\bi'?ll (draft|write|provide|outline|summarize)\b/i,
388
+ /\blet (me|us|'?s) (draft|write|outline|provide|start|begin|summarize)\b/i,
389
+ /\bhere ?(is|'?s) (the|a|my) (article|section|lead|prose|draft|body)\b/i,
390
+ /\bas an ai\b/i,
391
+ /\bi (cannot|can'?t|can not)\b/i,
392
+ /\bi'?m sorry\b/i,
393
+ /\bi am sorry\b/i,
394
+ /\bi apologize\b/i,
395
+ /\bi (do not|don'?t) have (enough|sufficient|any)\b/i,
396
+ /\bi'?m (unable|not able)\b/i,
397
+ /\bi am (unable|not able)\b/i,
398
+ /\bsorry,?\s+(i|but)\b/i,
399
+ ];
400
+
401
+ /**
402
+ * True when a drafted body is meta-commentary, a plan, or a refusal rather than
403
+ * article prose. Heuristic: any {@link LEAK_PATTERNS} hit, OR a leading
404
+ * numbered/bulleted PLAN that restates the task. Pure; empty text is not leaky.
405
+ */
406
+ export function isLeakyDraft(text: string): boolean {
407
+ const s = text.trim();
408
+ if (!s) return false;
409
+ for (const re of LEAK_PATTERNS) if (re.test(s)) return true;
410
+ // A leading numbered/bulleted PLAN that restates the task ("1. discuss the
411
+ // <subject>, 2. the <subject>") — a list item FIRST, plus task-restating vocabulary.
412
+ const firstLine = s.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
413
+ if (
414
+ /^(\d+[.)]|[-*+])\s+/.test(firstLine) &&
415
+ /\b(draft|discuss|summariz|describe the|outline|restate|the user|source material)\b/i.test(s)
416
+ ) {
417
+ return true;
418
+ }
419
+ return false;
420
+ }
421
+
422
+ /** Remove any model-emitted fenced ```info block from drafted text so the only
423
+ * infobox in the final article is the ONE code-constructed block. A fenced info
424
+ * block is a ``` fence whose info string is exactly `info`. */
425
+ export function stripFencedInfoBlocks(text: string): string {
426
+ const lines = text.split("\n");
427
+ const out: string[] = [];
428
+ let i = 0;
429
+ while (i < lines.length) {
430
+ if (isFenceLine(lines[i]!) && fenceLabel(lines[i]!) === "info") {
431
+ i++; // skip the opening ```info
432
+ while (i < lines.length && !isFenceLine(lines[i]!)) i++;
433
+ if (i < lines.length) i++; // skip the closing fence
434
+ continue;
435
+ }
436
+ out.push(lines[i]!);
437
+ i++;
438
+ }
439
+ return out.join("\n");
440
+ }
441
+
442
+ /** Count fenced ```info blocks in an article (the single-infobox invariant). */
443
+ export function countFencedInfoBlocks(content: string): number {
444
+ let count = 0;
445
+ let inFence = false;
446
+ for (const line of content.split(/\r?\n/)) {
447
+ if (isFenceLine(line)) {
448
+ if (!inFence && fenceLabel(line) === "info") count++;
449
+ inFence = !inFence;
450
+ }
451
+ }
452
+ return count;
453
+ }
454
+
455
+ /** Strip code fences, a repeated section heading, any model-emitted References /
456
+ * definition lines, a stray model-emitted infobox, and surrounding blank lines
457
+ * from a drafted section body. */
458
+ export function sanitizeSection(raw: string, title: string): string {
459
+ let s = stripFencedInfoBlocks(raw.trim()).trim();
460
+ s = s.replace(/^```[a-z]*\s*\n/i, "").replace(/\n```\s*$/i, "").trim();
461
+ // Drop any NON-numeric footnote marker (e.g. a parroted `[^N]` placeholder from
462
+ // the citation contract). Only `[^<digits>]` is a real, code-assigned marker.
463
+ s = s.replace(/ ?\[\^[^\]\d][^\]]*\]/g, "");
464
+ const wanted = title.trim().toLowerCase();
465
+ const kept: string[] = [];
466
+ let dropping = false;
467
+ for (const line of s.split("\n")) {
468
+ const t = line.trim();
469
+ const h = /^(#{1,6})\s+(.+)$/.exec(t);
470
+ if (h) {
471
+ const ht = h[2]!.trim().toLowerCase();
472
+ if (ht === wanted) continue; // model repeated our heading
473
+ if (ht === "references" || ht === "see also" || ht === "notes") {
474
+ dropping = true; // model spilled trailing meta sections — cut them
475
+ continue;
476
+ }
477
+ }
478
+ if (dropping) continue;
479
+ if (/^\[\^\d+\]:/.test(t)) continue; // a stray definition line — we own defs
480
+ kept.push(line);
481
+ }
482
+ // A drafted section that opened a ``` fence but never closed it would, once
483
+ // assembled, swallow every trailing section (## References / ## See also) of
484
+ // the WHOLE article. Balance the section's fences before it reaches assembly.
485
+ return repairFences(kept.join("\n").trim()).content.trim();
486
+ }
487
+
488
+ /** Strip fences / a leading heading / a stray infobox from a drafted lead,
489
+ * collapse to prose. */
490
+ export function sanitizeLead(raw: string): string {
491
+ let s = stripFencedInfoBlocks(raw.trim()).trim();
492
+ s = s.replace(/^```[a-z]*\s*\n/i, "").replace(/\n```\s*$/i, "").trim();
493
+ s = s.replace(/ ?\[\^[^\]\d][^\]]*\]/g, ""); // drop parroted non-numeric markers
494
+ const lines = s.split("\n").filter((l) => !/^#{1,6}\s+/.test(l.trim()) && !/^\[\^\d+\]:/.test(l.trim()));
495
+ // Balance any unclosed ``` so a lead fence can't swallow the rest of the article.
496
+ return repairFences(lines.join("\n").trim()).content.trim();
497
+ }
498
+
499
+ /**
500
+ * Physical spec-sheet keys describing a concrete object's hardware. They belong
501
+ * on a `thing` (a concrete named object) and must NEVER bleed onto a broad
502
+ * `domain` / `standard` / `person` / `project` entity — the policy-example
503
+ * parroting bug, where a broad DOMAIN article inherited a hardware spec key.
504
+ * Grounding the infobox in the entity's own kind blocks the leak deterministically.
505
+ */
506
+ export const PHYSICAL_SPEC_KEYS = new Set([
507
+ "mount", "sensor", "sensor_size", "focal_length", "aperture", "format",
508
+ "resolution", "megapixels", "coverage", "compatible_with", "lens_mount",
509
+ ]);
510
+
511
+ /**
512
+ * Parse model `key: value` lines into a clean info-block field list. Keeps only
513
+ * snake_case keys with a non-empty value (first occurrence wins), drops any
514
+ * model-supplied `kind`/`aliases` (we own those) and any physical-spec key that
515
+ * is ungrounded for this entity kind (a world-fact spec key on a `domain` is a
516
+ * parroted example, never a fact), then prepends `type:` (if any), the declared `kind:`,
517
+ * and appends `aliases:` when we have them. Always returns at least the `kind:`
518
+ * line, so the article always has a parseable infobox.
519
+ */
520
+ export function buildInfoboxFields(raw: string, kind: EntityKind, aliases: string[] = []): string[] {
521
+ const fields = new Map<string, string>();
522
+ for (const line of raw.split("\n")) {
523
+ const t = line.trim().replace(/^[-*+]\s+/, "");
524
+ const idx = t.indexOf(":");
525
+ if (idx <= 0) continue;
526
+ const key = t.slice(0, idx).trim();
527
+ const value = t.slice(idx + 1).trim();
528
+ if (!/^[a-z][a-z0-9_]*$/.test(key)) continue;
529
+ if (!value) continue;
530
+ if (key === "kind" || key === "aliases") continue;
531
+ // Ground the box in the entity's kind: a hardware spec on a non-`thing`
532
+ // entity is a parroted example, not a fact established by its chunks.
533
+ if (kind !== "thing" && PHYSICAL_SPEC_KEYS.has(key)) continue;
534
+ if (!fields.has(key)) fields.set(key, value);
535
+ }
536
+ const lines: string[] = [];
537
+ const type = fields.get("type");
538
+ if (type) lines.push(`type: ${type}`);
539
+ lines.push(`kind: ${kind}`);
540
+ for (const [k, v] of fields) {
541
+ if (k === "type") continue;
542
+ lines.push(`${k}: ${v}`);
543
+ }
544
+ const cleanAliases = [...new Set(aliases.map((a) => a.trim()).filter(Boolean))];
545
+ if (cleanAliases.length) lines.push(`aliases: ${cleanAliases.join(", ")}`);
546
+ return lines;
547
+ }
548
+
549
+ // ---- assembly --------------------------------------------------------------
550
+
551
+ export interface AssembledSection {
552
+ title: string;
553
+ anchor: string;
554
+ body: string;
555
+ }
556
+
557
+ /** Distinct wikilink targets in `text` that resolve to a known stem (excluding
558
+ * self / Category: / conv:) — the See also list, sorted. */
559
+ export function deriveSeeAlso(text: string, stems: Set<string>, selfStem: string): string[] {
560
+ const out = new Set<string>();
561
+ for (const m of text.matchAll(/\[\[([^\]]+?)\]\]/g)) {
562
+ const inner = m[1]!.split("|")[0]!.split("#")[0]!.trim();
563
+ if (!inner || /^conv:/i.test(inner) || /^Category:/i.test(inner)) continue;
564
+ const r = resolveWikilinkToStem(inner, stems);
565
+ if (r && r !== selfStem) out.add(r);
566
+ }
567
+ return [...out].sort((a, b) => a.localeCompare(b));
568
+ }
569
+
570
+ /** Compose the fixed article skeleton from its parts. `## See also` /
571
+ * `## References` are emitted only when non-empty; References is always last. */
572
+ export function assembleArticle(parts: {
573
+ stem: string;
574
+ infoboxFields: string[];
575
+ lead: string;
576
+ sections: AssembledSection[];
577
+ seeAlso: string[];
578
+ referenceDefs: string[];
579
+ }): string {
580
+ const title = parts.stem.replace(/_/g, " ");
581
+ const out: string[] = [`# ${title}`, ""];
582
+ if (parts.infoboxFields.length) {
583
+ out.push("```info", ...parts.infoboxFields, "```", "");
584
+ }
585
+ if (parts.lead.trim()) out.push(parts.lead.trim(), "");
586
+ for (const s of parts.sections) {
587
+ if (!s.body.trim()) continue;
588
+ out.push(`## ${s.title}`, "", s.body.trim(), "");
589
+ }
590
+ if (parts.seeAlso.length) {
591
+ out.push("## See also", "", ...parts.seeAlso.map((s) => `- [[${s}]]`), "");
592
+ }
593
+ if (parts.referenceDefs.length) {
594
+ out.push("## References", "", ...parts.referenceDefs, "");
595
+ }
596
+ const assembled = out.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
597
+ // Final assembly guard: a drafted section may still carry an unbalanced ``` that
598
+ // the per-section sanitizer missed; never emit an article whose fences are odd
599
+ // (it would read its own trailing ## References / ## See also as code). Closing
600
+ // the dangling fence before the first heading it would swallow keeps every
601
+ // trailing section a recognized heading.
602
+ const repaired = repairFences(assembled).content;
603
+ return repaired.replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
604
+ }
605
+
606
+ /** Distinct `[^N]` marker numbers present in body prose (definition lines and
607
+ * fenced blocks excluded — markers live in prose). */
608
+ function usedMarkers(text: string): Set<number> {
609
+ const used = new Set<number>();
610
+ for (const line of text.split("\n")) {
611
+ if (/^\[\^\d+\]:/.test(line.trim())) continue;
612
+ for (const m of line.matchAll(/\[\^(\d+)\]/g)) used.add(parseInt(m[1]!, 10));
613
+ }
614
+ return used;
615
+ }
616
+
617
+ /** Count sections (by the parsed skeleton) whose body carries ≥1 footnote
618
+ * marker — the "≥1 cited section" acceptance signal. */
619
+ export function countCitedSections(content: string): number {
620
+ const items = articleStructure(content);
621
+ const lines = content.split(/\r?\n/);
622
+ let cited = 0;
623
+ for (const it of items) {
624
+ if (it.kind !== "section") continue;
625
+ const body = lines.slice(it.startLine - 1, it.endLine).join("\n");
626
+ if (/\[\^\d+\](?!:)/.test(body)) cited++;
627
+ }
628
+ return cited;
629
+ }
630
+
631
+ // ---- CREATE (pure-ish: model via deps, no disk writes) ---------------------
632
+
633
+ /** Per-stage model call seam (defaults to the local synthesis stage). Injected
634
+ * by tests so the deterministic scaffolding runs with no GPU. */
635
+ export type SynthesisCall = (prompt: string, opts?: { maxTokens?: number }) => Promise<string>;
636
+
637
+ /** Retry instruction when a draft leaks meta/planning/refusal text. */
638
+ const ANTI_LEAK_HINT =
639
+ "Output ONLY the article prose for this section — no preamble, no meta-commentary, no plan, no restating the task.";
640
+
641
+ /**
642
+ * Draft prose that is NOT meta/planning/refusal leakage: draft once; if the
643
+ * sanitized body is {@link isLeakyDraft}, retry ONCE with a sharper instruction;
644
+ * if it still leaks, return "" so the caller DROPS the section rather than
645
+ * emitting garbage. The sanitizer (section vs lead) is injected.
646
+ */
647
+ async function draftCleanProse(
648
+ call: SynthesisCall,
649
+ prompt: string,
650
+ opts: { maxTokens?: number },
651
+ sanitize: (raw: string) => string,
652
+ ): Promise<string> {
653
+ let out = sanitize(await call(prompt, opts));
654
+ if (!isLeakyDraft(out)) return out;
655
+ out = sanitize(await call(`${prompt}\n\n${ANTI_LEAK_HINT}`, opts));
656
+ return isLeakyDraft(out) ? "" : out;
657
+ }
658
+
659
+ export interface CreateDeps {
660
+ /** Model call; defaults to `callLocal("synthesis", …)`. */
661
+ call?: SynthesisCall;
662
+ /** Other article stems (for wikilink resolution); defaults to the vault list. */
663
+ stems?: Set<string>;
664
+ /** Declared aliases to seed the infobox `aliases:` line. */
665
+ aliases?: string[];
666
+ /** Pre-existing article content for the gate's `before` (CREATE: ""). */
667
+ before?: string;
668
+ }
669
+
670
+ export interface CreateOutcome {
671
+ entity: string;
672
+ stem: string;
673
+ /** Final post-NORMALIZE markdown, or null when the gate vetoed the write. */
674
+ content: string | null;
675
+ action: "created" | "skipped";
676
+ /** Gate reason when skipped; null on success. */
677
+ reason: string | null;
678
+ /** (chunk_id, section_anchor) edges for the `synthesized_chunk_sections` ledger. */
679
+ chunkSections: { chunkId: string; anchor: string }[];
680
+ sectionAnchors: string[];
681
+ citedSections: number;
682
+ hasInfobox: boolean;
683
+ stub: boolean;
684
+ normalizeNotes: string[];
685
+ }
686
+
687
+ /**
688
+ * Build ONE entity article from its accumulated chunks, end to end:
689
+ * sub-cluster → OUTLINE → per-section DRAFT → seed INFOBOX → assemble →
690
+ * NORMALIZE → gate. Returns the final content (or null when the gate vetoes a
691
+ * weak pass). Pure with respect to disk — the caller persists.
692
+ */
693
+ export async function createArticle(
694
+ entity: string,
695
+ kind: EntityKind,
696
+ chunks: SynthesisChunk[],
697
+ deps: CreateDeps = {},
698
+ ): Promise<CreateOutcome> {
699
+ const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
700
+ const stem = entityStem(entity);
701
+ const stems = new Set(deps.stems ?? []);
702
+ stems.add(stem);
703
+ const otherArticles = otherArticlesBlock(stems, stem);
704
+ const policy = loadMetaPolicyQuiet(["Article_Conventions", "Infobox_Schemas", "Entities"]);
705
+ const { byConv, entries } = buildFootnoteMap(chunks);
706
+
707
+ // OUTLINE → sections; assign chunks; drop sections that won no chunk.
708
+ const outline = parseOutline(await call(buildOutlinePrompt(entity, chunks, policy), { maxTokens: MAX_OUTPUT_TOKENS }));
709
+ const clustered = subClusterChunks(chunks, outline);
710
+ const sections: AssembledSection[] = [];
711
+ const chunkSections: { chunkId: string; anchor: string }[] = [];
712
+ for (const title of outline) {
713
+ const anchor = sectionAnchor(title);
714
+ const secChunks = clustered.get(anchor);
715
+ if (!secChunks || secChunks.length === 0) continue;
716
+ const body = await draftCleanProse(
717
+ call,
718
+ buildSectionPrompt(entity, title, secChunks, byConv, otherArticles, policy),
719
+ { maxTokens: MAX_OUTPUT_TOKENS },
720
+ (raw) => sanitizeSection(raw, title),
721
+ );
722
+ // A leaky draft survives to "" after one retry — DROP the section (and its
723
+ // chunk edges); never emit meta/planning/refusal prose into the article.
724
+ if (!body.trim()) continue;
725
+ sections.push({ title, anchor, body });
726
+ for (const c of secChunks) chunkSections.push({ chunkId: c.id, anchor });
727
+ }
728
+
729
+ // LEAD + INFOBOX (bounded, single-purpose calls). The lead runs the same
730
+ // leak-rejection so a refusal/plan never lands as the article's abstract.
731
+ const lead = await draftCleanProse(
732
+ call,
733
+ buildLeadPrompt(entity, chunks, byConv, otherArticles, policy),
734
+ { maxTokens: MAX_OUTPUT_TOKENS },
735
+ sanitizeLead,
736
+ );
737
+ const infoboxFields = buildInfoboxFields(
738
+ await call(buildInfoboxPrompt(entity, kind, chunks, otherArticles, policy), { maxTokens: MAX_OUTPUT_TOKENS }),
739
+ kind,
740
+ deps.aliases ?? [],
741
+ );
742
+
743
+ // References: only the defs whose marker the model actually placed.
744
+ const bodyForMarkers = [lead, ...sections.map((s) => s.body)].join("\n");
745
+ const used = usedMarkers(bodyForMarkers);
746
+ const referenceDefs = entries.filter((e) => used.has(e.n)).map((e) => e.def);
747
+
748
+ const seeAlso = deriveSeeAlso([infoboxFields.join("\n"), bodyForMarkers].join("\n"), stems, stem);
749
+
750
+ const raw = assembleArticle({ stem, infoboxFields, lead, sections, seeAlso, referenceDefs });
751
+
752
+ // NORMALIZE (footnote bijection + renumber, wikilink canonicalization, infobox
753
+ // sort, structural guards, {{stub}} for thin), then the conservative gate.
754
+ const norm = normalizeArticle(raw, { stem, stems });
755
+
756
+ const base: Omit<CreateOutcome, "content" | "action" | "reason"> = {
757
+ entity,
758
+ stem,
759
+ chunkSections,
760
+ sectionAnchors: sections.map((s) => s.anchor),
761
+ citedSections: countCitedSections(norm.content),
762
+ hasInfobox: parseInfobox(norm.content) != null,
763
+ stub: hasStubMarker(norm.content),
764
+ normalizeNotes: norm.notes,
765
+ };
766
+
767
+ // Defect-2: every committed article MUST be footnote-bijective. NORMALIZE
768
+ // already repairs (drops orphan markers, renumbers, merges dup defs); if the
769
+ // result is STILL broken (an unrepairable malformed citation), veto the write
770
+ // rather than commit a citation-broken article.
771
+ const fn = checkFootnoteIntegrity(norm.content);
772
+ if (!fn.ok) {
773
+ return { ...base, content: null, action: "skipped", reason: `footnote bijection unrepairable: ${fn.errors.join("; ")}` };
774
+ }
775
+
776
+ // Defect-3: exactly ONE infobox — the code-constructed block. Drafted info
777
+ // blocks are stripped in sanitizeSection/sanitizeLead; this asserts they were,
778
+ // so a parroted second infobox can never reach the vault.
779
+ const infoboxCount = countFencedInfoBlocks(norm.content);
780
+ if (infoboxCount > 1) {
781
+ return { ...base, content: null, action: "skipped", reason: `multiple infoboxes (${infoboxCount})` };
782
+ }
783
+
784
+ const verdict: GateVerdict = gateEdit(deps.before ?? "", norm.content);
785
+ if (!verdict.ok) {
786
+ return { ...base, content: null, action: "skipped", reason: verdict.reason };
787
+ }
788
+ return { ...base, content: norm.content, action: "created", reason: null };
789
+ }
790
+
791
+ /** `loadMetaPolicy` that never throws — a missing Meta page yields an empty
792
+ * policy block so a fixture/smoke vault without seeded Meta still drafts. */
793
+ function loadMetaPolicyQuiet(names: string[]): string {
794
+ try {
795
+ return loadMetaPolicy(names);
796
+ } catch {
797
+ return "";
798
+ }
799
+ }
800
+
801
+ // ---- effectful entry: CREATE into the vault + ledger -----------------------
13
802
 
14
803
  export interface SynthesizeResult {
15
- articlesCreated: number;
16
- articlesUpdated: number;
17
- skippedByGate: number;
804
+ created: boolean;
805
+ stem: string;
806
+ skippedByGate: boolean;
807
+ reason: string | null;
808
+ hasInfobox: boolean;
809
+ citedSections: number;
810
+ sectionAnchors: string[];
811
+ }
812
+
813
+ export interface SynthesizeCreateOpts {
814
+ entity: string;
815
+ kind?: EntityKind;
816
+ /** The entity's accumulated chunk ids (the CREATE rule already fired). */
817
+ chunkIds: string[];
818
+ /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
819
+ root?: string;
820
+ /** Declared aliases for the infobox. */
821
+ aliases?: string[];
822
+ /** Model-call override (tests inject a fake). */
823
+ call?: SynthesisCall;
824
+ /** Skip the git commit (tests). */
825
+ commit?: boolean;
826
+ now?: number;
18
827
  }
19
828
 
20
- /** Bucket → article create/update + safety gate (local model). STUB. */
21
- export async function synthesizeBuckets(
22
- _model: unknown,
23
- _onEvent?: (e: SynthesisEvent) => void,
829
+ /** Read the chunk + its conversation metadata into a {@link SynthesisChunk}. */
830
+ export function loadSynthesisChunk(store: MemoryStore, chunkId: string): SynthesisChunk | null {
831
+ const row = store.db
832
+ .query(
833
+ "SELECT c.id, c.conv, c.label, conv.title, conv.updated_at FROM chunks c " +
834
+ "JOIN conversations conv ON conv.conv = c.conv WHERE c.id = ?",
835
+ )
836
+ .get(chunkId) as
837
+ | { id: string; conv: string; label: string | null; title: string | null; updated_at: number | null }
838
+ | null;
839
+ if (!row) return null;
840
+ return {
841
+ id: row.id,
842
+ conv: row.conv,
843
+ label: row.label,
844
+ text: store.chunkText(chunkId),
845
+ title: row.title ?? "",
846
+ dateMs: row.updated_at,
847
+ };
848
+ }
849
+
850
+ /**
851
+ * CREATE an entity article from its chunks and persist it: draft via
852
+ * {@link createArticle}, write the markdown to the vault (unless the gate vetoed
853
+ * it), commit via `commitVault`, and record the integrated (chunk, section)
854
+ * edges in `synthesized_chunk_sections`. The CREATE decision itself is upstream
855
+ * (ROUTE's B8/P1-T5 gate) — this assumes the entity earned its article.
856
+ */
857
+ export async function synthesizeCreate(
858
+ store: MemoryStore,
859
+ opts: SynthesizeCreateOpts,
24
860
  ): Promise<SynthesizeResult> {
25
- // TODO(M1): per touched bucket, feed unsynthesized chunks + current article
26
- // through the synthesize prompt; apply the conservative gate; write the
27
- // article; record (bucket, chunk_id) in synthesized_bucket_chunks.
28
- throw new Error("memory.synthesize is not implemented yet (M1)");
861
+ const root = opts.root ?? vaultRoot();
862
+ const stem = entityStem(opts.entity);
863
+ const chunks: SynthesisChunk[] = [];
864
+ for (const id of opts.chunkIds) {
865
+ const c = loadSynthesisChunk(store, id);
866
+ if (c) chunks.push(c);
867
+ }
868
+
869
+ const stems = new Set(await listArticles(root));
870
+ const outcome = await createArticle(opts.entity, opts.kind ?? "thing", chunks, {
871
+ call: opts.call,
872
+ stems,
873
+ aliases: opts.aliases,
874
+ });
875
+
876
+ const result: SynthesizeResult = {
877
+ created: outcome.action === "created",
878
+ stem,
879
+ skippedByGate: outcome.action === "skipped",
880
+ reason: outcome.reason,
881
+ hasInfobox: outcome.hasInfobox,
882
+ citedSections: outcome.citedSections,
883
+ sectionAnchors: outcome.sectionAnchors,
884
+ };
885
+
886
+ if (outcome.action !== "created" || outcome.content == null) return result;
887
+
888
+ // Write the article, then record the integration ledger + commit.
889
+ const dir = articlesDir(root);
890
+ await mkdir(dir, { recursive: true });
891
+ const path = join(dir, `${stem}.md`);
892
+ // Never clobber an existing article from the CREATE branch — that's UPDATE's job.
893
+ if (await fileExists(path)) {
894
+ result.created = false;
895
+ result.skippedByGate = false;
896
+ result.reason = "article already exists (CREATE is for new entities)";
897
+ return result;
898
+ }
899
+ await writeFile(path, outcome.content);
900
+
901
+ const now = opts.now ?? Date.now();
902
+ const insSec = store.db.query(
903
+ "INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
904
+ );
905
+ const insSynth = store.db.query(
906
+ "INSERT OR REPLACE INTO synthesized_chunk_sections " +
907
+ "(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
908
+ );
909
+ const tx = store.db.transaction((rows: { chunkId: string; anchor: string }[]) => {
910
+ for (const r of rows) {
911
+ insSec.run(r.chunkId, stem, r.anchor);
912
+ insSynth.run(r.chunkId, stem, r.anchor, now);
913
+ }
914
+ });
915
+ tx(outcome.chunkSections);
916
+
917
+ if (opts.commit !== false) {
918
+ await commitVault(root, `memory: create ${stem} (${outcome.chunkSections.length} chunk edges)`);
919
+ }
920
+
921
+ return result;
922
+ }
923
+
924
+ async function fileExists(path: string): Promise<boolean> {
925
+ try {
926
+ await readFile(path);
927
+ return true;
928
+ } catch {
929
+ return false;
930
+ }
931
+ }
932
+
933
+ // ===========================================================================
934
+ // PATCH — the UPDATE branch (P7-T2): fold ONE chunk into ONE existing section.
935
+ //
936
+ // The steady-state counterpart to CREATE. Where CREATE drafts a whole article
937
+ // from an entity's cold-start chunks, PATCH is the same bounded section-draft op
938
+ // run against an article that already exists: integrate THIS chunk into THIS
939
+ // (SECTION-ROUTE-chosen) section, citing it, changing nothing else. It is the
940
+ // self-healing mechanism — a user's in-conversation CORRECTION is just an
941
+ // ordinary chunk, so folding it rewrites the contradicted claim TOWARD the
942
+ // user's latest position (resolve-to-latest, not append-both, not ignore).
943
+ //
944
+ // Citations stay deterministic by construction exactly as in CREATE: the chunk's
945
+ // source conversation is assigned a stable `[^N]` (a re-cited conversation
946
+ // reuses its existing marker; a new one mints `maxN+1`) and we GENERATE the
947
+ // `[^N]:` definition line — the model only PLACES the marker. We swap only the
948
+ // target section's bytes via `extractSection`/`replaceSection` (every other
949
+ // section stays byte-identical) and append the def to `## References`, then run
950
+ // the SAME gate (weak → NO-OP, the vault is untouched and the chunk retries next
951
+ // run) + NORMALIZE as CREATE. Idempotency rides `synthesized_chunk_sections`'s PK
952
+ // — a second fold of the same (chunk, article, section) writes nothing.
953
+ // ===========================================================================
954
+
955
+ /** The article's H1 title (depth-1 TOC heading), or a neutral fallback. */
956
+ function articleH1Title(article: string): string {
957
+ const h1 = parseToc(article).find((e) => e.depth === 1);
958
+ return h1 ? h1.title : "this article";
959
+ }
960
+
961
+ /**
962
+ * The article's current footnote state: the highest `[^N]` number in use
963
+ * (markers ∪ definitions) and a map from each DEFINED `conv:HASH` to its number.
964
+ * PATCH assigns a re-cited conversation its existing marker (no duplicate def)
965
+ * and a brand-new one `maxN + 1`, so the article stays footnote-bijective and
966
+ * NORMALIZE's renumber short-circuits (no shuffle of other sections). Pure.
967
+ */
968
+ export function articleFootnoteState(article: string): { maxN: number; byHash: Map<string, number> } {
969
+ let maxN = 0;
970
+ const byHash = new Map<string, number>();
971
+ for (const line of article.split("\n")) {
972
+ const d = /^\[\^(\d+)\]:(.*)$/.exec(line);
973
+ if (d) {
974
+ const n = parseInt(d[1]!, 10);
975
+ if (n > maxN) maxN = n;
976
+ const h = d[2]!.match(/conv:([0-9a-z]{8})/i);
977
+ if (h) byHash.set(h[1]!.toLowerCase(), n);
978
+ continue;
979
+ }
980
+ for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
981
+ const n = parseInt(m[1]!, 10);
982
+ if (n > maxN) maxN = n;
983
+ }
984
+ }
985
+ return { maxN, byHash };
986
+ }
987
+
988
+ /**
989
+ * Map each `[^N]:` definition's marker number to the `YYYY-MM-DD` (or `undated`)
990
+ * date its generated def line carries — the dates we tagged via `footnoteDefLine`
991
+ * (`[^n]: \`conv:HASH\` (DATE, source)`). PATCH reads this so it can tell the model
992
+ * how old the claim it is correcting is, and resolve toward the LATER date. Pure.
993
+ */
994
+ export function referenceDatesByMarker(article: string): Map<number, string> {
995
+ const out = new Map<number, string>();
996
+ for (const m of article.matchAll(/^\[\^(\d+)\]:[^\n]*?\((\d{4}-\d{2}-\d{2}|undated),/gm)) {
997
+ out.set(parseInt(m[1]!, 10), m[2]!);
998
+ }
999
+ return out;
1000
+ }
1001
+
1002
+ /** The distinct `[^N]` marker numbers present in a single section/body block
1003
+ * (definition lines excluded), in first-appearance order. Pure. */
1004
+ function markersInBlock(body: string): number[] {
1005
+ const order: number[] = [];
1006
+ const seen = new Set<number>();
1007
+ for (const line of body.split("\n")) {
1008
+ if (/^\[\^\d+\]:/.test(line.trim())) continue;
1009
+ for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
1010
+ const n = parseInt(m[1]!, 10);
1011
+ if (!seen.has(n)) {
1012
+ seen.add(n);
1013
+ order.push(n);
1014
+ }
1015
+ }
1016
+ }
1017
+ return order;
1018
+ }
1019
+
1020
+ /**
1021
+ * Swap exactly the `anchor` section's block for `newBlock`, leaving every byte
1022
+ * OUTSIDE that section identical (the byte-preservation invariant PATCH and the
1023
+ * gate depend on). Returns null when the section is absent. Pure.
1024
+ */
1025
+ export function replaceSection(article: string, anchor: string, newBlock: string): string | null {
1026
+ const block = extractSection(article, anchor);
1027
+ if (block == null) return null;
1028
+ const idx = article.indexOf(block);
1029
+ if (idx < 0) return null;
1030
+ return article.slice(0, idx) + newBlock + article.slice(idx + block.length);
1031
+ }
1032
+
1033
+ /**
1034
+ * Insert a generated `[^N]:` definition line into the article's `## References`
1035
+ * section (after the last existing def), creating the section at the end when
1036
+ * the article has none. The model never writes defs — this is the only path that
1037
+ * does. Pure.
1038
+ */
1039
+ export function appendReferenceDef(article: string, defLine: string): string {
1040
+ const refs = extractSection(article, "references");
1041
+ if (refs == null) {
1042
+ return `${article.replace(/\s+$/, "")}\n\n## References\n\n${defLine}\n`;
1043
+ }
1044
+ const newRefs = `${refs.replace(/\s+$/, "")}\n${defLine}\n`;
1045
+ return replaceSection(article, "references", newRefs) ?? article;
1046
+ }
1047
+
1048
+ /** Retry hint when a patched section leaks meta/planning or fails to cite. */
1049
+ const PATCH_ANTI_LEAK_HINT =
1050
+ "Output ONLY the integrated section prose — no preamble, no meta-commentary, no plan, no restating the task.";
1051
+
1052
+ /**
1053
+ * Draft the integrated section body, rejecting leakage AND an uncited fold:
1054
+ * draft once; if the sanitized body is {@link isLeakyDraft} OR is missing the
1055
+ * assigned `[^marker]` (the integration didn't actually place the citation),
1056
+ * retry ONCE with a sharper instruction; if it still fails, return "" so the
1057
+ * caller NO-OPs (the vault is untouched; the chunk retries next run).
1058
+ */
1059
+ async function draftPatchedProse(
1060
+ call: SynthesisCall,
1061
+ prompt: string,
1062
+ marker: number,
1063
+ sanitize: (raw: string) => string,
1064
+ ): Promise<string> {
1065
+ const wants = new RegExp(`\\[\\^${marker}\\](?!:)`);
1066
+ const clean = (raw: string): string => {
1067
+ const body = sanitize(raw);
1068
+ return !isLeakyDraft(body) && wants.test(body) ? body : "";
1069
+ };
1070
+ const first = clean(await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS }));
1071
+ if (first) return first;
1072
+ return clean(await call(`${prompt}\n\n${PATCH_ANTI_LEAK_HINT} Place the [^${marker}] marker for the new note.`, { maxTokens: MAX_OUTPUT_TOKENS }));
1073
+ }
1074
+
1075
+ /** Draft an integrated SECTION body (sanitized as a section, must place the marker). */
1076
+ function draftPatchedSection(
1077
+ call: SynthesisCall,
1078
+ prompt: string,
1079
+ marker: number,
1080
+ heading: string,
1081
+ ): Promise<string> {
1082
+ return draftPatchedProse(call, prompt, marker, (raw) => sanitizeSection(raw, heading));
1083
+ }
1084
+
1085
+ /** The schematic PATCH prompt: current section markdown + the chunk + the inlined
1086
+ * footnote/correction contract. Kept free of copyable example values per the
1087
+ * small-model parroting failure mode. */
1088
+ export function buildPatchPrompt(
1089
+ title: string,
1090
+ heading: string,
1091
+ currentBody: string,
1092
+ chunk: SynthesisChunk,
1093
+ n: number,
1094
+ otherArticles: string,
1095
+ policy: string,
1096
+ priorDates: string[] = [],
1097
+ ): string {
1098
+ const chunkDate = footnoteDate(chunk.dateMs);
1099
+ return (
1100
+ `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
1101
+ `You are integrating ONE new note (dated ${chunkDate}) into the "${heading}" section of the personal-wiki article about "${title}". ` +
1102
+ `Keep the existing prose and every existing [^k] citation marker intact; weave in only what the new note adds.\n\n` +
1103
+ `CITATION — cite the new note by placing the marker [^${n}] immediately after the specific claim it supports. ` +
1104
+ `Use [^${n}] for the new note, and reuse any [^k] markers already in the section exactly where they are. ` +
1105
+ `Do NOT write any [^k]: definition line — definitions are generated for you. ` +
1106
+ `[^N] is the only citation syntax; never emit HTML, bare links, or invent another number.\n\n` +
1107
+ correctionRule(n, chunkDate, priorDates) +
1108
+ `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
1109
+ `CURRENT "${heading}" SECTION:\n${currentBody || "(empty)"}\n\n` +
1110
+ `NEW NOTE (${chunkDate})${chunk.label ? ` — ${chunk.label}` : ""}:\n${chunk.text.slice(0, 1400).trim()}\n\n` +
1111
+ `Output ONLY the rewritten section body prose — do NOT repeat the "## ${heading}" heading, do NOT add other sections, do NOT write a References section.`
1112
+ );
1113
+ }
1114
+
1115
+ /**
1116
+ * The PROVENANCE-PRESERVING correction rule (schematic). A correction resolves
1117
+ * toward the LATER-DATED statement so the latest position wins — but it KEEPS the
1118
+ * superseded claim's existing `[^k]` marker alongside the new `[^N]`, phrased as a
1119
+ * change over time, so no citation is lost. This is what lets the resolve fold pass
1120
+ * the gate's citation-survival check instead of being NO-OP'd for dropping `[^k]`.
1121
+ */
1122
+ function correctionRule(n: number, chunkDate: string, priorDates: string[]): string {
1123
+ const prior = priorDates.filter((d) => d && d !== "undated");
1124
+ const priorClause = prior.length
1125
+ ? `The claim it corrects is dated ${[...new Set(prior)].join(" / ")}, so the new note (dated ${chunkDate}) is LATER. `
1126
+ : `The new note is dated ${chunkDate}. `;
1127
+ return (
1128
+ `CORRECTION — if the new note CONTRADICTS a claim already in the section, this is a correction: resolve toward the LATER-DATED statement so the user's CURRENT position wins. ` +
1129
+ priorClause +
1130
+ `DEMOTE the superseded claim to the PAST TENSE — rewrite the OLD clause itself from present ("is my favourite", "currently", "the one I reach for") to past ("previously was my favourite", "originally", "used to be") — do NOT leave the old value asserted as still-current and merely append the new one. ` +
1131
+ `KEEP the superseded claim's existing [^k] citation marker in place ALONGSIDE the new note's [^${n}] marker, so no [^N] is lost — phrase it like "previously <earlier position>[^k]; as of <later date> <current position> is now my pick[^${n}]". ` +
1132
+ `Do NOT assert both as simultaneously true, and do NOT ignore the new note.\n\n`
1133
+ );
1134
+ }
1135
+
1136
+ /**
1137
+ * The CLASSIFIED LEAD correction rule (schematic). The lead must read as a CLEAN
1138
+ * current-state summary — never scattered "previously …" caveats (RULE 1). So when
1139
+ * a contradiction is classified:
1140
+ * - EVOLUTION (the user changed their own position) → resolve the lead to the
1141
+ * CURRENT value in the present tense; KEEP the superseded claim's `[^k]` marker
1142
+ * so provenance survives the gate (the full trajectory is consolidated into a
1143
+ * `## History` section by the reconcile pass), but do NOT narrate the change in
1144
+ * the lead itself.
1145
+ * - ERROR (a wrong fact being corrected) → SILENTLY replace the wrong value with
1146
+ * the correct one; the wrong value's `[^k]` citation MAY be dropped (it is
1147
+ * pruned as if it had never been there).
1148
+ */
1149
+ function leadCorrectionRule(kind: ContradictionKind, n: number, chunkDate: string): string {
1150
+ if (kind === "error") {
1151
+ return (
1152
+ `CORRECTION (factual error) — the new note (dated ${chunkDate}) corrects a WRONG fact already in the lead. ` +
1153
+ `Silently REPLACE the wrong value with the correct one and cite it [^${n}]; state ONLY the correct value. ` +
1154
+ `Do NOT mention the wrong value, do NOT write "previously" — it was never a real position. ` +
1155
+ `You MAY drop the wrong value's [^k] citation marker.\n\n`
1156
+ );
1157
+ }
1158
+ return (
1159
+ `CORRECTION (the user's view evolved) — the new note (dated ${chunkDate}) supersedes an earlier position in the lead. ` +
1160
+ `Resolve the lead to the user's CURRENT position in the PRESENT TENSE and cite it [^${n}]. ` +
1161
+ `Keep it a clean current-state summary: do NOT scatter "previously …" caveats through the lead — the trajectory is recorded separately in a History section. ` +
1162
+ `KEEP the superseded claim's existing [^k] citation marker somewhere in the lead so no [^N] is lost (provenance survives), but frame the lead around the current value, not the change.\n\n`
1163
+ );
1164
+ }
1165
+
1166
+ /** The schematic LEAD-PATCH prompt: the article's current lead + the chunk + the
1167
+ * inlined footnote/correction contract. The lead is where an entity's headline
1168
+ * verdict lives, so the CORRECTION rule is the load-bearing one. When a `kind` is
1169
+ * classified, the lead resolves to a CLEAN current-state summary (evolution) or a
1170
+ * silent overwrite (error); otherwise the default provenance-preserving fold runs. */
1171
+ export function buildLeadPatchPrompt(
1172
+ title: string,
1173
+ currentLead: string,
1174
+ chunk: SynthesisChunk,
1175
+ n: number,
1176
+ otherArticles: string,
1177
+ policy: string,
1178
+ priorDates: string[] = [],
1179
+ kind?: ContradictionKind,
1180
+ ): string {
1181
+ const chunkDate = footnoteDate(chunk.dateMs);
1182
+ return (
1183
+ `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
1184
+ `You are updating the LEAD (the opening summary) of the personal-wiki article about "${title}" with ONE new note (dated ${chunkDate}). ` +
1185
+ `The lead names **${title}** in bold and states what it is and how the user relates to it. ` +
1186
+ `Keep every existing [^k] citation marker that still applies; weave in only what the new note adds.\n\n` +
1187
+ `CITATION — cite the new note by placing the marker [^${n}] immediately after the specific claim it supports. ` +
1188
+ `Reuse any [^k] markers already in the lead exactly where they are. ` +
1189
+ `Do NOT write any [^k]: definition line — definitions are generated for you. ` +
1190
+ `[^N] is the only citation syntax; never emit HTML, bare links, or invent another number.\n\n` +
1191
+ (kind ? leadCorrectionRule(kind, n, chunkDate) : correctionRule(n, chunkDate, priorDates)) +
1192
+ `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
1193
+ `CURRENT LEAD:\n${currentLead || "(empty)"}\n\n` +
1194
+ `NEW NOTE (${chunkDate})${chunk.label ? ` — ${chunk.label}` : ""}:\n${chunk.text.slice(0, 1400).trim()}\n\n` +
1195
+ `Output ONLY the rewritten lead prose — 2–4 sentences, no heading, no list, no References section.`
1196
+ );
1197
+ }
1198
+
1199
+ export interface PatchInput {
1200
+ /** The full current article markdown. */
1201
+ article: string;
1202
+ /** The target section's anchor (a `chunk_sections` row's section_anchor), or
1203
+ * {@link LEAD_ANCHOR} to fold into the article's opening lead. */
1204
+ anchor: string;
1205
+ /** The chunk to integrate (text + footnote metadata). */
1206
+ chunk: SynthesisChunk;
1207
+ }
1208
+
1209
+ export interface PatchDeps {
1210
+ /** Model call; defaults to `callLocal("synthesis", …)` (same stage as CREATE). */
1211
+ call?: SynthesisCall;
1212
+ /** Other article stems for wikilink resolution. */
1213
+ stems?: Set<string>;
1214
+ /** Article stem (H1/NORMALIZE); defaults to the entity stem of the H1 title. */
1215
+ stem?: string;
1216
+ }
1217
+
1218
+ export interface PatchOutcome {
1219
+ anchor: string;
1220
+ /** Final post-NORMALIZE article, or null when the patch was a NO-OP. */
1221
+ content: string | null;
1222
+ action: "patched" | "skipped";
1223
+ /** Skip reason; null on success. */
1224
+ reason: string | null;
1225
+ /** The `[^N]` number assigned to the chunk (null when never reached). */
1226
+ footnote: number | null;
1227
+ /** True when the chunk's conversation was already cited (reused marker, no new def). */
1228
+ reusedFootnote: boolean;
1229
+ /** True when a new `[^N]:` definition line was appended. */
1230
+ addedDef: boolean;
1231
+ normalizeNotes: string[];
1232
+ }
1233
+
1234
+ /**
1235
+ * Fold ONE chunk into ONE existing section of an article, end to end:
1236
+ * assign the chunk's `[^N]` → draft the integrated body (leak/uncited → retry →
1237
+ * NO-OP) → swap ONLY that section's bytes → append the generated `[^N]:` def →
1238
+ * NORMALIZE → gate. Returns the patched article (or null when the gate vetoes a
1239
+ * weak pass). Pure with respect to disk — the caller persists.
1240
+ */
1241
+ export async function patchSection(input: PatchInput, deps: PatchDeps = {}): Promise<PatchOutcome> {
1242
+ const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
1243
+ const { article, anchor, chunk } = input;
1244
+ // The LEAD is patched by swapping exactly the lead bytes (no `## heading`).
1245
+ if (anchor === LEAD_ANCHOR) return patchLead(article, chunk, deps, call);
1246
+ const skip = (reason: string, footnote: number | null = null, reusedFootnote = false, addedDef = false): PatchOutcome => ({
1247
+ anchor,
1248
+ content: null,
1249
+ action: "skipped",
1250
+ reason,
1251
+ footnote,
1252
+ reusedFootnote,
1253
+ addedDef,
1254
+ normalizeNotes: [],
1255
+ });
1256
+
1257
+ const block = extractSection(article, anchor);
1258
+ if (block == null) return skip(`section not found: ${anchor}`);
1259
+ const headingLine = block.split("\n")[0] ?? `## ${anchor}`;
1260
+ const heading = headingLine.replace(/^#{1,6}\s+/, "").trim();
1261
+ const currentBody = block.split("\n").slice(1).join("\n").trim();
1262
+
1263
+ const title = articleH1Title(article);
1264
+ const stem = deps.stem ?? entityStem(title);
1265
+ const stems = new Set(deps.stems ?? []);
1266
+ stems.add(stem);
1267
+ const otherArticles = otherArticlesBlock(stems, stem);
1268
+ const policy = loadMetaPolicyQuiet(["Article_Conventions"]);
1269
+
1270
+ // Assign the chunk's footnote: reuse the conversation's existing marker, else
1271
+ // mint maxN+1 (keeps the article bijective+contiguous → NORMALIZE no-ops).
1272
+ const state = articleFootnoteState(article);
1273
+ const hash = convHash(chunk.conv);
1274
+ const reused = state.byHash.get(hash);
1275
+ const n = reused ?? state.maxN + 1;
1276
+ const addedDef = reused === undefined;
1277
+
1278
+ // The dates of the claims this section already cites — so a contradicting fold
1279
+ // can be told it is the LATER statement and resolve toward itself (provenance
1280
+ // preserved: it keeps the old [^k] marker).
1281
+ const refDates = referenceDatesByMarker(article);
1282
+ const priorDates = markersInBlock(currentBody)
1283
+ .map((m) => refDates.get(m))
1284
+ .filter((d): d is string => d != null);
1285
+
1286
+ const newBody = await draftPatchedSection(
1287
+ call,
1288
+ buildPatchPrompt(title, heading, currentBody, chunk, n, otherArticles, policy, priorDates),
1289
+ n,
1290
+ heading,
1291
+ );
1292
+ // Empty after one retry = leaky/uncited → NO-OP (vault untouched, chunk retries).
1293
+ if (!newBody.trim()) return skip("patched section leaked, refused, or did not cite the new note (NO-OP)", n, reused !== undefined, addedDef);
1294
+
1295
+ let next = replaceSection(article, anchor, `${headingLine}\n\n${newBody}\n`);
1296
+ if (next == null) return skip(`section vanished on swap: ${anchor}`, n, reused !== undefined, addedDef);
1297
+ if (addedDef) next = appendReferenceDef(next, footnoteDefLine(n, chunk.conv, chunk.title, chunk.dateMs));
1298
+
1299
+ const norm = normalizeArticle(next, { stem, stems });
1300
+
1301
+ // Footnote bijection must hold (a hallucinated marker number the gate cannot
1302
+ // repair → NO-OP rather than a citation-broken commit).
1303
+ const fn = checkFootnoteIntegrity(norm.content);
1304
+ if (!fn.ok) return skip(`footnote bijection unrepairable: ${fn.errors.join("; ")}`, n, reused !== undefined, addedDef);
1305
+
1306
+ // The conservative gate over (before = original article, after = patched).
1307
+ const verdict: GateVerdict = gateEdit(article, norm.content);
1308
+ if (!verdict.ok) return skip(verdict.reason ?? "gate veto", n, reused !== undefined, addedDef);
1309
+
1310
+ return {
1311
+ anchor,
1312
+ content: norm.content,
1313
+ action: "patched",
1314
+ reason: null,
1315
+ footnote: n,
1316
+ reusedFootnote: reused !== undefined,
1317
+ addedDef,
1318
+ normalizeNotes: norm.notes,
1319
+ };
1320
+ }
1321
+
1322
+ /**
1323
+ * Fold ONE chunk into the article's LEAD (its opening summary), swapping exactly
1324
+ * the lead bytes — every section and the References stay byte-identical except for
1325
+ * the appended def. Same assign-footnote → draft (leak/uncited → retry → NO-OP) →
1326
+ * NORMALIZE → gate contract as {@link patchSection}, but the lead has no `##`
1327
+ * heading so we locate it via the structural skeleton and replace those lines.
1328
+ */
1329
+ async function patchLead(
1330
+ article: string,
1331
+ chunk: SynthesisChunk,
1332
+ deps: PatchDeps,
1333
+ call: SynthesisCall,
1334
+ ): Promise<PatchOutcome> {
1335
+ const skip = (reason: string, footnote: number | null = null, reusedFootnote = false, addedDef = false): PatchOutcome => ({
1336
+ anchor: LEAD_ANCHOR,
1337
+ content: null,
1338
+ action: "skipped",
1339
+ reason,
1340
+ footnote,
1341
+ reusedFootnote,
1342
+ addedDef,
1343
+ normalizeNotes: [],
1344
+ });
1345
+
1346
+ const leadItem = articleStructure(article).find((it) => it.kind === "lead");
1347
+ if (!leadItem) return skip("article has no lead to patch");
1348
+ const lines = article.split("\n");
1349
+ const start = leadItem.startLine - 1;
1350
+ const end = leadItem.endLine - 1; // 1-based inclusive → 0-based inclusive
1351
+ const currentLead = lines.slice(start, end + 1).join("\n").trim();
1352
+
1353
+ const title = articleH1Title(article);
1354
+ const stem = deps.stem ?? entityStem(title);
1355
+ const stems = new Set(deps.stems ?? []);
1356
+ stems.add(stem);
1357
+ const otherArticles = otherArticlesBlock(stems, stem);
1358
+ const policy = loadMetaPolicyQuiet(["Article_Conventions"]);
1359
+
1360
+ // Assign the chunk's footnote exactly as the section path does.
1361
+ const state = articleFootnoteState(article);
1362
+ const hash = convHash(chunk.conv);
1363
+ const reused = state.byHash.get(hash);
1364
+ const n = reused ?? state.maxN + 1;
1365
+ const addedDef = reused === undefined;
1366
+
1367
+ const refDates = referenceDatesByMarker(article);
1368
+ const priorMarkers = markersInBlock(currentLead);
1369
+ const priorDates = priorMarkers
1370
+ .map((m) => refDates.get(m))
1371
+ .filter((d): d is string => d != null);
1372
+
1373
+ // CLASSIFY the contradiction when the lead already cites a prior claim the new
1374
+ // note may conflict with: is the user CHANGING THEIR OWN view (evolution — the
1375
+ // lead resolves to current-state, the trajectory preserved in History by the
1376
+ // reconcile pass) or correcting a WRONG fact (error — silent overwrite, the
1377
+ // wrong value's citation may be pruned)?
1378
+ let kind: ContradictionKind | undefined;
1379
+ const allowDroppedHashes = new Set<string>();
1380
+ if (priorMarkers.length) {
1381
+ kind = classifyContradiction(
1382
+ await call(buildContradictionClassifierPrompt(currentLead, chunk.text.slice(0, 1400).trim()), { maxTokens: MAX_OUTPUT_TOKENS }),
1383
+ );
1384
+ if (kind === "error") {
1385
+ // A factual fix may drop the wrong value's citation. Permit dropping the
1386
+ // hashes the corrected lead cited (bounded to the lead's own prior claims).
1387
+ const hashByMarker = new Map<number, string>();
1388
+ for (const [h, m] of state.byHash) hashByMarker.set(m, h);
1389
+ for (const m of priorMarkers) {
1390
+ const h = hashByMarker.get(m);
1391
+ if (h) allowDroppedHashes.add(h);
1392
+ }
1393
+ }
1394
+ }
1395
+
1396
+ const newLead = await draftPatchedProse(
1397
+ call,
1398
+ buildLeadPatchPrompt(title, currentLead, chunk, n, otherArticles, policy, priorDates, kind),
1399
+ n,
1400
+ sanitizeLead,
1401
+ );
1402
+ if (!newLead.trim()) return skip("patched lead leaked, refused, or did not cite the new note (NO-OP)", n, reused !== undefined, addedDef);
1403
+
1404
+ // Swap exactly the lead lines; every other byte is preserved.
1405
+ let next = [...lines.slice(0, start), newLead, ...lines.slice(end + 1)].join("\n");
1406
+ if (addedDef) next = appendReferenceDef(next, footnoteDefLine(n, chunk.conv, chunk.title, chunk.dateMs));
1407
+
1408
+ const norm = normalizeArticle(next, { stem, stems });
1409
+
1410
+ const fn = checkFootnoteIntegrity(norm.content);
1411
+ if (!fn.ok) return skip(`footnote bijection unrepairable: ${fn.errors.join("; ")}`, n, reused !== undefined, addedDef);
1412
+
1413
+ const verdict: GateVerdict = gateEdit(article, norm.content, { allowDroppedHashes });
1414
+ if (!verdict.ok) return skip(verdict.reason ?? "gate veto", n, reused !== undefined, addedDef);
1415
+
1416
+ return {
1417
+ anchor: LEAD_ANCHOR,
1418
+ content: norm.content,
1419
+ action: "patched",
1420
+ reason: null,
1421
+ footnote: n,
1422
+ reusedFootnote: reused !== undefined,
1423
+ addedDef,
1424
+ normalizeNotes: norm.notes,
1425
+ };
1426
+ }
1427
+
1428
+ // ---- effectful entry: PATCH a vault article + ledger -----------------------
1429
+
1430
+ export interface SynthesizePatchOpts {
1431
+ /** The article to patch (its filename stem). */
1432
+ stem: string;
1433
+ /** The section anchor to fold into (a `chunk_sections` row). */
1434
+ anchor: string;
1435
+ /** The chunk id to integrate. */
1436
+ chunkId: string;
1437
+ /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
1438
+ root?: string;
1439
+ /** Model-call override (tests inject a fake). */
1440
+ call?: SynthesisCall;
1441
+ /** Skip the git commit (tests). */
1442
+ commit?: boolean;
1443
+ now?: number;
1444
+ }
1445
+
1446
+ export interface PatchResult {
1447
+ patched: boolean;
1448
+ stem: string;
1449
+ anchor: string;
1450
+ /** The gate (or a draft NO-OP) vetoed the write. */
1451
+ skippedByGate: boolean;
1452
+ /** The (chunk, article, section) edge was already folded (idempotent NO-OP). */
1453
+ alreadyIntegrated: boolean;
1454
+ reason: string | null;
1455
+ footnote: number | null;
1456
+ }
1457
+
1458
+ /**
1459
+ * PATCH one chunk into one section of a vault article and persist it. Idempotent
1460
+ * BEFORE the model: a present `synthesized_chunk_sections` PK row short-circuits
1461
+ * to a NO-OP (no GPU, no write). Otherwise drafts via {@link patchSection}, writes
1462
+ * the swapped article (unless the gate vetoed), records the integration edge, and
1463
+ * commits.
1464
+ */
1465
+ export async function synthesizePatch(store: MemoryStore, opts: SynthesizePatchOpts): Promise<PatchResult> {
1466
+ const root = opts.root ?? vaultRoot();
1467
+ const base: PatchResult = {
1468
+ patched: false,
1469
+ stem: opts.stem,
1470
+ anchor: opts.anchor,
1471
+ skippedByGate: false,
1472
+ alreadyIntegrated: false,
1473
+ reason: null,
1474
+ footnote: null,
1475
+ };
1476
+
1477
+ // Idempotency: the (chunk, article, section) PK blocks a double-fold up front.
1478
+ const already = store.db
1479
+ .query("SELECT 1 FROM synthesized_chunk_sections WHERE chunk_id = ? AND article_stem = ? AND section_anchor = ?")
1480
+ .get(opts.chunkId, opts.stem, opts.anchor);
1481
+ if (already) return { ...base, alreadyIntegrated: true, reason: "already integrated (synthesized_chunk_sections PK)" };
1482
+
1483
+ const path = join(articlesDir(root), `${opts.stem}.md`);
1484
+ let article: string;
1485
+ try {
1486
+ article = await readFile(path, "utf8");
1487
+ } catch {
1488
+ return { ...base, reason: `article not found: ${opts.stem}` };
1489
+ }
1490
+
1491
+ const chunk = loadSynthesisChunk(store, opts.chunkId);
1492
+ if (!chunk) return { ...base, reason: `chunk not found: ${opts.chunkId}` };
1493
+
1494
+ const stems = new Set(await listArticles(root));
1495
+ const outcome = await patchSection({ article, anchor: opts.anchor, chunk }, { call: opts.call, stems, stem: opts.stem });
1496
+
1497
+ if (outcome.action !== "patched" || outcome.content == null) {
1498
+ return { ...base, skippedByGate: true, reason: outcome.reason, footnote: outcome.footnote };
1499
+ }
1500
+
1501
+ await writeFile(path, outcome.content);
1502
+
1503
+ const now = opts.now ?? Date.now();
1504
+ const insSec = store.db.query(
1505
+ "INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
1506
+ );
1507
+ const insSynth = store.db.query(
1508
+ "INSERT OR REPLACE INTO synthesized_chunk_sections " +
1509
+ "(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
1510
+ );
1511
+ const tx = store.db.transaction(() => {
1512
+ insSec.run(opts.chunkId, opts.stem, opts.anchor);
1513
+ insSynth.run(opts.chunkId, opts.stem, opts.anchor, now);
1514
+ });
1515
+ tx();
1516
+
1517
+ if (opts.commit !== false) {
1518
+ await commitVault(root, `memory: patch ${opts.stem} #${opts.anchor} (+chunk ${opts.chunkId})`);
1519
+ }
1520
+
1521
+ return { ...base, patched: true, footnote: outcome.footnote };
1522
+ }
1523
+
1524
+ // ===========================================================================
1525
+ // NEW SECTION — the third SECTION-ROUTE outcome (P7): a chunk that fits NO
1526
+ // existing section AND is not a verdict-level correction earns its OWN section.
1527
+ //
1528
+ // SECTION-ROUTE returns a NAMED new section for a substantive chunk that matched
1529
+ // nothing; honoring it (rather than dropping the chunk) is the difference between
1530
+ // self-healing and silent loss. We mint an EMPTY `## Title` section, then run the
1531
+ // SAME bounded fold as PATCH against it — so the new section is drafted, cited,
1532
+ // gated, and normalized exactly like every other integration, never a raw dump.
1533
+ // ===========================================================================
1534
+
1535
+ /** Insert an empty `## title` section into the article just before its `## See
1536
+ * also` / `## References` tail (or at the end when neither exists), so a fresh
1537
+ * topical section lands among the body sections, not after the references. The
1538
+ * body is left empty — {@link patchSection} fills it by folding the chunk in. */
1539
+ export function insertEmptySection(article: string, title: string): string {
1540
+ const lines = article.replace(/\s+$/, "").split("\n");
1541
+ const idx = lines.findIndex((l) => /^##\s+(see also|references)\s*$/i.test(l.trim()));
1542
+ if (idx < 0) return [...lines, "", `## ${title}`, ""].join("\n") + "\n";
1543
+ return [...lines.slice(0, idx), `## ${title}`, "", "", ...lines.slice(idx)].join("\n") + "\n";
1544
+ }
1545
+
1546
+ export interface SynthesizeNewSectionOpts {
1547
+ /** The article to grow (its filename stem). */
1548
+ stem: string;
1549
+ /** The new section's display title. */
1550
+ title: string;
1551
+ /** The new section's anchor (slug of `title`); also the ledger key. */
1552
+ anchor: string;
1553
+ /** The chunk id to fold into the new section. */
1554
+ chunkId: string;
1555
+ /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
1556
+ root?: string;
1557
+ /** Model-call override (tests inject a fake). */
1558
+ call?: SynthesisCall;
1559
+ /** Skip the git commit (tests). */
1560
+ commit?: boolean;
1561
+ now?: number;
1562
+ }
1563
+
1564
+ /**
1565
+ * CREATE a new section in an existing article and fold ONE chunk into it. Mints
1566
+ * the `## Title` heading then drafts its body via {@link patchSection} (so the
1567
+ * fold is cited, gated, NORMALIZED, and footnote-bijective just like a PATCH),
1568
+ * writes the grown article, and records the (chunk, section) edge. Idempotent via
1569
+ * `synthesized_chunk_sections`'s PK — a re-run of the same (chunk, article,
1570
+ * section) writes nothing. Used by SYNTHESIZE when SECTION-ROUTE returns a
1571
+ * `newSection` for a chunk that fit no existing section.
1572
+ */
1573
+ export async function synthesizeNewSection(
1574
+ store: MemoryStore,
1575
+ opts: SynthesizeNewSectionOpts,
1576
+ ): Promise<PatchResult> {
1577
+ const root = opts.root ?? vaultRoot();
1578
+ const base: PatchResult = {
1579
+ patched: false,
1580
+ stem: opts.stem,
1581
+ anchor: opts.anchor,
1582
+ skippedByGate: false,
1583
+ alreadyIntegrated: false,
1584
+ reason: null,
1585
+ footnote: null,
1586
+ };
1587
+
1588
+ const already = store.db
1589
+ .query("SELECT 1 FROM synthesized_chunk_sections WHERE chunk_id = ? AND article_stem = ? AND section_anchor = ?")
1590
+ .get(opts.chunkId, opts.stem, opts.anchor);
1591
+ if (already) return { ...base, alreadyIntegrated: true, reason: "already integrated (synthesized_chunk_sections PK)" };
1592
+
1593
+ const path = join(articlesDir(root), `${opts.stem}.md`);
1594
+ let article: string;
1595
+ try {
1596
+ article = await readFile(path, "utf8");
1597
+ } catch {
1598
+ return { ...base, reason: `article not found: ${opts.stem}` };
1599
+ }
1600
+
1601
+ const chunk = loadSynthesisChunk(store, opts.chunkId);
1602
+ if (!chunk) return { ...base, reason: `chunk not found: ${opts.chunkId}` };
1603
+
1604
+ // Mint the empty section unless the article already carries it (idempotent
1605
+ // across the same name), then fold the chunk into it via the PATCH op.
1606
+ const withSection = extractSection(article, opts.anchor) != null ? article : insertEmptySection(article, opts.title);
1607
+ const stems = new Set(await listArticles(root));
1608
+ const outcome = await patchSection(
1609
+ { article: withSection, anchor: opts.anchor, chunk },
1610
+ { call: opts.call, stems, stem: opts.stem },
1611
+ );
1612
+
1613
+ if (outcome.action !== "patched" || outcome.content == null) {
1614
+ return { ...base, skippedByGate: true, reason: outcome.reason, footnote: outcome.footnote };
1615
+ }
1616
+
1617
+ await writeFile(path, outcome.content);
1618
+
1619
+ const now = opts.now ?? Date.now();
1620
+ const insSec = store.db.query(
1621
+ "INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
1622
+ );
1623
+ const insSynth = store.db.query(
1624
+ "INSERT OR REPLACE INTO synthesized_chunk_sections " +
1625
+ "(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
1626
+ );
1627
+ const tx = store.db.transaction(() => {
1628
+ insSec.run(opts.chunkId, opts.stem, opts.anchor);
1629
+ insSynth.run(opts.chunkId, opts.stem, opts.anchor, now);
1630
+ });
1631
+ tx();
1632
+
1633
+ if (opts.commit !== false) {
1634
+ await commitVault(root, `memory: new section ${opts.stem} #${opts.anchor} (+chunk ${opts.chunkId})`);
1635
+ }
1636
+
1637
+ return { ...base, patched: true, footnote: outcome.footnote };
29
1638
  }