mlx-bun 0.4.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (601) hide show
  1. package/README.md +1453 -88
  2. package/THIRD_PARTY_NOTICES.md +882 -0
  3. package/bin/mlx-bun.convert.mjs +4 -0
  4. package/bin/mlx-bun.fuse.mjs +4 -0
  5. package/bin/mlx-bun.generate.mjs +4 -0
  6. package/bin/mlx-bun.lora.mjs +4 -0
  7. package/bin/mlx-bun.mjs +13 -21
  8. package/bin/mlx-bun.server.mjs +4 -0
  9. package/bin/mlx-bun.upload.mjs +4 -0
  10. package/dist/web/app.js +9584 -0
  11. package/package.json +50 -53
  12. package/src/cli/ane-install.ts +76 -0
  13. package/src/cli/args.ts +179 -0
  14. package/src/cli/engine-entry.ts +68 -0
  15. package/src/cli/inference.ts +214 -0
  16. package/src/cli/job-entry.ts +97 -0
  17. package/src/cli/library-host.ts +117 -0
  18. package/src/cli/main.ts +57 -0
  19. package/src/cli/memory-engine.ts +163 -0
  20. package/src/cli/memory.ts +36 -0
  21. package/src/cli/mlx-lm-aliases.ts +309 -0
  22. package/src/cli/model-choice.ts +33 -0
  23. package/src/cli/model-selection.ts +84 -0
  24. package/src/cli/module-host.ts +11 -0
  25. package/src/cli/module-verbs.ts +50 -0
  26. package/src/cli/numerical-policy.ts +12 -0
  27. package/src/cli/perplexity.ts +89 -0
  28. package/src/cli/serve-host.ts +423 -0
  29. package/src/cli/serve-isolated.ts +217 -0
  30. package/src/cli/serve-options.ts +106 -0
  31. package/src/cli/serve-state.ts +205 -0
  32. package/src/cli/serve.ts +500 -0
  33. package/src/cli/served-model-host.ts +102 -0
  34. package/src/cli/server-entry.ts +148 -0
  35. package/src/cli/serving-unit.ts +177 -0
  36. package/src/{tui.ts → cli/terminal.ts} +13 -8
  37. package/src/cli/worker-adapters.ts +34 -0
  38. package/src/cli/worker-entry.ts +272 -0
  39. package/src/cli/worker-unit.ts +179 -0
  40. package/src/engine/adapter-operation.ts +26 -0
  41. package/src/engine/cache-services.ts +114 -0
  42. package/src/engine/completion.ts +39 -0
  43. package/src/engine/generation-gateway.ts +358 -0
  44. package/src/engine/index.ts +59 -0
  45. package/src/engine/kv-budget.ts +120 -0
  46. package/src/engine/media-preparation.ts +232 -0
  47. package/src/engine/model-binding.ts +24 -0
  48. package/src/engine/model-host.ts +513 -0
  49. package/src/engine/model-serving.ts +89 -0
  50. package/src/{serve → engine}/preparation.ts +10 -8
  51. package/src/{serve → engine}/session-completion-engine.ts +6 -6
  52. package/src/engine/telemetry.ts +109 -0
  53. package/src/hub/downloads.ts +116 -0
  54. package/src/jobs/db.ts +16 -7
  55. package/src/jobs/events.ts +29 -0
  56. package/src/jobs/executable.ts +5 -0
  57. package/src/jobs/host.ts +51 -0
  58. package/src/jobs/in-process.ts +55 -0
  59. package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
  60. package/src/jobs/runner.ts +150 -231
  61. package/src/jobs/service.ts +75 -0
  62. package/src/jobs/sse.ts +59 -23
  63. package/src/jobs/worker-memory.ts +46 -0
  64. package/src/jobs/worker-process.ts +163 -0
  65. package/src/jobs/worker-supervisor.ts +258 -0
  66. package/src/modules.ts +58 -0
  67. package/src/publishing/catalog-hub.ts +18 -0
  68. package/src/publishing/credentials.ts +31 -0
  69. package/src/publishing/upload.ts +37 -0
  70. package/src/residency/leased-adapters.ts +21 -0
  71. package/src/residency/local-records.ts +16 -0
  72. package/src/residency/model-residency.ts +405 -0
  73. package/src/residency/record-index.ts +28 -0
  74. package/src/residency/resident-estimate.ts +50 -0
  75. package/src/{anthropic.ts → server/anthropic.ts} +3 -81
  76. package/src/server/cache-routes.ts +36 -0
  77. package/src/{serve → server}/chat-request.ts +12 -10
  78. package/src/{serve → server}/chat-stage.ts +27 -22
  79. package/src/{client.ts → server/client.ts} +28 -2
  80. package/src/{serve → server}/completion-executor.ts +42 -42
  81. package/src/server/decision-body.ts +55 -0
  82. package/src/server/decision-routes.ts +134 -0
  83. package/src/server/decision-wire.ts +228 -0
  84. package/src/{serve → server}/discovery-routes.ts +42 -83
  85. package/src/server/generated-token-history.ts +57 -0
  86. package/src/{serve → server}/http.ts +25 -13
  87. package/src/{serve → server}/inference-request.ts +5 -5
  88. package/src/server/job-routes.ts +20 -0
  89. package/src/server/media-prompt.ts +63 -0
  90. package/src/server/memory-completion-client.ts +46 -0
  91. package/src/server/model-routes.ts +192 -0
  92. package/src/{serve → server}/openai-wire.ts +3 -2
  93. package/src/{serve → server}/pipeline.ts +1 -1
  94. package/src/server/prompt-builder.ts +13 -0
  95. package/src/server/prompt-contracts.ts +17 -0
  96. package/src/server/proxy-routes.ts +187 -0
  97. package/src/server/publishing-routes.ts +41 -0
  98. package/src/{serve → server}/request-plan.ts +8 -8
  99. package/src/{serve → server}/request-prep.ts +82 -68
  100. package/src/{serve → server}/responses-client.ts +30 -13
  101. package/src/{responses.ts → server/responses.ts} +12 -93
  102. package/src/server/routes.ts +205 -0
  103. package/src/server/start.ts +72 -0
  104. package/src/server/status-routes.ts +108 -0
  105. package/src/{serve → server}/text-completion-stage.ts +25 -4
  106. package/src/server/text-prompt.ts +12 -0
  107. package/src/{serve → server}/token-streams.ts +36 -38
  108. package/src/server/worker-routes.ts +311 -0
  109. package/src/storage/paths.ts +61 -0
  110. package/src/web/assets.ts +37 -0
  111. package/src/web/{src → browser}/api.ts +5 -23
  112. package/src/web/browser/chat-host.ts +17 -0
  113. package/src/web/browser/installed-panels.ts +6 -0
  114. package/src/web/browser/main.ts +55 -0
  115. package/src/web/browser/markdown.ts +279 -0
  116. package/src/web/browser/memory-panel.ts +9 -0
  117. package/src/web/browser/palette.ts +64 -0
  118. package/src/web/browser/protocol.ts +13 -0
  119. package/src/web/browser/shell.ts +467 -0
  120. package/src/web/{src → browser}/status.ts +32 -36
  121. package/src/web/build.ts +51 -0
  122. package/src/web/public/app.html +1054 -0
  123. package/src/web/{sw.js → public/sw.js} +4 -3
  124. package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
  125. package/src/web/{vendor → public/vendor}/hljs.js +2 -2
  126. package/THIRD_PARTY_LICENSES.md +0 -51
  127. package/docs/README.md +0 -94
  128. package/docs/archive/hlg-sampling.md +0 -350
  129. package/docs/archive/investigations/audio-input-plan.md +0 -356
  130. package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
  131. package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
  132. package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
  133. package/docs/archive/investigations/curve-bisector-routing.md +0 -123
  134. package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
  135. package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
  136. package/docs/archive/investigations/dspark-handoff.md +0 -224
  137. package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
  138. package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
  139. package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
  140. package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
  141. package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
  142. package/docs/archive/investigations/lab-build-journal.md +0 -157
  143. package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
  144. package/docs/archive/investigations/openjarvis-fit.md +0 -198
  145. package/docs/archive/investigations/openwiki-evaluation.md +0 -249
  146. package/docs/archive/investigations/optimization_plan.md +0 -346
  147. package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
  148. package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
  149. package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
  150. package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
  151. package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
  152. package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
  153. package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
  154. package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
  155. package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
  156. package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
  157. package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
  158. package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
  159. package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
  160. package/docs/archive/planning/chat-ui-vision.md +0 -50
  161. package/docs/archive/planning/chunk-task-roadmap.md +0 -77
  162. package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
  163. package/docs/archive/planning/journal.md +0 -316
  164. package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
  165. package/docs/archive/planning/pi-integration-review.md +0 -178
  166. package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
  167. package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
  168. package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
  169. package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
  170. package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
  171. package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
  172. package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
  173. package/docs/archive/planning/web-ui-pass-plan.md +0 -213
  174. package/docs/archive/planning/website-readme-pass-plan.md +0 -224
  175. package/docs/design/batching.md +0 -1841
  176. package/docs/design/decode-speed-program.md +0 -2349
  177. package/docs/design/dreaming-nightly-pipeline.md +0 -684
  178. package/docs/design/generic-model-support.md +0 -705
  179. package/docs/design/kv-cache.md +0 -824
  180. package/docs/design/orpo-training.md +0 -680
  181. package/docs/design/speculative-decoding.md +0 -1990
  182. package/docs/design/turboquant.md +0 -2212
  183. package/docs/design/unified-engine-frontier-plan.md +0 -1880
  184. package/docs/design/web-chat-redesign.md +0 -695
  185. package/docs/planning/PRODUCT_ROADMAP.md +0 -594
  186. package/docs/planning/ResearchTopics.md +0 -119
  187. package/docs/planning/release-notes-v0.4.0.md +0 -45
  188. package/docs/planning/video-series.md +0 -421
  189. package/docs/reference/benchmarks.md +0 -3613
  190. package/docs/reference/cli.md +0 -585
  191. package/docs/reference/distribution.md +0 -355
  192. package/docs/reference/environment.md +0 -398
  193. package/docs/reference/glossary.md +0 -39
  194. package/docs/reference/library-api.md +0 -674
  195. package/docs/reference/memory.md +0 -260
  196. package/docs/reference/models.md +0 -334
  197. package/docs/reference/server-api.md +0 -1775
  198. package/docs/reference/server-config.md +0 -846
  199. package/docs/reference/training.md +0 -462
  200. package/docs/reference/troubleshooting.md +0 -158
  201. package/src/admission-plan.ts +0 -18
  202. package/src/audio/conformer.ts +0 -647
  203. package/src/audio/decode.ts +0 -133
  204. package/src/audio/features.ts +0 -202
  205. package/src/audio/transcode.ts +0 -67
  206. package/src/backends/mlx/assistant-target.ts +0 -46
  207. package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
  208. package/src/backends/mlx/autoregressive.ts +0 -108
  209. package/src/backends/mlx/batch-group.ts +0 -1790
  210. package/src/backends/mlx/cache-attachment.ts +0 -67
  211. package/src/backends/mlx/cache-layout.ts +0 -31
  212. package/src/backends/mlx/checkpoint-state.ts +0 -53
  213. package/src/backends/mlx/continuation-persistence.ts +0 -106
  214. package/src/backends/mlx/continuation-request.ts +0 -43
  215. package/src/backends/mlx/continuation.ts +0 -77
  216. package/src/backends/mlx/diffusion.ts +0 -39
  217. package/src/backends/mlx/draft-target.ts +0 -32
  218. package/src/backends/mlx/evaluation.ts +0 -105
  219. package/src/backends/mlx/fill-append.ts +0 -45
  220. package/src/backends/mlx/full-kv-row-append.ts +0 -32
  221. package/src/backends/mlx/full-kv-row-donor.ts +0 -54
  222. package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
  223. package/src/backends/mlx/gateway-binding.ts +0 -130
  224. package/src/backends/mlx/graph.ts +0 -38
  225. package/src/backends/mlx/kv-codec.ts +0 -9
  226. package/src/backends/mlx/kv-maintenance.ts +0 -133
  227. package/src/backends/mlx/kv-tensor-rows.ts +0 -212
  228. package/src/backends/mlx/memory-client.ts +0 -189
  229. package/src/backends/mlx/methods.ts +0 -90
  230. package/src/backends/mlx/model-prompt.ts +0 -164
  231. package/src/backends/mlx/model-serving.ts +0 -79
  232. package/src/backends/mlx/prefill-cohort.ts +0 -89
  233. package/src/backends/mlx/prefill-rows.ts +0 -200
  234. package/src/backends/mlx/prefill.ts +0 -33
  235. package/src/backends/mlx/projected-context-rows.ts +0 -72
  236. package/src/backends/mlx/request-policy.ts +0 -20
  237. package/src/backends/mlx/request-state-policy.ts +0 -41
  238. package/src/backends/mlx/rollback.ts +0 -89
  239. package/src/backends/mlx/row-sampling.ts +0 -49
  240. package/src/backends/mlx/serial-executor.ts +0 -242
  241. package/src/backends/mlx/speculative-group.ts +0 -359
  242. package/src/backends/mlx/speculative-round.ts +0 -161
  243. package/src/backends/mlx/speculative.ts +0 -114
  244. package/src/backends/mlx/state-rows.ts +0 -104
  245. package/src/backends/mlx/state-views.ts +0 -46
  246. package/src/backends/mlx/target-layout-capability.ts +0 -35
  247. package/src/backends/mlx/text-engine.ts +0 -73
  248. package/src/backends/mlx/transitioning-kv-rows.ts +0 -165
  249. package/src/chat-template.ts +0 -382
  250. package/src/cli.ts +0 -3168
  251. package/src/config.ts +0 -399
  252. package/src/contracts/completion.ts +0 -16
  253. package/src/contracts/execution.ts +0 -60
  254. package/src/contracts/generation.ts +0 -103
  255. package/src/contracts/host.ts +0 -7
  256. package/src/contracts/pi-provider.ts +0 -30
  257. package/src/contracts/pi.ts +0 -259
  258. package/src/contracts/prefix-cache.ts +0 -23
  259. package/src/contracts/resources.ts +0 -25
  260. package/src/contracts/sampling.ts +0 -23
  261. package/src/contracts/scheduling.ts +0 -45
  262. package/src/contracts/tsconfig.json +0 -14
  263. package/src/dataset/generators.ts +0 -1087
  264. package/src/dataset/index.ts +0 -12
  265. package/src/dataset/job.ts +0 -81
  266. package/src/dataset/llm.ts +0 -71
  267. package/src/dataset/registry.ts +0 -528
  268. package/src/diffusion/diffusion-generate.ts +0 -507
  269. package/src/download.ts +0 -505
  270. package/src/embed.ts +0 -77
  271. package/src/engine/admission.ts +0 -59
  272. package/src/engine/cancellation.ts +0 -38
  273. package/src/engine/engine.ts +0 -56
  274. package/src/engine/execution-coordinator.ts +0 -70
  275. package/src/engine/execution-plan.ts +0 -53
  276. package/src/engine/resources.ts +0 -53
  277. package/src/engine/scheduler.ts +0 -72
  278. package/src/engine/session.ts +0 -247
  279. package/src/eval/README.md +0 -130
  280. package/src/eval/ifeval.ts +0 -81
  281. package/src/eval/kl-prompts.ts +0 -29
  282. package/src/eval/kl.ts +0 -590
  283. package/src/eval/perplexity.ts +0 -164
  284. package/src/eval/quality-db.ts +0 -92
  285. package/src/eval/reference-logits.ts +0 -207
  286. package/src/eval/runner.ts +0 -124
  287. package/src/eval/score.ts +0 -49
  288. package/src/eval/tasks/bfcl.ts +0 -535
  289. package/src/eval/tasks/gsm8k.ts +0 -98
  290. package/src/eval/tasks/hashhop.ts +0 -139
  291. package/src/eval/tasks/humaneval.ts +0 -275
  292. package/src/eval/tasks/ifeval.ts +0 -525
  293. package/src/eval/tasks/mmlu.ts +0 -157
  294. package/src/eval/ultrafeedback.ts +0 -60
  295. package/src/evaldb.ts +0 -199
  296. package/src/expert-io.ts +0 -393
  297. package/src/expert-offload-build.ts +0 -77
  298. package/src/expert-offload.ts +0 -76
  299. package/src/expert-residency.ts +0 -1134
  300. package/src/expert-usage.ts +0 -415
  301. package/src/fill/echo-index.ts +0 -188
  302. package/src/fill/fill-session.ts +0 -452
  303. package/src/fill/proposal.ts +0 -68
  304. package/src/fill/schema-rows.ts +0 -506
  305. package/src/fill/tool-boundary.ts +0 -103
  306. package/src/fit.ts +0 -249
  307. package/src/generate.ts +0 -1346
  308. package/src/grammar.ts +0 -591
  309. package/src/harness-pi.ts +0 -184
  310. package/src/hf-push.ts +0 -628
  311. package/src/hub-rest.ts +0 -242
  312. package/src/index.ts +0 -63
  313. package/src/inference/denoising.ts +0 -17
  314. package/src/inference/draft-acceptance.ts +0 -41
  315. package/src/inference/draft-output.ts +0 -49
  316. package/src/inference/graph.ts +0 -33
  317. package/src/inference/prefill.ts +0 -39
  318. package/src/inference/rollback.ts +0 -7
  319. package/src/jobs/index.ts +0 -31
  320. package/src/jobs/job-entry.ts +0 -107
  321. package/src/jobs/task-client.ts +0 -19
  322. package/src/jobs/types.ts +0 -2
  323. package/src/kv-scheme.ts +0 -257
  324. package/src/kv-store.ts +0 -1099
  325. package/src/lab/README.md +0 -18
  326. package/src/lab/curve/curve-designer.html +0 -303
  327. package/src/lab/curve/curve-sampler.ts +0 -122
  328. package/src/lab/expert-trace/expert-trace.ts +0 -69
  329. package/src/lab/paged-kv/paged-kv-rows.ts +0 -83
  330. package/src/lab/paged-kv/paged-kv.ts +0 -270
  331. package/src/library.ts +0 -40
  332. package/src/lora.ts +0 -408
  333. package/src/md-modules.d.ts +0 -49
  334. package/src/media-fetch.ts +0 -442
  335. package/src/memory/article.ts +0 -558
  336. package/src/memory/chunk-validation.ts +0 -150
  337. package/src/memory/chunk.ts +0 -318
  338. package/src/memory/cluster.ts +0 -351
  339. package/src/memory/coverage.ts +0 -134
  340. package/src/memory/crosslink.ts +0 -0
  341. package/src/memory/db.ts +0 -253
  342. package/src/memory/entity.ts +0 -203
  343. package/src/memory/gate.ts +0 -187
  344. package/src/memory/history.ts +0 -152
  345. package/src/memory/ingest.ts +0 -335
  346. package/src/memory/model.ts +0 -183
  347. package/src/memory/normalize.ts +0 -671
  348. package/src/memory/parse.ts +0 -22
  349. package/src/memory/pipeline.ts +0 -275
  350. package/src/memory/prompts.ts +0 -65
  351. package/src/memory/query.ts +0 -334
  352. package/src/memory/reconcile.ts +0 -734
  353. package/src/memory/redirect.ts +0 -108
  354. package/src/memory/reindex.ts +0 -325
  355. package/src/memory/resolve.ts +0 -307
  356. package/src/memory/rest.ts +0 -245
  357. package/src/memory/route.ts +0 -272
  358. package/src/memory/schedule.ts +0 -165
  359. package/src/memory/stages.ts +0 -777
  360. package/src/memory/synthesize.ts +0 -1638
  361. package/src/memory/tools.ts +0 -749
  362. package/src/memory/vault.ts +0 -1194
  363. package/src/memory/wikify.ts +0 -1064
  364. package/src/mlx/array.ts +0 -388
  365. package/src/mlx/autograd.ts +0 -304
  366. package/src/mlx/checkpoint.ts +0 -149
  367. package/src/mlx/compile.ts +0 -168
  368. package/src/mlx/custom-vjp.ts +0 -188
  369. package/src/mlx/ffi.ts +0 -512
  370. package/src/mlx/materialize.ts +0 -23
  371. package/src/mlx/metal-kernel.ts +0 -159
  372. package/src/mlx/ops.ts +0 -1021
  373. package/src/mlx/quantized-rows.ts +0 -36
  374. package/src/mlx/token-bitmask.ts +0 -52
  375. package/src/mlx/turboquant-kv-decode.ts +0 -82
  376. package/src/mlx/turboquant-kv-inverse.ts +0 -122
  377. package/src/mlx/turboquant-ops.ts +0 -861
  378. package/src/mlx/turboquant-tables.ts +0 -237
  379. package/src/mmap.ts +0 -90
  380. package/src/model/artifact-identity.ts +0 -73
  381. package/src/model/batched-kv.ts +0 -235
  382. package/src/model/batched-mask.ts +0 -279
  383. package/src/model/batched-quant.ts +0 -212
  384. package/src/model/batched-quantized-kv.ts +0 -144
  385. package/src/model/batched-rotating-quant.ts +0 -353
  386. package/src/model/batched-rotating-state.ts +0 -149
  387. package/src/model/batched-rotating.ts +0 -338
  388. package/src/model/batched-row-storage.ts +0 -179
  389. package/src/model/batched-ssm.ts +0 -52
  390. package/src/model/batched-turboquant-kv.ts +0 -93
  391. package/src/model/compiled-decode.ts +0 -712
  392. package/src/model/decoded-kv-donor.ts +0 -12
  393. package/src/model/delayed-quantized-kv.ts +0 -72
  394. package/src/model/delayed-rotating-quantized-kv.ts +0 -93
  395. package/src/model/delayed-turboquant-kv.ts +0 -55
  396. package/src/model/diffusion-gemma.ts +0 -1026
  397. package/src/model/factory.ts +0 -189
  398. package/src/model/fingerprint.ts +0 -33
  399. package/src/model/flash-attention.ts +0 -694
  400. package/src/model/full-prefill-padding.ts +0 -56
  401. package/src/model/full-prefill-row.ts +0 -105
  402. package/src/model/gemma4-base.ts +0 -2351
  403. package/src/model/gemma4.ts +0 -1315
  404. package/src/model/generated/gemma4-12b.ts +0 -429
  405. package/src/model/generated/gemma4-26b.ts +0 -448
  406. package/src/model/generated/gemma4-e4b.ts +0 -717
  407. package/src/model/generated/index.ts +0 -18
  408. package/src/model/glm52-atlas.ts +0 -511
  409. package/src/model/glm52-cache.ts +0 -792
  410. package/src/model/glm52-config.ts +0 -262
  411. package/src/model/glm52-container.ts +0 -305
  412. package/src/model/glm52-coupling.ts +0 -388
  413. package/src/model/glm52-dsa.ts +0 -606
  414. package/src/model/glm52-expert-layout.ts +0 -253
  415. package/src/model/glm52-layout.ts +0 -152
  416. package/src/model/glm52-memory.ts +0 -500
  417. package/src/model/glm52-mla.ts +0 -792
  418. package/src/model/glm52-moe.ts +0 -223
  419. package/src/model/glm52-pilot.ts +0 -537
  420. package/src/model/glm52-reference.ts +0 -445
  421. package/src/model/glm52-residency.ts +0 -557
  422. package/src/model/glm52-resident-weights.ts +0 -135
  423. package/src/model/glm52-streamed-experts.ts +0 -453
  424. package/src/model/glm52-streamed-metal.ts +0 -610
  425. package/src/model/glm52-weights.ts +0 -503
  426. package/src/model/glm52.ts +0 -1132
  427. package/src/model/implementation.ts +0 -69
  428. package/src/model/kv-attention-view.ts +0 -67
  429. package/src/model/minicpm5.ts +0 -376
  430. package/src/model/profile.ts +0 -443
  431. package/src/model/qwen-append.ts +0 -12
  432. package/src/model/qwen-conv.ts +0 -52
  433. package/src/model/qwen3-delta-state.ts +0 -74
  434. package/src/model/qwen3-delta.ts +0 -451
  435. package/src/model/qwen3-moe.ts +0 -404
  436. package/src/model/qwen3-mrope.ts +0 -252
  437. package/src/model/qwen3.ts +0 -277
  438. package/src/model/qwen3_5-checkpoint.ts +0 -164
  439. package/src/model/qwen3_5.ts +0 -819
  440. package/src/model/rotating-kv-donor.ts +0 -35
  441. package/src/model/rotating-kv-layout.ts +0 -262
  442. package/src/model/rotating-row-transaction.ts +0 -35
  443. package/src/model/speculative-rotating-kv.ts +0 -101
  444. package/src/model/ssm-prefill-padding.ts +0 -39
  445. package/src/model/support.ts +0 -102
  446. package/src/model/trellis-balanced-scatter.ts +0 -51
  447. package/src/model/trellis-linear.ts +0 -672
  448. package/src/model/trellis-shared-m.ts +0 -151
  449. package/src/model/trellis-shared-scatter.ts +0 -144
  450. package/src/model/trellis-splitk-prefill.ts +0 -158
  451. package/src/model/trellis-tiled-prefill.ts +0 -136
  452. package/src/model/trellis-vector-expand.ts +0 -62
  453. package/src/model/trellis-wide-prefill.ts +0 -94
  454. package/src/model/turboquant-codec.ts +0 -83
  455. package/src/model/universal/archs.ts +0 -332
  456. package/src/model/universal/dense.ts +0 -612
  457. package/src/model/universal/modules.ts +0 -170
  458. package/src/model/universal/rope.ts +0 -381
  459. package/src/native/expert_io.c +0 -523
  460. package/src/native/frame_extract.swift +0 -140
  461. package/src/native-pack.ts +0 -162
  462. package/src/pi-provider.ts +0 -53
  463. package/src/pi-session.ts +0 -81
  464. package/src/pi-terminal.ts +0 -274
  465. package/src/pi-web.ts +0 -1932
  466. package/src/preflight.ts +0 -158
  467. package/src/prompt-cache.ts +0 -460
  468. package/src/quantize/allocator.ts +0 -627
  469. package/src/quantize/atomic-output.ts +0 -38
  470. package/src/quantize/calibration.ts +0 -181
  471. package/src/quantize/config-writer.ts +0 -145
  472. package/src/quantize/index.ts +0 -84
  473. package/src/quantize/job.ts +0 -162
  474. package/src/quantize/quantizer.ts +0 -666
  475. package/src/quantize/rotate.ts +0 -603
  476. package/src/quantize/safetensors-writer.ts +0 -249
  477. package/src/quantize/sensitivity.ts +0 -216
  478. package/src/quantize/trellis.ts +0 -586
  479. package/src/quantize/weight-transform.ts +0 -292
  480. package/src/registry.ts +0 -602
  481. package/src/runtime-config.ts +0 -81
  482. package/src/safetensors.ts +0 -158
  483. package/src/sampler.ts +0 -1011
  484. package/src/serve/admin-routes.ts +0 -184
  485. package/src/serve/assets/curve-terrain.html +0 -105
  486. package/src/serve/assets/training-inference-map.html +0 -575
  487. package/src/serve/aux-routes.ts +0 -132
  488. package/src/serve/batch-scheduler.ts +0 -8
  489. package/src/serve/checkpoint-identity.ts +0 -64
  490. package/src/serve/generation-gateway.ts +0 -496
  491. package/src/serve/isolate.ts +0 -609
  492. package/src/serve/kv-budget.ts +0 -14
  493. package/src/serve/lab-routes.ts +0 -341
  494. package/src/serve/lane-registry.ts +0 -45
  495. package/src/serve/model-admin-routes.ts +0 -155
  496. package/src/serve/model-binding.ts +0 -43
  497. package/src/serve/model-host.ts +0 -511
  498. package/src/serve/parent-application.ts +0 -56
  499. package/src/serve/prompt-response-trace.ts +0 -147
  500. package/src/serve/response-lease.ts +0 -24
  501. package/src/serve/session-search.ts +0 -326
  502. package/src/serve/static-routes.ts +0 -101
  503. package/src/server.ts +0 -1314
  504. package/src/spec/assistant-rows.ts +0 -92
  505. package/src/spec/assistant-source.ts +0 -126
  506. package/src/spec/cache-identity.ts +0 -22
  507. package/src/spec/deepspec-rows.ts +0 -29
  508. package/src/spec/deepspec-source.ts +0 -145
  509. package/src/spec/dflash-source.ts +0 -149
  510. package/src/spec/drafter.ts +0 -304
  511. package/src/spec/dspark/ab-stats.ts +0 -136
  512. package/src/spec/dspark/calibration.ts +0 -112
  513. package/src/spec/dspark/data-dflash.ts +0 -156
  514. package/src/spec/dspark/data.ts +0 -201
  515. package/src/spec/dspark/deepspec-module.ts +0 -776
  516. package/src/spec/dspark/generate-dflash.ts +0 -245
  517. package/src/spec/dspark/generate.ts +0 -290
  518. package/src/spec/dspark/loader.ts +0 -23
  519. package/src/spec/dspark/loss.ts +0 -213
  520. package/src/spec/dspark/module-dflash.ts +0 -560
  521. package/src/spec/dspark/module.ts +0 -540
  522. package/src/spec/dspark/quantize-drafter.ts +0 -91
  523. package/src/spec/dspark/sample.ts +0 -112
  524. package/src/spec/generate.ts +0 -276
  525. package/src/spec/glm52-mtp-graph.ts +0 -29
  526. package/src/spec/glm52-mtp-rows.ts +0 -132
  527. package/src/spec/glm52-mtp-source.ts +0 -317
  528. package/src/spec/glm52-mtp-state.ts +0 -33
  529. package/src/spec/ngram-source.ts +0 -204
  530. package/src/spec/projected-draft-rows.ts +0 -73
  531. package/src/spec/qwen-mtp-module.ts +0 -171
  532. package/src/spec/qwen-mtp-rows.ts +0 -219
  533. package/src/spec/qwen-mtp-source.ts +0 -383
  534. package/src/spec/qwen-mtp-state.ts +0 -30
  535. package/src/spec/serve-loop.ts +0 -585
  536. package/src/spec/source.ts +0 -215
  537. package/src/spec/two-model-rows.ts +0 -61
  538. package/src/spec/two-model.ts +0 -113
  539. package/src/ssd-cache.ts +0 -372
  540. package/src/ssd-durability.ts +0 -211
  541. package/src/tokenizer.ts +0 -335
  542. package/src/tool-approvals.ts +0 -131
  543. package/src/tool-call.ts +0 -695
  544. package/src/train/dataset.ts +0 -484
  545. package/src/train/diffusion-lora.ts +0 -216
  546. package/src/train/export.ts +0 -32
  547. package/src/train/flash-cce.ts +0 -1234
  548. package/src/train/forward.ts +0 -266
  549. package/src/train/fuse.ts +0 -243
  550. package/src/train/index.ts +0 -46
  551. package/src/train/job.ts +0 -205
  552. package/src/train/lora-params.ts +0 -254
  553. package/src/train/loss.ts +0 -1591
  554. package/src/train/merge.ts +0 -195
  555. package/src/train/optimizer.ts +0 -183
  556. package/src/train/prefix-shared.ts +0 -490
  557. package/src/train/rank.ts +0 -137
  558. package/src/train/segmented.ts +0 -1828
  559. package/src/train/steel-qmm-header.ts +0 -1461
  560. package/src/train/trainer.ts +0 -1221
  561. package/src/train/watch.ts +0 -542
  562. package/src/vision/diffusion-vision.ts +0 -620
  563. package/src/vision/embedder.ts +0 -168
  564. package/src/vision/preprocess.ts +0 -247
  565. package/src/vision/prompt.ts +0 -373
  566. package/src/vision/qwen3vl-preprocess.ts +0 -409
  567. package/src/vision/qwen3vl-prompt.ts +0 -196
  568. package/src/vision/qwen3vl-tower.ts +0 -555
  569. package/src/vision/siglip.ts +0 -599
  570. package/src/vision/video-frames.ts +0 -148
  571. package/src/web/app.html +0 -2143
  572. package/src/web/app.js +0 -5703
  573. package/src/web/skills/memory/SKILL.md +0 -120
  574. package/src/web/skills/web-research/SKILL.md +0 -46
  575. package/src/web/skills.ts +0 -97
  576. package/src/web/src/adapters-panel.ts +0 -288
  577. package/src/web/src/assistant.ts +0 -451
  578. package/src/web/src/chat.ts +0 -1057
  579. package/src/web/src/composer.ts +0 -959
  580. package/src/web/src/dataset.ts +0 -132
  581. package/src/web/src/finetune.ts +0 -267
  582. package/src/web/src/hub.ts +0 -428
  583. package/src/web/src/main.ts +0 -98
  584. package/src/web/src/markdown.ts +0 -0
  585. package/src/web/src/memory-panel.ts +0 -572
  586. package/src/web/src/model-picker.ts +0 -159
  587. package/src/web/src/palette.ts +0 -346
  588. package/src/web/src/protocol.ts +0 -26
  589. package/src/web/src/quantize.ts +0 -167
  590. package/src/web/src/rag.ts +0 -274
  591. package/src/web/src/sessions.ts +0 -469
  592. package/src/web/src/shell.ts +0 -804
  593. package/src/web/src/ui-catalog.ts +0 -78
  594. package/src/web/vendor/README.md +0 -61
  595. package/src/web-assets.ts +0 -47
  596. package/src/web-tools.ts +0 -478
  597. package/src/weights.ts +0 -255
  598. /package/src/{serve → server}/completion-sink.ts +0 -0
  599. /package/src/web/{icon.svg → public/icon.svg} +0 -0
  600. /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
  601. /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
@@ -1,1638 +0,0 @@
1
- // mlx-bun memory — SYNTHESIZE / CREATE flow (P6-T3).
2
- //
3
- // The write pipeline's cold-start branch: turn an entity's accumulated chunks
4
- // into a fresh Wikipedia-shaped article. Entered ONLY when the P1-T5/P5-T4
5
- // CREATE rule fired for the entity (≥3 routed chunks, or ≥1 substantive
6
- // ownership/decision chunk). The complementary UPDATE branch (fold one chunk
7
- // into an existing section) is P7's section-patch — not built here.
8
- //
9
- // The decomposition is the whole point (capacity-by-decomposition): we NEVER
10
- // hold a whole article in the model's head. CREATE is
11
- // sub-cluster → OUTLINE (a clean TOC of single-topic sections) → per-section
12
- // DRAFT (each section drafted from empty, the SAME bounded op as a patch) →
13
- // seed INFOBOX → assemble.
14
- // Sharp section structure is the primary quality goal: the read path's
15
- // TOC→section reads ride this outline.
16
- //
17
- // Citations are deterministic by construction (AI does judgment, CODE enforces
18
- // invariants): we assign each source conversation a stable `[^N]` up front and
19
- // generate the `## References` `[^N]:` definition lines ourselves in the fixed
20
- // Lucien standard — `[^N]: `conv:HASH` (YYYY-MM-DD, source) — desc`, HASH = the
21
- // first 8 lowercase hex of the conversation UUID with hyphens stripped. The
22
- // model only PLACES the markers; it never writes a definition or invents a hash.
23
- // This sidesteps the most common base-model defect (malformed/fabricated
24
- // footnotes) entirely.
25
- //
26
- // Citation/wikilink rules are ported from lucien `scripts/synthesize.ts`
27
- // (SYNTHESIS_PROMPT_BOOTSTRAP). Every CREATE output is run through the P6-T1
28
- // gate (a weak pass → NO-OP, vault unchanged) and the P6-T2 NORMALIZE pass, then
29
- // committed via `commitVault`; the integrated (chunk, section) edges are written
30
- // to the `synthesized_chunk_sections` ledger.
31
-
32
- import { mkdir, readFile, writeFile } from "node:fs/promises";
33
- import { join } from "node:path";
34
-
35
- import {
36
- articleStructure,
37
- fenceLabel,
38
- isFenceLine,
39
- parseInfobox,
40
- repairFences,
41
- type EntityKind,
42
- } from "./article";
43
- import { LEAD_ANCHOR } from "./cluster";
44
- import type { MemoryStore } from "./db";
45
- import { checkFootnoteIntegrity, gateEdit, type GateVerdict } from "./gate";
46
- import {
47
- buildContradictionClassifierPrompt,
48
- classifyContradiction,
49
- type ContradictionKind,
50
- } from "./history";
51
- import { callLocal, MAX_OUTPUT_TOKENS } from "./model";
52
- import { hasStubMarker, normalizeArticle } from "./normalize";
53
- import { parseLines } from "./parse";
54
- import { loadMetaPolicy } from "./prompts";
55
- import {
56
- articlesDir,
57
- commitVault,
58
- extractSection,
59
- listArticles,
60
- parseToc,
61
- resolveWikilinkToStem,
62
- vaultRoot,
63
- } from "./vault";
64
-
65
- // ---- ported citation / wikilink rules (lucien SYNTHESIS_PROMPT_BOOTSTRAP) ----
66
-
67
- /** Use ONLY content from the provided chunk(s); never copy names/values from these instructions. */
68
- const NO_COPY_GUARD = `Use ONLY content from the provided chunk(s)/source notes below; never copy any name, value, or example from these instructions into the article.`;
69
-
70
- /** Wikilink contract: exact stem, underscores, never the spaced phantom form. */
71
- const WIKILINK_RULES = `WIKILINKS — link to OTHER ARTICLES below using the exact article stem, underscores not spaces: [[Exact_Stem]] or with display text [[Exact_Stem|display text]]. The spaced form [[Exact Stem]] resolves to nothing and creates a broken orphan link; it is forbidden. Only link to a stem that appears in OTHER ARTICLES.`;
72
-
73
- /** Citation contract: place the GIVEN markers, never write defs or hashes. */
74
- const CITATION_RULES = `CITATIONS — cite a substantive claim by placing one of the GIVEN footnote markers ([^1], [^2], …) immediately after it, e.g. "<claim>.[^1]". Two sources for one sentence: adjacent markers, no space — [^1][^2]. Reuse the same marker every time you cite that source. Do NOT write any footnote DEFINITION line, do NOT invent or guess a hash, do NOT emit a ## References section — the definitions are generated for you. NEVER fabricate a citation: an uncited true statement is fine, but only ever use a marker FROM THE LIST ABOVE — never a placeholder letter or any marker not in that list. FORBIDDEN — never emit HTML <sup>, #ref-/#cite- links, escaped [\\[1\\]] markers, or [[1]] wikilink markers; [^N] is the ONLY citation syntax.`;
75
-
76
- /** Date contract (CREATE): every source note is tagged with its date. The model
77
- * must treat the LATER-DATED note as the user's CURRENT position when notes
78
- * conflict — chronology is a first-class signal, not just processing order. Kept
79
- * schematic (no copyable concrete value) per the small-model parroting failure. */
80
- const DATE_RESOLVE_RULES = `DATES — each source note below is tagged with its date in [YYYY-MM-DD] form (the same date that appears in its [^N] footnote). When two notes CONFLICT (a changed favourite, a reversed preference, an updated fact), the LATER-DATED note is the user's CURRENT position: resolve toward it. Write a superseded value as a past stage ("originally …, now …"), never as still-true.`;
81
-
82
- // ---- conv:HASH + footnote scaffolding --------------------------------------
83
-
84
- /** A filesystem-safe bare article stem from a free-form entity name: collapse
85
- * whitespace to `_`, map path separators (a value containing `/`) and other
86
- * filename-hostile characters to `-`, and strip leading dots, so a model-minted
87
- * name like `<value>/<value>` yields a bare `<value>-<value>` stem (never a
88
- * `articles/…` subpath). Stems with no separators are unchanged. The original
89
- * surface survives as an `aliases:` entry, so name resolution still reaches the
90
- * article. */
91
- export function entityStem(entity: string): string {
92
- const stem = entity
93
- .trim()
94
- .replace(/[/\\]+/g, "-")
95
- .replace(/\s+/g, "_")
96
- .replace(/[<>:"|?*]+/g, "")
97
- .replace(/_+/g, "_")
98
- .replace(/^\.+/, "")
99
- .replace(/^[-_]+|[-_]+$/g, "");
100
- return stem || "entity";
101
- }
102
-
103
- /** The fixed Lucien `conv:HASH`: the first 8 lowercase hex digits of the
104
- * conversation UUID with hyphens stripped (`<8hex>-<4hex>-…` → `<8hex>`). An
105
- * already-8-char hex id passes through unchanged. */
106
- export function convHash(conv: string): string {
107
- return conv.toLowerCase().replace(/[^0-9a-f]/g, "").slice(0, 8);
108
- }
109
-
110
- /** `YYYY-MM-DD` (UTC) from an epoch-millis timestamp; `undated` when missing. */
111
- export function footnoteDate(ms: number | null | undefined): string {
112
- if (ms == null || !Number.isFinite(ms)) return "undated";
113
- return new Date(ms).toISOString().slice(0, 10);
114
- }
115
-
116
- /** The fixed Lucien footnote definition line for a source conversation as
117
- * `[^n]: `conv:HASH` (YYYY-MM-DD, source)`. The single source of truth both
118
- * CREATE (buildFootnoteMap) and PATCH (appendReferenceDef) generate defs from,
119
- * so the model never writes — and cannot malform — a definition line. */
120
- export function footnoteDefLine(n: number, conv: string, title: string, dateMs: number | null): string {
121
- const source = (title ?? "").trim() || "conversation";
122
- return `[^${n}]: \`conv:${convHash(conv)}\` (${footnoteDate(dateMs)}, ${source})`;
123
- }
124
-
125
- /** One source-conversation footnote: its stable per-article number, hash, and
126
- * the generated `[^N]: `conv:HASH` (date, source)` definition line. */
127
- export interface FootnoteEntry {
128
- conv: string;
129
- n: number;
130
- hash: string;
131
- source: string;
132
- def: string;
133
- }
134
-
135
- /** Assign each distinct source conversation a stable `[^N]` (first-appearance
136
- * order across the entity's chunks) and pre-generate its definition line in the
137
- * fixed Lucien standard. The model places these markers; it never writes defs. */
138
- export function buildFootnoteMap(chunks: SynthesisChunk[]): {
139
- byConv: Map<string, FootnoteEntry>;
140
- entries: FootnoteEntry[];
141
- } {
142
- const byConv = new Map<string, FootnoteEntry>();
143
- const entries: FootnoteEntry[] = [];
144
- for (const c of chunks) {
145
- if (byConv.has(c.conv)) continue;
146
- const n = entries.length + 1;
147
- const hash = convHash(c.conv);
148
- const source = (c.title ?? "").trim() || "conversation";
149
- const def = footnoteDefLine(n, c.conv, c.title, c.dateMs);
150
- const entry: FootnoteEntry = { conv: c.conv, n, hash, source, def };
151
- byConv.set(c.conv, entry);
152
- entries.push(entry);
153
- }
154
- return { byConv, entries };
155
- }
156
-
157
- // ---- chunk input -----------------------------------------------------------
158
-
159
- /** One source chunk for synthesis: its id, conversation, reassembled text, and
160
- * the metadata the footnote line needs. Text is NOT stored — the caller pulls
161
- * it from `store.chunkText(id)`. */
162
- export interface SynthesisChunk {
163
- id: string;
164
- conv: string;
165
- label: string | null;
166
- text: string;
167
- title: string;
168
- dateMs: number | null;
169
- }
170
-
171
- // ---- deterministic tokenizer (sub-clustering + see-also) -------------------
172
-
173
- const TOK_STOP = new Set([
174
- "the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with", "is",
175
- "vs", "and", "its", "my", "your", "our", "their",
176
- ]);
177
-
178
- function tokens(s: string): Set<string> {
179
- const out = new Set<string>();
180
- for (const t of s.toLowerCase().split(/[^a-z0-9]+/)) {
181
- if (t.length > 1 && !TOK_STOP.has(t)) out.add(t);
182
- }
183
- return out;
184
- }
185
-
186
- // ---- OUTLINE ---------------------------------------------------------------
187
-
188
- const GENERIC_SECTION = new Set(["overview", "introduction", "summary", "details", "misc", "other"]);
189
-
190
- /** Parse a model OUTLINE into clean single-topic section titles: strip bullet /
191
- * heading / numbering markup, drop empties + bare generics, dedupe by anchor,
192
- * cap at `max`. */
193
- export function parseOutline(raw: string, max = 6): string[] {
194
- const seen = new Set<string>();
195
- const out: string[] = [];
196
- for (const line of parseLines(raw)) {
197
- const title = line
198
- .replace(/^#{1,6}\s+/, "")
199
- .replace(/^[-*+]\s+/, "")
200
- .replace(/^\d+[.)]\s+/, "")
201
- .replace(/[`*_]/g, "")
202
- .trim();
203
- if (!title) continue;
204
- const key = title.toLowerCase();
205
- if (seen.has(key)) continue;
206
- seen.add(key);
207
- out.push(title);
208
- if (out.length >= max) break;
209
- }
210
- // An outline that is ONLY a generic single section is fine (→ stub); an empty
211
- // outline falls back to one "Overview" so we always draft something.
212
- if (out.length === 0) return ["Overview"];
213
- return out;
214
- }
215
-
216
- /** Anchor (heading slug) for a section title — must match vault.ts slugifyHeading
217
- * so the read path's TOC→section anchors line up. */
218
- export function sectionAnchor(title: string): string {
219
- return title
220
- .trim()
221
- .toLowerCase()
222
- .replace(/[^a-z0-9]+/g, "-")
223
- .replace(/^-+|-+$/g, "");
224
- }
225
-
226
- /**
227
- * Assign each chunk to the single best-matching section by label/text token
228
- * overlap (deterministic, GPU-free), so each section drafts only from its own
229
- * sub-topic chunks and the (chunk → section) edge is recorded. A chunk with no
230
- * overlap lands in the first section; sections that win no chunk are dropped by
231
- * the caller so we never draft an empty section.
232
- */
233
- export function subClusterChunks(
234
- chunks: SynthesisChunk[],
235
- sections: string[],
236
- ): Map<string, SynthesisChunk[]> {
237
- const byAnchor = new Map<string, SynthesisChunk[]>();
238
- const sectionToks = sections.map((t) => ({ title: t, anchor: sectionAnchor(t), toks: tokens(t) }));
239
- for (const c of chunks) {
240
- const ctoks = tokens(`${c.label ?? ""} ${c.text.slice(0, 400)}`);
241
- let best = sectionToks[0]!;
242
- let bestScore = -1;
243
- for (const s of sectionToks) {
244
- let score = 0;
245
- for (const t of s.toks) if (ctoks.has(t)) score++;
246
- if (score > bestScore) {
247
- bestScore = score;
248
- best = s;
249
- }
250
- }
251
- const list = byAnchor.get(best.anchor) ?? [];
252
- list.push(c);
253
- byAnchor.set(best.anchor, list);
254
- }
255
- return byAnchor;
256
- }
257
-
258
- // ---- prompt builders -------------------------------------------------------
259
-
260
- /** A `getOtherArticles`-style stem list (lucien synthesize.ts) so the model's
261
- * [[wikilinks]] resolve to real articles. Capped to keep the prefill bounded. */
262
- function otherArticlesBlock(stems: Iterable<string>, selfStem: string, cap = 200): string {
263
- const list = [...stems].filter((s) => s !== selfStem).sort((a, b) => a.localeCompare(b)).slice(0, cap);
264
- return list.length ? list.map((s) => `- ${s}`).join("\n") : "(none yet)";
265
- }
266
-
267
- /** Compact source block for the OUTLINE/INFOBOX/LEAD calls: each chunk DATE-TAGGED
268
- * (so the model knows which note is most recent), its label + a short text excerpt
269
- * (the model reads the gist, not the whole transcript). The `[YYYY-MM-DD]` tag is
270
- * the same date carried in the chunk's `[^N]` footnote. */
271
- function sourceDigest(chunks: SynthesisChunk[], perChunk = 600): string {
272
- return chunks
273
- .map(
274
- (c, i) =>
275
- `[${i + 1}] [${footnoteDate(c.dateMs)}] ${c.label ?? "(chunk)"}\n${c.text.slice(0, perChunk).trim()}`,
276
- )
277
- .join("\n\n");
278
- }
279
-
280
- function buildOutlinePrompt(entity: string, chunks: SynthesisChunk[], policy: string): string {
281
- return (
282
- `${policy}\n\n${NO_COPY_GUARD}\n\n` +
283
- `You are outlining a personal-wiki article about "${entity}". Below are the source notes (chunk label + excerpt). ` +
284
- `Propose a clean table of contents: 2 to 6 section headings, each a single sub-topic, in natural reading order. ` +
285
- `Each heading is 2–5 words, Title Case, specific (not "Overview"/"Details"). Output ONE heading per line, nothing else, NONE if there is only one sub-topic.\n\n` +
286
- `SOURCE NOTES:\n${sourceDigest(chunks)}\n`
287
- );
288
- }
289
-
290
- function buildLeadPrompt(
291
- entity: string,
292
- chunks: SynthesisChunk[],
293
- footnotes: Map<string, FootnoteEntry>,
294
- otherArticles: string,
295
- policy: string,
296
- ): string {
297
- return (
298
- `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n${CITATION_RULES}\n\n${DATE_RESOLVE_RULES}\n\n` +
299
- `${footnoteList(chunks, footnotes)}\n\n` +
300
- `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
301
- `Write the LEAD for a personal-wiki article about "${entity}": 2–4 sentences, present tense, naming **${entity}** in bold in the first sentence, describing what it is AND how the user relates to it (uses / owns / chose / considered). Write in the THIRD PERSON, Wikipedia-style — NEVER the first person ("I", "my"); refer to the user in the third person ("the user"). When the notes disagree, frame the user's CURRENT position from the LATEST-DATED note. Output ONLY the lead prose — no heading, no list, no References.\n\n` +
302
- `SOURCE NOTES:\n${sourceDigest(chunks)}\n`
303
- );
304
- }
305
-
306
- function buildInfoboxPrompt(
307
- entity: string,
308
- kind: EntityKind,
309
- chunks: SynthesisChunk[],
310
- otherArticles: string,
311
- policy: string,
312
- ): string {
313
- return (
314
- `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
315
- `Produce the INFOBOX facts for a personal-wiki article about "${entity}". Output ONLY \`key: value\` lines, one per line, no fence, no prose. Rules:\n` +
316
- `- snake_case keys.\n` +
317
- `- A value that names another entity is a [[wikilink]] to a stem in OTHER ARTICLES (e.g. \`<entity_key>: [[Linked Entity]]\`).\n` +
318
- `- Include \`type:\` (the emergent label — a short free-form noun phrase) when clear.\n` +
319
- `- Include world-facts only if the notes support them; include relationship-facts (owned / acquired / used_for / chosen_over / opinion) the notes establish.\n` +
320
- `- Do NOT emit \`kind:\` or \`aliases:\` — those are added for you.\n` +
321
- `Emit nothing for a fact the notes do not support — never guess a spec.\n\n` +
322
- `OTHER ARTICLES (link targets):\n${otherArticles}\n\n` +
323
- `SOURCE NOTES:\n${sourceDigest(chunks)}\n`
324
- );
325
- }
326
-
327
- /** The "Cite source X as [^n]" lines for the chunks in one draft call. */
328
- function footnoteList(chunks: SynthesisChunk[], footnotes: Map<string, FootnoteEntry>): string {
329
- const seen = new Set<number>();
330
- const lines: string[] = [];
331
- for (const c of chunks) {
332
- const e = footnotes.get(c.conv);
333
- if (!e || seen.has(e.n)) continue;
334
- seen.add(e.n);
335
- lines.push(`- [^${e.n}] = "${e.source}"`);
336
- }
337
- return lines.length
338
- ? `Cite sources with these footnote markers (use ONLY these):\n${lines.join("\n")}`
339
- : `(no citable sources for this section — leave it uncited)`;
340
- }
341
-
342
- function buildSectionPrompt(
343
- entity: string,
344
- sectionTitle: string,
345
- chunks: SynthesisChunk[],
346
- footnotes: Map<string, FootnoteEntry>,
347
- otherArticles: string,
348
- policy: string,
349
- ): string {
350
- const sources = chunks
351
- .map((c) => {
352
- const e = footnotes.get(c.conv);
353
- const tag = e ? `[^${e.n}]` : "(uncited)";
354
- return `SOURCE ${tag} [${footnoteDate(c.dateMs)}] — ${c.label ?? "chunk"}:\n${c.text.slice(0, 1400).trim()}`;
355
- })
356
- .join("\n\n---\n\n");
357
- return (
358
- `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n${CITATION_RULES}\n\n${DATE_RESOLVE_RULES}\n\n` +
359
- `${footnoteList(chunks, footnotes)}\n\n` +
360
- `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
361
- `Draft ONLY the body of the "${sectionTitle}" section of the article about "${entity}". Factual prose in the THIRD PERSON, Wikipedia-style — NEVER the first person ("I", "my"); refer to the user in the third person ("the user"). These articles document what the user thinks, so they are NOT neutral: capture the user's positions and opinions accurately. Preserve specifics, and when two sources conflict fold the correction toward the user's LATEST-DATED position. Output ONLY this section's prose — do NOT repeat the "## ${sectionTitle}" heading, do NOT write other sections, do NOT write a References section.\n\n` +
362
- `SOURCE MATERIAL:\n${sources}\n`
363
- );
364
- }
365
-
366
- // ---- model-output sanitizers -----------------------------------------------
367
-
368
- /**
369
- * Meta / planning / refusal phrasings — the base model narrating ABOUT the task
370
- * instead of writing the article body. A hit on any one rejects the draft (the
371
- * real defect: a section body that literally read "The user is asking to draft
372
- * the body prose for the <section name> section. The source material
373
- * discusses: 1. …"). Tuned to NOT fire on legitimate personal-wiki prose, which
374
- * IS about the user ("the user owned…", "the user weighed…").
375
- */
376
- const LEAK_PATTERNS: RegExp[] = [
377
- /\bthe user (is asking|is requesting|wants me|asked me|requests|would like me|needs me|wanted me)\b/i,
378
- /\bthe source material\b/i,
379
- /\bthe source notes?\b/i,
380
- /\bsource material discusses\b/i,
381
- /\bbased on the (source|provided|notes|material)\b/i,
382
- /\bdraft the body\b/i,
383
- /\bbody prose for\b/i,
384
- /\bfor the [^.\n]{0,40} section\b/i,
385
- /\bi need to\b/i,
386
- /\bi (will|shall) (draft|write|now|create|provide|outline|summarize)\b/i,
387
- /\bi'?ll (draft|write|provide|outline|summarize)\b/i,
388
- /\blet (me|us|'?s) (draft|write|outline|provide|start|begin|summarize)\b/i,
389
- /\bhere ?(is|'?s) (the|a|my) (article|section|lead|prose|draft|body)\b/i,
390
- /\bas an ai\b/i,
391
- /\bi (cannot|can'?t|can not)\b/i,
392
- /\bi'?m sorry\b/i,
393
- /\bi am sorry\b/i,
394
- /\bi apologize\b/i,
395
- /\bi (do not|don'?t) have (enough|sufficient|any)\b/i,
396
- /\bi'?m (unable|not able)\b/i,
397
- /\bi am (unable|not able)\b/i,
398
- /\bsorry,?\s+(i|but)\b/i,
399
- ];
400
-
401
- /**
402
- * True when a drafted body is meta-commentary, a plan, or a refusal rather than
403
- * article prose. Heuristic: any {@link LEAK_PATTERNS} hit, OR a leading
404
- * numbered/bulleted PLAN that restates the task. Pure; empty text is not leaky.
405
- */
406
- export function isLeakyDraft(text: string): boolean {
407
- const s = text.trim();
408
- if (!s) return false;
409
- for (const re of LEAK_PATTERNS) if (re.test(s)) return true;
410
- // A leading numbered/bulleted PLAN that restates the task ("1. discuss the
411
- // <subject>, 2. the <subject>") — a list item FIRST, plus task-restating vocabulary.
412
- const firstLine = s.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
413
- if (
414
- /^(\d+[.)]|[-*+])\s+/.test(firstLine) &&
415
- /\b(draft|discuss|summariz|describe the|outline|restate|the user|source material)\b/i.test(s)
416
- ) {
417
- return true;
418
- }
419
- return false;
420
- }
421
-
422
- /** Remove any model-emitted fenced ```info block from drafted text so the only
423
- * infobox in the final article is the ONE code-constructed block. A fenced info
424
- * block is a ``` fence whose info string is exactly `info`. */
425
- export function stripFencedInfoBlocks(text: string): string {
426
- const lines = text.split("\n");
427
- const out: string[] = [];
428
- let i = 0;
429
- while (i < lines.length) {
430
- if (isFenceLine(lines[i]!) && fenceLabel(lines[i]!) === "info") {
431
- i++; // skip the opening ```info
432
- while (i < lines.length && !isFenceLine(lines[i]!)) i++;
433
- if (i < lines.length) i++; // skip the closing fence
434
- continue;
435
- }
436
- out.push(lines[i]!);
437
- i++;
438
- }
439
- return out.join("\n");
440
- }
441
-
442
- /** Count fenced ```info blocks in an article (the single-infobox invariant). */
443
- export function countFencedInfoBlocks(content: string): number {
444
- let count = 0;
445
- let inFence = false;
446
- for (const line of content.split(/\r?\n/)) {
447
- if (isFenceLine(line)) {
448
- if (!inFence && fenceLabel(line) === "info") count++;
449
- inFence = !inFence;
450
- }
451
- }
452
- return count;
453
- }
454
-
455
- /** Strip code fences, a repeated section heading, any model-emitted References /
456
- * definition lines, a stray model-emitted infobox, and surrounding blank lines
457
- * from a drafted section body. */
458
- export function sanitizeSection(raw: string, title: string): string {
459
- let s = stripFencedInfoBlocks(raw.trim()).trim();
460
- s = s.replace(/^```[a-z]*\s*\n/i, "").replace(/\n```\s*$/i, "").trim();
461
- // Drop any NON-numeric footnote marker (e.g. a parroted `[^N]` placeholder from
462
- // the citation contract). Only `[^<digits>]` is a real, code-assigned marker.
463
- s = s.replace(/ ?\[\^[^\]\d][^\]]*\]/g, "");
464
- const wanted = title.trim().toLowerCase();
465
- const kept: string[] = [];
466
- let dropping = false;
467
- for (const line of s.split("\n")) {
468
- const t = line.trim();
469
- const h = /^(#{1,6})\s+(.+)$/.exec(t);
470
- if (h) {
471
- const ht = h[2]!.trim().toLowerCase();
472
- if (ht === wanted) continue; // model repeated our heading
473
- if (ht === "references" || ht === "see also" || ht === "notes") {
474
- dropping = true; // model spilled trailing meta sections — cut them
475
- continue;
476
- }
477
- }
478
- if (dropping) continue;
479
- if (/^\[\^\d+\]:/.test(t)) continue; // a stray definition line — we own defs
480
- kept.push(line);
481
- }
482
- // A drafted section that opened a ``` fence but never closed it would, once
483
- // assembled, swallow every trailing section (## References / ## See also) of
484
- // the WHOLE article. Balance the section's fences before it reaches assembly.
485
- return repairFences(kept.join("\n").trim()).content.trim();
486
- }
487
-
488
- /** Strip fences / a leading heading / a stray infobox from a drafted lead,
489
- * collapse to prose. */
490
- export function sanitizeLead(raw: string): string {
491
- let s = stripFencedInfoBlocks(raw.trim()).trim();
492
- s = s.replace(/^```[a-z]*\s*\n/i, "").replace(/\n```\s*$/i, "").trim();
493
- s = s.replace(/ ?\[\^[^\]\d][^\]]*\]/g, ""); // drop parroted non-numeric markers
494
- const lines = s.split("\n").filter((l) => !/^#{1,6}\s+/.test(l.trim()) && !/^\[\^\d+\]:/.test(l.trim()));
495
- // Balance any unclosed ``` so a lead fence can't swallow the rest of the article.
496
- return repairFences(lines.join("\n").trim()).content.trim();
497
- }
498
-
499
- /**
500
- * Physical spec-sheet keys describing a concrete object's hardware. They belong
501
- * on a `thing` (a concrete named object) and must NEVER bleed onto a broad
502
- * `domain` / `standard` / `person` / `project` entity — the policy-example
503
- * parroting bug, where a broad DOMAIN article inherited a hardware spec key.
504
- * Grounding the infobox in the entity's own kind blocks the leak deterministically.
505
- */
506
- export const PHYSICAL_SPEC_KEYS = new Set([
507
- "mount", "sensor", "sensor_size", "focal_length", "aperture", "format",
508
- "resolution", "megapixels", "coverage", "compatible_with", "lens_mount",
509
- ]);
510
-
511
- /**
512
- * Parse model `key: value` lines into a clean info-block field list. Keeps only
513
- * snake_case keys with a non-empty value (first occurrence wins), drops any
514
- * model-supplied `kind`/`aliases` (we own those) and any physical-spec key that
515
- * is ungrounded for this entity kind (a world-fact spec key on a `domain` is a
516
- * parroted example, never a fact), then prepends `type:` (if any), the declared `kind:`,
517
- * and appends `aliases:` when we have them. Always returns at least the `kind:`
518
- * line, so the article always has a parseable infobox.
519
- */
520
- export function buildInfoboxFields(raw: string, kind: EntityKind, aliases: string[] = []): string[] {
521
- const fields = new Map<string, string>();
522
- for (const line of raw.split("\n")) {
523
- const t = line.trim().replace(/^[-*+]\s+/, "");
524
- const idx = t.indexOf(":");
525
- if (idx <= 0) continue;
526
- const key = t.slice(0, idx).trim();
527
- const value = t.slice(idx + 1).trim();
528
- if (!/^[a-z][a-z0-9_]*$/.test(key)) continue;
529
- if (!value) continue;
530
- if (key === "kind" || key === "aliases") continue;
531
- // Ground the box in the entity's kind: a hardware spec on a non-`thing`
532
- // entity is a parroted example, not a fact established by its chunks.
533
- if (kind !== "thing" && PHYSICAL_SPEC_KEYS.has(key)) continue;
534
- if (!fields.has(key)) fields.set(key, value);
535
- }
536
- const lines: string[] = [];
537
- const type = fields.get("type");
538
- if (type) lines.push(`type: ${type}`);
539
- lines.push(`kind: ${kind}`);
540
- for (const [k, v] of fields) {
541
- if (k === "type") continue;
542
- lines.push(`${k}: ${v}`);
543
- }
544
- const cleanAliases = [...new Set(aliases.map((a) => a.trim()).filter(Boolean))];
545
- if (cleanAliases.length) lines.push(`aliases: ${cleanAliases.join(", ")}`);
546
- return lines;
547
- }
548
-
549
- // ---- assembly --------------------------------------------------------------
550
-
551
- export interface AssembledSection {
552
- title: string;
553
- anchor: string;
554
- body: string;
555
- }
556
-
557
- /** Distinct wikilink targets in `text` that resolve to a known stem (excluding
558
- * self / Category: / conv:) — the See also list, sorted. */
559
- export function deriveSeeAlso(text: string, stems: Set<string>, selfStem: string): string[] {
560
- const out = new Set<string>();
561
- for (const m of text.matchAll(/\[\[([^\]]+?)\]\]/g)) {
562
- const inner = m[1]!.split("|")[0]!.split("#")[0]!.trim();
563
- if (!inner || /^conv:/i.test(inner) || /^Category:/i.test(inner)) continue;
564
- const r = resolveWikilinkToStem(inner, stems);
565
- if (r && r !== selfStem) out.add(r);
566
- }
567
- return [...out].sort((a, b) => a.localeCompare(b));
568
- }
569
-
570
- /** Compose the fixed article skeleton from its parts. `## See also` /
571
- * `## References` are emitted only when non-empty; References is always last. */
572
- export function assembleArticle(parts: {
573
- stem: string;
574
- infoboxFields: string[];
575
- lead: string;
576
- sections: AssembledSection[];
577
- seeAlso: string[];
578
- referenceDefs: string[];
579
- }): string {
580
- const title = parts.stem.replace(/_/g, " ");
581
- const out: string[] = [`# ${title}`, ""];
582
- if (parts.infoboxFields.length) {
583
- out.push("```info", ...parts.infoboxFields, "```", "");
584
- }
585
- if (parts.lead.trim()) out.push(parts.lead.trim(), "");
586
- for (const s of parts.sections) {
587
- if (!s.body.trim()) continue;
588
- out.push(`## ${s.title}`, "", s.body.trim(), "");
589
- }
590
- if (parts.seeAlso.length) {
591
- out.push("## See also", "", ...parts.seeAlso.map((s) => `- [[${s}]]`), "");
592
- }
593
- if (parts.referenceDefs.length) {
594
- out.push("## References", "", ...parts.referenceDefs, "");
595
- }
596
- const assembled = out.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
597
- // Final assembly guard: a drafted section may still carry an unbalanced ``` that
598
- // the per-section sanitizer missed; never emit an article whose fences are odd
599
- // (it would read its own trailing ## References / ## See also as code). Closing
600
- // the dangling fence before the first heading it would swallow keeps every
601
- // trailing section a recognized heading.
602
- const repaired = repairFences(assembled).content;
603
- return repaired.replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
604
- }
605
-
606
- /** Distinct `[^N]` marker numbers present in body prose (definition lines and
607
- * fenced blocks excluded — markers live in prose). */
608
- function usedMarkers(text: string): Set<number> {
609
- const used = new Set<number>();
610
- for (const line of text.split("\n")) {
611
- if (/^\[\^\d+\]:/.test(line.trim())) continue;
612
- for (const m of line.matchAll(/\[\^(\d+)\]/g)) used.add(parseInt(m[1]!, 10));
613
- }
614
- return used;
615
- }
616
-
617
- /** Count sections (by the parsed skeleton) whose body carries ≥1 footnote
618
- * marker — the "≥1 cited section" acceptance signal. */
619
- export function countCitedSections(content: string): number {
620
- const items = articleStructure(content);
621
- const lines = content.split(/\r?\n/);
622
- let cited = 0;
623
- for (const it of items) {
624
- if (it.kind !== "section") continue;
625
- const body = lines.slice(it.startLine - 1, it.endLine).join("\n");
626
- if (/\[\^\d+\](?!:)/.test(body)) cited++;
627
- }
628
- return cited;
629
- }
630
-
631
- // ---- CREATE (pure-ish: model via deps, no disk writes) ---------------------
632
-
633
- /** Per-stage model call seam (defaults to the local synthesis stage). Injected
634
- * by tests so the deterministic scaffolding runs with no GPU. */
635
- export type SynthesisCall = (prompt: string, opts?: { maxTokens?: number }) => Promise<string>;
636
-
637
- /** Retry instruction when a draft leaks meta/planning/refusal text. */
638
- const ANTI_LEAK_HINT =
639
- "Output ONLY the article prose for this section — no preamble, no meta-commentary, no plan, no restating the task.";
640
-
641
- /**
642
- * Draft prose that is NOT meta/planning/refusal leakage: draft once; if the
643
- * sanitized body is {@link isLeakyDraft}, retry ONCE with a sharper instruction;
644
- * if it still leaks, return "" so the caller DROPS the section rather than
645
- * emitting garbage. The sanitizer (section vs lead) is injected.
646
- */
647
- async function draftCleanProse(
648
- call: SynthesisCall,
649
- prompt: string,
650
- opts: { maxTokens?: number },
651
- sanitize: (raw: string) => string,
652
- ): Promise<string> {
653
- let out = sanitize(await call(prompt, opts));
654
- if (!isLeakyDraft(out)) return out;
655
- out = sanitize(await call(`${prompt}\n\n${ANTI_LEAK_HINT}`, opts));
656
- return isLeakyDraft(out) ? "" : out;
657
- }
658
-
659
- export interface CreateDeps {
660
- /** Model call; defaults to `callLocal("synthesis", …)`. */
661
- call?: SynthesisCall;
662
- /** Other article stems (for wikilink resolution); defaults to the vault list. */
663
- stems?: Set<string>;
664
- /** Declared aliases to seed the infobox `aliases:` line. */
665
- aliases?: string[];
666
- /** Pre-existing article content for the gate's `before` (CREATE: ""). */
667
- before?: string;
668
- }
669
-
670
- export interface CreateOutcome {
671
- entity: string;
672
- stem: string;
673
- /** Final post-NORMALIZE markdown, or null when the gate vetoed the write. */
674
- content: string | null;
675
- action: "created" | "skipped";
676
- /** Gate reason when skipped; null on success. */
677
- reason: string | null;
678
- /** (chunk_id, section_anchor) edges for the `synthesized_chunk_sections` ledger. */
679
- chunkSections: { chunkId: string; anchor: string }[];
680
- sectionAnchors: string[];
681
- citedSections: number;
682
- hasInfobox: boolean;
683
- stub: boolean;
684
- normalizeNotes: string[];
685
- }
686
-
687
- /**
688
- * Build ONE entity article from its accumulated chunks, end to end:
689
- * sub-cluster → OUTLINE → per-section DRAFT → seed INFOBOX → assemble →
690
- * NORMALIZE → gate. Returns the final content (or null when the gate vetoes a
691
- * weak pass). Pure with respect to disk — the caller persists.
692
- */
693
- export async function createArticle(
694
- entity: string,
695
- kind: EntityKind,
696
- chunks: SynthesisChunk[],
697
- deps: CreateDeps = {},
698
- ): Promise<CreateOutcome> {
699
- const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
700
- const stem = entityStem(entity);
701
- const stems = new Set(deps.stems ?? []);
702
- stems.add(stem);
703
- const otherArticles = otherArticlesBlock(stems, stem);
704
- const policy = loadMetaPolicyQuiet(["Article_Conventions", "Infobox_Schemas", "Entities"]);
705
- const { byConv, entries } = buildFootnoteMap(chunks);
706
-
707
- // OUTLINE → sections; assign chunks; drop sections that won no chunk.
708
- const outline = parseOutline(await call(buildOutlinePrompt(entity, chunks, policy), { maxTokens: MAX_OUTPUT_TOKENS }));
709
- const clustered = subClusterChunks(chunks, outline);
710
- const sections: AssembledSection[] = [];
711
- const chunkSections: { chunkId: string; anchor: string }[] = [];
712
- for (const title of outline) {
713
- const anchor = sectionAnchor(title);
714
- const secChunks = clustered.get(anchor);
715
- if (!secChunks || secChunks.length === 0) continue;
716
- const body = await draftCleanProse(
717
- call,
718
- buildSectionPrompt(entity, title, secChunks, byConv, otherArticles, policy),
719
- { maxTokens: MAX_OUTPUT_TOKENS },
720
- (raw) => sanitizeSection(raw, title),
721
- );
722
- // A leaky draft survives to "" after one retry — DROP the section (and its
723
- // chunk edges); never emit meta/planning/refusal prose into the article.
724
- if (!body.trim()) continue;
725
- sections.push({ title, anchor, body });
726
- for (const c of secChunks) chunkSections.push({ chunkId: c.id, anchor });
727
- }
728
-
729
- // LEAD + INFOBOX (bounded, single-purpose calls). The lead runs the same
730
- // leak-rejection so a refusal/plan never lands as the article's abstract.
731
- const lead = await draftCleanProse(
732
- call,
733
- buildLeadPrompt(entity, chunks, byConv, otherArticles, policy),
734
- { maxTokens: MAX_OUTPUT_TOKENS },
735
- sanitizeLead,
736
- );
737
- const infoboxFields = buildInfoboxFields(
738
- await call(buildInfoboxPrompt(entity, kind, chunks, otherArticles, policy), { maxTokens: MAX_OUTPUT_TOKENS }),
739
- kind,
740
- deps.aliases ?? [],
741
- );
742
-
743
- // References: only the defs whose marker the model actually placed.
744
- const bodyForMarkers = [lead, ...sections.map((s) => s.body)].join("\n");
745
- const used = usedMarkers(bodyForMarkers);
746
- const referenceDefs = entries.filter((e) => used.has(e.n)).map((e) => e.def);
747
-
748
- const seeAlso = deriveSeeAlso([infoboxFields.join("\n"), bodyForMarkers].join("\n"), stems, stem);
749
-
750
- const raw = assembleArticle({ stem, infoboxFields, lead, sections, seeAlso, referenceDefs });
751
-
752
- // NORMALIZE (footnote bijection + renumber, wikilink canonicalization, infobox
753
- // sort, structural guards, {{stub}} for thin), then the conservative gate.
754
- const norm = normalizeArticle(raw, { stem, stems });
755
-
756
- const base: Omit<CreateOutcome, "content" | "action" | "reason"> = {
757
- entity,
758
- stem,
759
- chunkSections,
760
- sectionAnchors: sections.map((s) => s.anchor),
761
- citedSections: countCitedSections(norm.content),
762
- hasInfobox: parseInfobox(norm.content) != null,
763
- stub: hasStubMarker(norm.content),
764
- normalizeNotes: norm.notes,
765
- };
766
-
767
- // Defect-2: every committed article MUST be footnote-bijective. NORMALIZE
768
- // already repairs (drops orphan markers, renumbers, merges dup defs); if the
769
- // result is STILL broken (an unrepairable malformed citation), veto the write
770
- // rather than commit a citation-broken article.
771
- const fn = checkFootnoteIntegrity(norm.content);
772
- if (!fn.ok) {
773
- return { ...base, content: null, action: "skipped", reason: `footnote bijection unrepairable: ${fn.errors.join("; ")}` };
774
- }
775
-
776
- // Defect-3: exactly ONE infobox — the code-constructed block. Drafted info
777
- // blocks are stripped in sanitizeSection/sanitizeLead; this asserts they were,
778
- // so a parroted second infobox can never reach the vault.
779
- const infoboxCount = countFencedInfoBlocks(norm.content);
780
- if (infoboxCount > 1) {
781
- return { ...base, content: null, action: "skipped", reason: `multiple infoboxes (${infoboxCount})` };
782
- }
783
-
784
- const verdict: GateVerdict = gateEdit(deps.before ?? "", norm.content);
785
- if (!verdict.ok) {
786
- return { ...base, content: null, action: "skipped", reason: verdict.reason };
787
- }
788
- return { ...base, content: norm.content, action: "created", reason: null };
789
- }
790
-
791
- /** `loadMetaPolicy` that never throws — a missing Meta page yields an empty
792
- * policy block so a fixture/smoke vault without seeded Meta still drafts. */
793
- function loadMetaPolicyQuiet(names: string[]): string {
794
- try {
795
- return loadMetaPolicy(names);
796
- } catch {
797
- return "";
798
- }
799
- }
800
-
801
- // ---- effectful entry: CREATE into the vault + ledger -----------------------
802
-
803
- export interface SynthesizeResult {
804
- created: boolean;
805
- stem: string;
806
- skippedByGate: boolean;
807
- reason: string | null;
808
- hasInfobox: boolean;
809
- citedSections: number;
810
- sectionAnchors: string[];
811
- }
812
-
813
- export interface SynthesizeCreateOpts {
814
- entity: string;
815
- kind?: EntityKind;
816
- /** The entity's accumulated chunk ids (the CREATE rule already fired). */
817
- chunkIds: string[];
818
- /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
819
- root?: string;
820
- /** Declared aliases for the infobox. */
821
- aliases?: string[];
822
- /** Model-call override (tests inject a fake). */
823
- call?: SynthesisCall;
824
- /** Skip the git commit (tests). */
825
- commit?: boolean;
826
- now?: number;
827
- }
828
-
829
- /** Read the chunk + its conversation metadata into a {@link SynthesisChunk}. */
830
- export function loadSynthesisChunk(store: MemoryStore, chunkId: string): SynthesisChunk | null {
831
- const row = store.db
832
- .query(
833
- "SELECT c.id, c.conv, c.label, conv.title, conv.updated_at FROM chunks c " +
834
- "JOIN conversations conv ON conv.conv = c.conv WHERE c.id = ?",
835
- )
836
- .get(chunkId) as
837
- | { id: string; conv: string; label: string | null; title: string | null; updated_at: number | null }
838
- | null;
839
- if (!row) return null;
840
- return {
841
- id: row.id,
842
- conv: row.conv,
843
- label: row.label,
844
- text: store.chunkText(chunkId),
845
- title: row.title ?? "",
846
- dateMs: row.updated_at,
847
- };
848
- }
849
-
850
- /**
851
- * CREATE an entity article from its chunks and persist it: draft via
852
- * {@link createArticle}, write the markdown to the vault (unless the gate vetoed
853
- * it), commit via `commitVault`, and record the integrated (chunk, section)
854
- * edges in `synthesized_chunk_sections`. The CREATE decision itself is upstream
855
- * (ROUTE's B8/P1-T5 gate) — this assumes the entity earned its article.
856
- */
857
- export async function synthesizeCreate(
858
- store: MemoryStore,
859
- opts: SynthesizeCreateOpts,
860
- ): Promise<SynthesizeResult> {
861
- const root = opts.root ?? vaultRoot();
862
- const stem = entityStem(opts.entity);
863
- const chunks: SynthesisChunk[] = [];
864
- for (const id of opts.chunkIds) {
865
- const c = loadSynthesisChunk(store, id);
866
- if (c) chunks.push(c);
867
- }
868
-
869
- const stems = new Set(await listArticles(root));
870
- const outcome = await createArticle(opts.entity, opts.kind ?? "thing", chunks, {
871
- call: opts.call,
872
- stems,
873
- aliases: opts.aliases,
874
- });
875
-
876
- const result: SynthesizeResult = {
877
- created: outcome.action === "created",
878
- stem,
879
- skippedByGate: outcome.action === "skipped",
880
- reason: outcome.reason,
881
- hasInfobox: outcome.hasInfobox,
882
- citedSections: outcome.citedSections,
883
- sectionAnchors: outcome.sectionAnchors,
884
- };
885
-
886
- if (outcome.action !== "created" || outcome.content == null) return result;
887
-
888
- // Write the article, then record the integration ledger + commit.
889
- const dir = articlesDir(root);
890
- await mkdir(dir, { recursive: true });
891
- const path = join(dir, `${stem}.md`);
892
- // Never clobber an existing article from the CREATE branch — that's UPDATE's job.
893
- if (await fileExists(path)) {
894
- result.created = false;
895
- result.skippedByGate = false;
896
- result.reason = "article already exists (CREATE is for new entities)";
897
- return result;
898
- }
899
- await writeFile(path, outcome.content);
900
-
901
- const now = opts.now ?? Date.now();
902
- const insSec = store.db.query(
903
- "INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
904
- );
905
- const insSynth = store.db.query(
906
- "INSERT OR REPLACE INTO synthesized_chunk_sections " +
907
- "(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
908
- );
909
- const tx = store.db.transaction((rows: { chunkId: string; anchor: string }[]) => {
910
- for (const r of rows) {
911
- insSec.run(r.chunkId, stem, r.anchor);
912
- insSynth.run(r.chunkId, stem, r.anchor, now);
913
- }
914
- });
915
- tx(outcome.chunkSections);
916
-
917
- if (opts.commit !== false) {
918
- await commitVault(root, `memory: create ${stem} (${outcome.chunkSections.length} chunk edges)`);
919
- }
920
-
921
- return result;
922
- }
923
-
924
- async function fileExists(path: string): Promise<boolean> {
925
- try {
926
- await readFile(path);
927
- return true;
928
- } catch {
929
- return false;
930
- }
931
- }
932
-
933
- // ===========================================================================
934
- // PATCH — the UPDATE branch (P7-T2): fold ONE chunk into ONE existing section.
935
- //
936
- // The steady-state counterpart to CREATE. Where CREATE drafts a whole article
937
- // from an entity's cold-start chunks, PATCH is the same bounded section-draft op
938
- // run against an article that already exists: integrate THIS chunk into THIS
939
- // (SECTION-ROUTE-chosen) section, citing it, changing nothing else. It is the
940
- // self-healing mechanism — a user's in-conversation CORRECTION is just an
941
- // ordinary chunk, so folding it rewrites the contradicted claim TOWARD the
942
- // user's latest position (resolve-to-latest, not append-both, not ignore).
943
- //
944
- // Citations stay deterministic by construction exactly as in CREATE: the chunk's
945
- // source conversation is assigned a stable `[^N]` (a re-cited conversation
946
- // reuses its existing marker; a new one mints `maxN+1`) and we GENERATE the
947
- // `[^N]:` definition line — the model only PLACES the marker. We swap only the
948
- // target section's bytes via `extractSection`/`replaceSection` (every other
949
- // section stays byte-identical) and append the def to `## References`, then run
950
- // the SAME gate (weak → NO-OP, the vault is untouched and the chunk retries next
951
- // run) + NORMALIZE as CREATE. Idempotency rides `synthesized_chunk_sections`'s PK
952
- // — a second fold of the same (chunk, article, section) writes nothing.
953
- // ===========================================================================
954
-
955
- /** The article's H1 title (depth-1 TOC heading), or a neutral fallback. */
956
- function articleH1Title(article: string): string {
957
- const h1 = parseToc(article).find((e) => e.depth === 1);
958
- return h1 ? h1.title : "this article";
959
- }
960
-
961
- /**
962
- * The article's current footnote state: the highest `[^N]` number in use
963
- * (markers ∪ definitions) and a map from each DEFINED `conv:HASH` to its number.
964
- * PATCH assigns a re-cited conversation its existing marker (no duplicate def)
965
- * and a brand-new one `maxN + 1`, so the article stays footnote-bijective and
966
- * NORMALIZE's renumber short-circuits (no shuffle of other sections). Pure.
967
- */
968
- export function articleFootnoteState(article: string): { maxN: number; byHash: Map<string, number> } {
969
- let maxN = 0;
970
- const byHash = new Map<string, number>();
971
- for (const line of article.split("\n")) {
972
- const d = /^\[\^(\d+)\]:(.*)$/.exec(line);
973
- if (d) {
974
- const n = parseInt(d[1]!, 10);
975
- if (n > maxN) maxN = n;
976
- const h = d[2]!.match(/conv:([0-9a-z]{8})/i);
977
- if (h) byHash.set(h[1]!.toLowerCase(), n);
978
- continue;
979
- }
980
- for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
981
- const n = parseInt(m[1]!, 10);
982
- if (n > maxN) maxN = n;
983
- }
984
- }
985
- return { maxN, byHash };
986
- }
987
-
988
- /**
989
- * Map each `[^N]:` definition's marker number to the `YYYY-MM-DD` (or `undated`)
990
- * date its generated def line carries — the dates we tagged via `footnoteDefLine`
991
- * (`[^n]: \`conv:HASH\` (DATE, source)`). PATCH reads this so it can tell the model
992
- * how old the claim it is correcting is, and resolve toward the LATER date. Pure.
993
- */
994
- export function referenceDatesByMarker(article: string): Map<number, string> {
995
- const out = new Map<number, string>();
996
- for (const m of article.matchAll(/^\[\^(\d+)\]:[^\n]*?\((\d{4}-\d{2}-\d{2}|undated),/gm)) {
997
- out.set(parseInt(m[1]!, 10), m[2]!);
998
- }
999
- return out;
1000
- }
1001
-
1002
- /** The distinct `[^N]` marker numbers present in a single section/body block
1003
- * (definition lines excluded), in first-appearance order. Pure. */
1004
- function markersInBlock(body: string): number[] {
1005
- const order: number[] = [];
1006
- const seen = new Set<number>();
1007
- for (const line of body.split("\n")) {
1008
- if (/^\[\^\d+\]:/.test(line.trim())) continue;
1009
- for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
1010
- const n = parseInt(m[1]!, 10);
1011
- if (!seen.has(n)) {
1012
- seen.add(n);
1013
- order.push(n);
1014
- }
1015
- }
1016
- }
1017
- return order;
1018
- }
1019
-
1020
- /**
1021
- * Swap exactly the `anchor` section's block for `newBlock`, leaving every byte
1022
- * OUTSIDE that section identical (the byte-preservation invariant PATCH and the
1023
- * gate depend on). Returns null when the section is absent. Pure.
1024
- */
1025
- export function replaceSection(article: string, anchor: string, newBlock: string): string | null {
1026
- const block = extractSection(article, anchor);
1027
- if (block == null) return null;
1028
- const idx = article.indexOf(block);
1029
- if (idx < 0) return null;
1030
- return article.slice(0, idx) + newBlock + article.slice(idx + block.length);
1031
- }
1032
-
1033
- /**
1034
- * Insert a generated `[^N]:` definition line into the article's `## References`
1035
- * section (after the last existing def), creating the section at the end when
1036
- * the article has none. The model never writes defs — this is the only path that
1037
- * does. Pure.
1038
- */
1039
- export function appendReferenceDef(article: string, defLine: string): string {
1040
- const refs = extractSection(article, "references");
1041
- if (refs == null) {
1042
- return `${article.replace(/\s+$/, "")}\n\n## References\n\n${defLine}\n`;
1043
- }
1044
- const newRefs = `${refs.replace(/\s+$/, "")}\n${defLine}\n`;
1045
- return replaceSection(article, "references", newRefs) ?? article;
1046
- }
1047
-
1048
- /** Retry hint when a patched section leaks meta/planning or fails to cite. */
1049
- const PATCH_ANTI_LEAK_HINT =
1050
- "Output ONLY the integrated section prose — no preamble, no meta-commentary, no plan, no restating the task.";
1051
-
1052
- /**
1053
- * Draft the integrated section body, rejecting leakage AND an uncited fold:
1054
- * draft once; if the sanitized body is {@link isLeakyDraft} OR is missing the
1055
- * assigned `[^marker]` (the integration didn't actually place the citation),
1056
- * retry ONCE with a sharper instruction; if it still fails, return "" so the
1057
- * caller NO-OPs (the vault is untouched; the chunk retries next run).
1058
- */
1059
- async function draftPatchedProse(
1060
- call: SynthesisCall,
1061
- prompt: string,
1062
- marker: number,
1063
- sanitize: (raw: string) => string,
1064
- ): Promise<string> {
1065
- const wants = new RegExp(`\\[\\^${marker}\\](?!:)`);
1066
- const clean = (raw: string): string => {
1067
- const body = sanitize(raw);
1068
- return !isLeakyDraft(body) && wants.test(body) ? body : "";
1069
- };
1070
- const first = clean(await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS }));
1071
- if (first) return first;
1072
- return clean(await call(`${prompt}\n\n${PATCH_ANTI_LEAK_HINT} Place the [^${marker}] marker for the new note.`, { maxTokens: MAX_OUTPUT_TOKENS }));
1073
- }
1074
-
1075
- /** Draft an integrated SECTION body (sanitized as a section, must place the marker). */
1076
- function draftPatchedSection(
1077
- call: SynthesisCall,
1078
- prompt: string,
1079
- marker: number,
1080
- heading: string,
1081
- ): Promise<string> {
1082
- return draftPatchedProse(call, prompt, marker, (raw) => sanitizeSection(raw, heading));
1083
- }
1084
-
1085
- /** The schematic PATCH prompt: current section markdown + the chunk + the inlined
1086
- * footnote/correction contract. Kept free of copyable example values per the
1087
- * small-model parroting failure mode. */
1088
- export function buildPatchPrompt(
1089
- title: string,
1090
- heading: string,
1091
- currentBody: string,
1092
- chunk: SynthesisChunk,
1093
- n: number,
1094
- otherArticles: string,
1095
- policy: string,
1096
- priorDates: string[] = [],
1097
- ): string {
1098
- const chunkDate = footnoteDate(chunk.dateMs);
1099
- return (
1100
- `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
1101
- `You are integrating ONE new note (dated ${chunkDate}) into the "${heading}" section of the personal-wiki article about "${title}". ` +
1102
- `Keep the existing prose and every existing [^k] citation marker intact; weave in only what the new note adds.\n\n` +
1103
- `CITATION — cite the new note by placing the marker [^${n}] immediately after the specific claim it supports. ` +
1104
- `Use [^${n}] for the new note, and reuse any [^k] markers already in the section exactly where they are. ` +
1105
- `Do NOT write any [^k]: definition line — definitions are generated for you. ` +
1106
- `[^N] is the only citation syntax; never emit HTML, bare links, or invent another number.\n\n` +
1107
- correctionRule(n, chunkDate, priorDates) +
1108
- `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
1109
- `CURRENT "${heading}" SECTION:\n${currentBody || "(empty)"}\n\n` +
1110
- `NEW NOTE (${chunkDate})${chunk.label ? ` — ${chunk.label}` : ""}:\n${chunk.text.slice(0, 1400).trim()}\n\n` +
1111
- `Output ONLY the rewritten section body prose — do NOT repeat the "## ${heading}" heading, do NOT add other sections, do NOT write a References section.`
1112
- );
1113
- }
1114
-
1115
- /**
1116
- * The PROVENANCE-PRESERVING correction rule (schematic). A correction resolves
1117
- * toward the LATER-DATED statement so the latest position wins — but it KEEPS the
1118
- * superseded claim's existing `[^k]` marker alongside the new `[^N]`, phrased as a
1119
- * change over time, so no citation is lost. This is what lets the resolve fold pass
1120
- * the gate's citation-survival check instead of being NO-OP'd for dropping `[^k]`.
1121
- */
1122
- function correctionRule(n: number, chunkDate: string, priorDates: string[]): string {
1123
- const prior = priorDates.filter((d) => d && d !== "undated");
1124
- const priorClause = prior.length
1125
- ? `The claim it corrects is dated ${[...new Set(prior)].join(" / ")}, so the new note (dated ${chunkDate}) is LATER. `
1126
- : `The new note is dated ${chunkDate}. `;
1127
- return (
1128
- `CORRECTION — if the new note CONTRADICTS a claim already in the section, this is a correction: resolve toward the LATER-DATED statement so the user's CURRENT position wins. ` +
1129
- priorClause +
1130
- `DEMOTE the superseded claim to the PAST TENSE — rewrite the OLD clause itself from present ("is my favourite", "currently", "the one I reach for") to past ("previously was my favourite", "originally", "used to be") — do NOT leave the old value asserted as still-current and merely append the new one. ` +
1131
- `KEEP the superseded claim's existing [^k] citation marker in place ALONGSIDE the new note's [^${n}] marker, so no [^N] is lost — phrase it like "previously <earlier position>[^k]; as of <later date> <current position> is now my pick[^${n}]". ` +
1132
- `Do NOT assert both as simultaneously true, and do NOT ignore the new note.\n\n`
1133
- );
1134
- }
1135
-
1136
- /**
1137
- * The CLASSIFIED LEAD correction rule (schematic). The lead must read as a CLEAN
1138
- * current-state summary — never scattered "previously …" caveats (RULE 1). So when
1139
- * a contradiction is classified:
1140
- * - EVOLUTION (the user changed their own position) → resolve the lead to the
1141
- * CURRENT value in the present tense; KEEP the superseded claim's `[^k]` marker
1142
- * so provenance survives the gate (the full trajectory is consolidated into a
1143
- * `## History` section by the reconcile pass), but do NOT narrate the change in
1144
- * the lead itself.
1145
- * - ERROR (a wrong fact being corrected) → SILENTLY replace the wrong value with
1146
- * the correct one; the wrong value's `[^k]` citation MAY be dropped (it is
1147
- * pruned as if it had never been there).
1148
- */
1149
- function leadCorrectionRule(kind: ContradictionKind, n: number, chunkDate: string): string {
1150
- if (kind === "error") {
1151
- return (
1152
- `CORRECTION (factual error) — the new note (dated ${chunkDate}) corrects a WRONG fact already in the lead. ` +
1153
- `Silently REPLACE the wrong value with the correct one and cite it [^${n}]; state ONLY the correct value. ` +
1154
- `Do NOT mention the wrong value, do NOT write "previously" — it was never a real position. ` +
1155
- `You MAY drop the wrong value's [^k] citation marker.\n\n`
1156
- );
1157
- }
1158
- return (
1159
- `CORRECTION (the user's view evolved) — the new note (dated ${chunkDate}) supersedes an earlier position in the lead. ` +
1160
- `Resolve the lead to the user's CURRENT position in the PRESENT TENSE and cite it [^${n}]. ` +
1161
- `Keep it a clean current-state summary: do NOT scatter "previously …" caveats through the lead — the trajectory is recorded separately in a History section. ` +
1162
- `KEEP the superseded claim's existing [^k] citation marker somewhere in the lead so no [^N] is lost (provenance survives), but frame the lead around the current value, not the change.\n\n`
1163
- );
1164
- }
1165
-
1166
- /** The schematic LEAD-PATCH prompt: the article's current lead + the chunk + the
1167
- * inlined footnote/correction contract. The lead is where an entity's headline
1168
- * verdict lives, so the CORRECTION rule is the load-bearing one. When a `kind` is
1169
- * classified, the lead resolves to a CLEAN current-state summary (evolution) or a
1170
- * silent overwrite (error); otherwise the default provenance-preserving fold runs. */
1171
- export function buildLeadPatchPrompt(
1172
- title: string,
1173
- currentLead: string,
1174
- chunk: SynthesisChunk,
1175
- n: number,
1176
- otherArticles: string,
1177
- policy: string,
1178
- priorDates: string[] = [],
1179
- kind?: ContradictionKind,
1180
- ): string {
1181
- const chunkDate = footnoteDate(chunk.dateMs);
1182
- return (
1183
- `${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
1184
- `You are updating the LEAD (the opening summary) of the personal-wiki article about "${title}" with ONE new note (dated ${chunkDate}). ` +
1185
- `The lead names **${title}** in bold and states what it is and how the user relates to it. ` +
1186
- `Keep every existing [^k] citation marker that still applies; weave in only what the new note adds.\n\n` +
1187
- `CITATION — cite the new note by placing the marker [^${n}] immediately after the specific claim it supports. ` +
1188
- `Reuse any [^k] markers already in the lead exactly where they are. ` +
1189
- `Do NOT write any [^k]: definition line — definitions are generated for you. ` +
1190
- `[^N] is the only citation syntax; never emit HTML, bare links, or invent another number.\n\n` +
1191
- (kind ? leadCorrectionRule(kind, n, chunkDate) : correctionRule(n, chunkDate, priorDates)) +
1192
- `OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
1193
- `CURRENT LEAD:\n${currentLead || "(empty)"}\n\n` +
1194
- `NEW NOTE (${chunkDate})${chunk.label ? ` — ${chunk.label}` : ""}:\n${chunk.text.slice(0, 1400).trim()}\n\n` +
1195
- `Output ONLY the rewritten lead prose — 2–4 sentences, no heading, no list, no References section.`
1196
- );
1197
- }
1198
-
1199
- export interface PatchInput {
1200
- /** The full current article markdown. */
1201
- article: string;
1202
- /** The target section's anchor (a `chunk_sections` row's section_anchor), or
1203
- * {@link LEAD_ANCHOR} to fold into the article's opening lead. */
1204
- anchor: string;
1205
- /** The chunk to integrate (text + footnote metadata). */
1206
- chunk: SynthesisChunk;
1207
- }
1208
-
1209
- export interface PatchDeps {
1210
- /** Model call; defaults to `callLocal("synthesis", …)` (same stage as CREATE). */
1211
- call?: SynthesisCall;
1212
- /** Other article stems for wikilink resolution. */
1213
- stems?: Set<string>;
1214
- /** Article stem (H1/NORMALIZE); defaults to the entity stem of the H1 title. */
1215
- stem?: string;
1216
- }
1217
-
1218
- export interface PatchOutcome {
1219
- anchor: string;
1220
- /** Final post-NORMALIZE article, or null when the patch was a NO-OP. */
1221
- content: string | null;
1222
- action: "patched" | "skipped";
1223
- /** Skip reason; null on success. */
1224
- reason: string | null;
1225
- /** The `[^N]` number assigned to the chunk (null when never reached). */
1226
- footnote: number | null;
1227
- /** True when the chunk's conversation was already cited (reused marker, no new def). */
1228
- reusedFootnote: boolean;
1229
- /** True when a new `[^N]:` definition line was appended. */
1230
- addedDef: boolean;
1231
- normalizeNotes: string[];
1232
- }
1233
-
1234
- /**
1235
- * Fold ONE chunk into ONE existing section of an article, end to end:
1236
- * assign the chunk's `[^N]` → draft the integrated body (leak/uncited → retry →
1237
- * NO-OP) → swap ONLY that section's bytes → append the generated `[^N]:` def →
1238
- * NORMALIZE → gate. Returns the patched article (or null when the gate vetoes a
1239
- * weak pass). Pure with respect to disk — the caller persists.
1240
- */
1241
- export async function patchSection(input: PatchInput, deps: PatchDeps = {}): Promise<PatchOutcome> {
1242
- const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
1243
- const { article, anchor, chunk } = input;
1244
- // The LEAD is patched by swapping exactly the lead bytes (no `## heading`).
1245
- if (anchor === LEAD_ANCHOR) return patchLead(article, chunk, deps, call);
1246
- const skip = (reason: string, footnote: number | null = null, reusedFootnote = false, addedDef = false): PatchOutcome => ({
1247
- anchor,
1248
- content: null,
1249
- action: "skipped",
1250
- reason,
1251
- footnote,
1252
- reusedFootnote,
1253
- addedDef,
1254
- normalizeNotes: [],
1255
- });
1256
-
1257
- const block = extractSection(article, anchor);
1258
- if (block == null) return skip(`section not found: ${anchor}`);
1259
- const headingLine = block.split("\n")[0] ?? `## ${anchor}`;
1260
- const heading = headingLine.replace(/^#{1,6}\s+/, "").trim();
1261
- const currentBody = block.split("\n").slice(1).join("\n").trim();
1262
-
1263
- const title = articleH1Title(article);
1264
- const stem = deps.stem ?? entityStem(title);
1265
- const stems = new Set(deps.stems ?? []);
1266
- stems.add(stem);
1267
- const otherArticles = otherArticlesBlock(stems, stem);
1268
- const policy = loadMetaPolicyQuiet(["Article_Conventions"]);
1269
-
1270
- // Assign the chunk's footnote: reuse the conversation's existing marker, else
1271
- // mint maxN+1 (keeps the article bijective+contiguous → NORMALIZE no-ops).
1272
- const state = articleFootnoteState(article);
1273
- const hash = convHash(chunk.conv);
1274
- const reused = state.byHash.get(hash);
1275
- const n = reused ?? state.maxN + 1;
1276
- const addedDef = reused === undefined;
1277
-
1278
- // The dates of the claims this section already cites — so a contradicting fold
1279
- // can be told it is the LATER statement and resolve toward itself (provenance
1280
- // preserved: it keeps the old [^k] marker).
1281
- const refDates = referenceDatesByMarker(article);
1282
- const priorDates = markersInBlock(currentBody)
1283
- .map((m) => refDates.get(m))
1284
- .filter((d): d is string => d != null);
1285
-
1286
- const newBody = await draftPatchedSection(
1287
- call,
1288
- buildPatchPrompt(title, heading, currentBody, chunk, n, otherArticles, policy, priorDates),
1289
- n,
1290
- heading,
1291
- );
1292
- // Empty after one retry = leaky/uncited → NO-OP (vault untouched, chunk retries).
1293
- if (!newBody.trim()) return skip("patched section leaked, refused, or did not cite the new note (NO-OP)", n, reused !== undefined, addedDef);
1294
-
1295
- let next = replaceSection(article, anchor, `${headingLine}\n\n${newBody}\n`);
1296
- if (next == null) return skip(`section vanished on swap: ${anchor}`, n, reused !== undefined, addedDef);
1297
- if (addedDef) next = appendReferenceDef(next, footnoteDefLine(n, chunk.conv, chunk.title, chunk.dateMs));
1298
-
1299
- const norm = normalizeArticle(next, { stem, stems });
1300
-
1301
- // Footnote bijection must hold (a hallucinated marker number the gate cannot
1302
- // repair → NO-OP rather than a citation-broken commit).
1303
- const fn = checkFootnoteIntegrity(norm.content);
1304
- if (!fn.ok) return skip(`footnote bijection unrepairable: ${fn.errors.join("; ")}`, n, reused !== undefined, addedDef);
1305
-
1306
- // The conservative gate over (before = original article, after = patched).
1307
- const verdict: GateVerdict = gateEdit(article, norm.content);
1308
- if (!verdict.ok) return skip(verdict.reason ?? "gate veto", n, reused !== undefined, addedDef);
1309
-
1310
- return {
1311
- anchor,
1312
- content: norm.content,
1313
- action: "patched",
1314
- reason: null,
1315
- footnote: n,
1316
- reusedFootnote: reused !== undefined,
1317
- addedDef,
1318
- normalizeNotes: norm.notes,
1319
- };
1320
- }
1321
-
1322
- /**
1323
- * Fold ONE chunk into the article's LEAD (its opening summary), swapping exactly
1324
- * the lead bytes — every section and the References stay byte-identical except for
1325
- * the appended def. Same assign-footnote → draft (leak/uncited → retry → NO-OP) →
1326
- * NORMALIZE → gate contract as {@link patchSection}, but the lead has no `##`
1327
- * heading so we locate it via the structural skeleton and replace those lines.
1328
- */
1329
- async function patchLead(
1330
- article: string,
1331
- chunk: SynthesisChunk,
1332
- deps: PatchDeps,
1333
- call: SynthesisCall,
1334
- ): Promise<PatchOutcome> {
1335
- const skip = (reason: string, footnote: number | null = null, reusedFootnote = false, addedDef = false): PatchOutcome => ({
1336
- anchor: LEAD_ANCHOR,
1337
- content: null,
1338
- action: "skipped",
1339
- reason,
1340
- footnote,
1341
- reusedFootnote,
1342
- addedDef,
1343
- normalizeNotes: [],
1344
- });
1345
-
1346
- const leadItem = articleStructure(article).find((it) => it.kind === "lead");
1347
- if (!leadItem) return skip("article has no lead to patch");
1348
- const lines = article.split("\n");
1349
- const start = leadItem.startLine - 1;
1350
- const end = leadItem.endLine - 1; // 1-based inclusive → 0-based inclusive
1351
- const currentLead = lines.slice(start, end + 1).join("\n").trim();
1352
-
1353
- const title = articleH1Title(article);
1354
- const stem = deps.stem ?? entityStem(title);
1355
- const stems = new Set(deps.stems ?? []);
1356
- stems.add(stem);
1357
- const otherArticles = otherArticlesBlock(stems, stem);
1358
- const policy = loadMetaPolicyQuiet(["Article_Conventions"]);
1359
-
1360
- // Assign the chunk's footnote exactly as the section path does.
1361
- const state = articleFootnoteState(article);
1362
- const hash = convHash(chunk.conv);
1363
- const reused = state.byHash.get(hash);
1364
- const n = reused ?? state.maxN + 1;
1365
- const addedDef = reused === undefined;
1366
-
1367
- const refDates = referenceDatesByMarker(article);
1368
- const priorMarkers = markersInBlock(currentLead);
1369
- const priorDates = priorMarkers
1370
- .map((m) => refDates.get(m))
1371
- .filter((d): d is string => d != null);
1372
-
1373
- // CLASSIFY the contradiction when the lead already cites a prior claim the new
1374
- // note may conflict with: is the user CHANGING THEIR OWN view (evolution — the
1375
- // lead resolves to current-state, the trajectory preserved in History by the
1376
- // reconcile pass) or correcting a WRONG fact (error — silent overwrite, the
1377
- // wrong value's citation may be pruned)?
1378
- let kind: ContradictionKind | undefined;
1379
- const allowDroppedHashes = new Set<string>();
1380
- if (priorMarkers.length) {
1381
- kind = classifyContradiction(
1382
- await call(buildContradictionClassifierPrompt(currentLead, chunk.text.slice(0, 1400).trim()), { maxTokens: MAX_OUTPUT_TOKENS }),
1383
- );
1384
- if (kind === "error") {
1385
- // A factual fix may drop the wrong value's citation. Permit dropping the
1386
- // hashes the corrected lead cited (bounded to the lead's own prior claims).
1387
- const hashByMarker = new Map<number, string>();
1388
- for (const [h, m] of state.byHash) hashByMarker.set(m, h);
1389
- for (const m of priorMarkers) {
1390
- const h = hashByMarker.get(m);
1391
- if (h) allowDroppedHashes.add(h);
1392
- }
1393
- }
1394
- }
1395
-
1396
- const newLead = await draftPatchedProse(
1397
- call,
1398
- buildLeadPatchPrompt(title, currentLead, chunk, n, otherArticles, policy, priorDates, kind),
1399
- n,
1400
- sanitizeLead,
1401
- );
1402
- if (!newLead.trim()) return skip("patched lead leaked, refused, or did not cite the new note (NO-OP)", n, reused !== undefined, addedDef);
1403
-
1404
- // Swap exactly the lead lines; every other byte is preserved.
1405
- let next = [...lines.slice(0, start), newLead, ...lines.slice(end + 1)].join("\n");
1406
- if (addedDef) next = appendReferenceDef(next, footnoteDefLine(n, chunk.conv, chunk.title, chunk.dateMs));
1407
-
1408
- const norm = normalizeArticle(next, { stem, stems });
1409
-
1410
- const fn = checkFootnoteIntegrity(norm.content);
1411
- if (!fn.ok) return skip(`footnote bijection unrepairable: ${fn.errors.join("; ")}`, n, reused !== undefined, addedDef);
1412
-
1413
- const verdict: GateVerdict = gateEdit(article, norm.content, { allowDroppedHashes });
1414
- if (!verdict.ok) return skip(verdict.reason ?? "gate veto", n, reused !== undefined, addedDef);
1415
-
1416
- return {
1417
- anchor: LEAD_ANCHOR,
1418
- content: norm.content,
1419
- action: "patched",
1420
- reason: null,
1421
- footnote: n,
1422
- reusedFootnote: reused !== undefined,
1423
- addedDef,
1424
- normalizeNotes: norm.notes,
1425
- };
1426
- }
1427
-
1428
- // ---- effectful entry: PATCH a vault article + ledger -----------------------
1429
-
1430
- export interface SynthesizePatchOpts {
1431
- /** The article to patch (its filename stem). */
1432
- stem: string;
1433
- /** The section anchor to fold into (a `chunk_sections` row). */
1434
- anchor: string;
1435
- /** The chunk id to integrate. */
1436
- chunkId: string;
1437
- /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
1438
- root?: string;
1439
- /** Model-call override (tests inject a fake). */
1440
- call?: SynthesisCall;
1441
- /** Skip the git commit (tests). */
1442
- commit?: boolean;
1443
- now?: number;
1444
- }
1445
-
1446
- export interface PatchResult {
1447
- patched: boolean;
1448
- stem: string;
1449
- anchor: string;
1450
- /** The gate (or a draft NO-OP) vetoed the write. */
1451
- skippedByGate: boolean;
1452
- /** The (chunk, article, section) edge was already folded (idempotent NO-OP). */
1453
- alreadyIntegrated: boolean;
1454
- reason: string | null;
1455
- footnote: number | null;
1456
- }
1457
-
1458
- /**
1459
- * PATCH one chunk into one section of a vault article and persist it. Idempotent
1460
- * BEFORE the model: a present `synthesized_chunk_sections` PK row short-circuits
1461
- * to a NO-OP (no GPU, no write). Otherwise drafts via {@link patchSection}, writes
1462
- * the swapped article (unless the gate vetoed), records the integration edge, and
1463
- * commits.
1464
- */
1465
- export async function synthesizePatch(store: MemoryStore, opts: SynthesizePatchOpts): Promise<PatchResult> {
1466
- const root = opts.root ?? vaultRoot();
1467
- const base: PatchResult = {
1468
- patched: false,
1469
- stem: opts.stem,
1470
- anchor: opts.anchor,
1471
- skippedByGate: false,
1472
- alreadyIntegrated: false,
1473
- reason: null,
1474
- footnote: null,
1475
- };
1476
-
1477
- // Idempotency: the (chunk, article, section) PK blocks a double-fold up front.
1478
- const already = store.db
1479
- .query("SELECT 1 FROM synthesized_chunk_sections WHERE chunk_id = ? AND article_stem = ? AND section_anchor = ?")
1480
- .get(opts.chunkId, opts.stem, opts.anchor);
1481
- if (already) return { ...base, alreadyIntegrated: true, reason: "already integrated (synthesized_chunk_sections PK)" };
1482
-
1483
- const path = join(articlesDir(root), `${opts.stem}.md`);
1484
- let article: string;
1485
- try {
1486
- article = await readFile(path, "utf8");
1487
- } catch {
1488
- return { ...base, reason: `article not found: ${opts.stem}` };
1489
- }
1490
-
1491
- const chunk = loadSynthesisChunk(store, opts.chunkId);
1492
- if (!chunk) return { ...base, reason: `chunk not found: ${opts.chunkId}` };
1493
-
1494
- const stems = new Set(await listArticles(root));
1495
- const outcome = await patchSection({ article, anchor: opts.anchor, chunk }, { call: opts.call, stems, stem: opts.stem });
1496
-
1497
- if (outcome.action !== "patched" || outcome.content == null) {
1498
- return { ...base, skippedByGate: true, reason: outcome.reason, footnote: outcome.footnote };
1499
- }
1500
-
1501
- await writeFile(path, outcome.content);
1502
-
1503
- const now = opts.now ?? Date.now();
1504
- const insSec = store.db.query(
1505
- "INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
1506
- );
1507
- const insSynth = store.db.query(
1508
- "INSERT OR REPLACE INTO synthesized_chunk_sections " +
1509
- "(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
1510
- );
1511
- const tx = store.db.transaction(() => {
1512
- insSec.run(opts.chunkId, opts.stem, opts.anchor);
1513
- insSynth.run(opts.chunkId, opts.stem, opts.anchor, now);
1514
- });
1515
- tx();
1516
-
1517
- if (opts.commit !== false) {
1518
- await commitVault(root, `memory: patch ${opts.stem} #${opts.anchor} (+chunk ${opts.chunkId})`);
1519
- }
1520
-
1521
- return { ...base, patched: true, footnote: outcome.footnote };
1522
- }
1523
-
1524
- // ===========================================================================
1525
- // NEW SECTION — the third SECTION-ROUTE outcome (P7): a chunk that fits NO
1526
- // existing section AND is not a verdict-level correction earns its OWN section.
1527
- //
1528
- // SECTION-ROUTE returns a NAMED new section for a substantive chunk that matched
1529
- // nothing; honoring it (rather than dropping the chunk) is the difference between
1530
- // self-healing and silent loss. We mint an EMPTY `## Title` section, then run the
1531
- // SAME bounded fold as PATCH against it — so the new section is drafted, cited,
1532
- // gated, and normalized exactly like every other integration, never a raw dump.
1533
- // ===========================================================================
1534
-
1535
- /** Insert an empty `## title` section into the article just before its `## See
1536
- * also` / `## References` tail (or at the end when neither exists), so a fresh
1537
- * topical section lands among the body sections, not after the references. The
1538
- * body is left empty — {@link patchSection} fills it by folding the chunk in. */
1539
- export function insertEmptySection(article: string, title: string): string {
1540
- const lines = article.replace(/\s+$/, "").split("\n");
1541
- const idx = lines.findIndex((l) => /^##\s+(see also|references)\s*$/i.test(l.trim()));
1542
- if (idx < 0) return [...lines, "", `## ${title}`, ""].join("\n") + "\n";
1543
- return [...lines.slice(0, idx), `## ${title}`, "", "", ...lines.slice(idx)].join("\n") + "\n";
1544
- }
1545
-
1546
- export interface SynthesizeNewSectionOpts {
1547
- /** The article to grow (its filename stem). */
1548
- stem: string;
1549
- /** The new section's display title. */
1550
- title: string;
1551
- /** The new section's anchor (slug of `title`); also the ledger key. */
1552
- anchor: string;
1553
- /** The chunk id to fold into the new section. */
1554
- chunkId: string;
1555
- /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
1556
- root?: string;
1557
- /** Model-call override (tests inject a fake). */
1558
- call?: SynthesisCall;
1559
- /** Skip the git commit (tests). */
1560
- commit?: boolean;
1561
- now?: number;
1562
- }
1563
-
1564
- /**
1565
- * CREATE a new section in an existing article and fold ONE chunk into it. Mints
1566
- * the `## Title` heading then drafts its body via {@link patchSection} (so the
1567
- * fold is cited, gated, NORMALIZED, and footnote-bijective just like a PATCH),
1568
- * writes the grown article, and records the (chunk, section) edge. Idempotent via
1569
- * `synthesized_chunk_sections`'s PK — a re-run of the same (chunk, article,
1570
- * section) writes nothing. Used by SYNTHESIZE when SECTION-ROUTE returns a
1571
- * `newSection` for a chunk that fit no existing section.
1572
- */
1573
- export async function synthesizeNewSection(
1574
- store: MemoryStore,
1575
- opts: SynthesizeNewSectionOpts,
1576
- ): Promise<PatchResult> {
1577
- const root = opts.root ?? vaultRoot();
1578
- const base: PatchResult = {
1579
- patched: false,
1580
- stem: opts.stem,
1581
- anchor: opts.anchor,
1582
- skippedByGate: false,
1583
- alreadyIntegrated: false,
1584
- reason: null,
1585
- footnote: null,
1586
- };
1587
-
1588
- const already = store.db
1589
- .query("SELECT 1 FROM synthesized_chunk_sections WHERE chunk_id = ? AND article_stem = ? AND section_anchor = ?")
1590
- .get(opts.chunkId, opts.stem, opts.anchor);
1591
- if (already) return { ...base, alreadyIntegrated: true, reason: "already integrated (synthesized_chunk_sections PK)" };
1592
-
1593
- const path = join(articlesDir(root), `${opts.stem}.md`);
1594
- let article: string;
1595
- try {
1596
- article = await readFile(path, "utf8");
1597
- } catch {
1598
- return { ...base, reason: `article not found: ${opts.stem}` };
1599
- }
1600
-
1601
- const chunk = loadSynthesisChunk(store, opts.chunkId);
1602
- if (!chunk) return { ...base, reason: `chunk not found: ${opts.chunkId}` };
1603
-
1604
- // Mint the empty section unless the article already carries it (idempotent
1605
- // across the same name), then fold the chunk into it via the PATCH op.
1606
- const withSection = extractSection(article, opts.anchor) != null ? article : insertEmptySection(article, opts.title);
1607
- const stems = new Set(await listArticles(root));
1608
- const outcome = await patchSection(
1609
- { article: withSection, anchor: opts.anchor, chunk },
1610
- { call: opts.call, stems, stem: opts.stem },
1611
- );
1612
-
1613
- if (outcome.action !== "patched" || outcome.content == null) {
1614
- return { ...base, skippedByGate: true, reason: outcome.reason, footnote: outcome.footnote };
1615
- }
1616
-
1617
- await writeFile(path, outcome.content);
1618
-
1619
- const now = opts.now ?? Date.now();
1620
- const insSec = store.db.query(
1621
- "INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
1622
- );
1623
- const insSynth = store.db.query(
1624
- "INSERT OR REPLACE INTO synthesized_chunk_sections " +
1625
- "(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
1626
- );
1627
- const tx = store.db.transaction(() => {
1628
- insSec.run(opts.chunkId, opts.stem, opts.anchor);
1629
- insSynth.run(opts.chunkId, opts.stem, opts.anchor, now);
1630
- });
1631
- tx();
1632
-
1633
- if (opts.commit !== false) {
1634
- await commitVault(root, `memory: new section ${opts.stem} #${opts.anchor} (+chunk ${opts.chunkId})`);
1635
- }
1636
-
1637
- return { ...base, patched: true, footnote: outcome.footnote };
1638
- }