mlx-bun 0.4.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (601) hide show
  1. package/README.md +1453 -88
  2. package/THIRD_PARTY_NOTICES.md +882 -0
  3. package/bin/mlx-bun.convert.mjs +4 -0
  4. package/bin/mlx-bun.fuse.mjs +4 -0
  5. package/bin/mlx-bun.generate.mjs +4 -0
  6. package/bin/mlx-bun.lora.mjs +4 -0
  7. package/bin/mlx-bun.mjs +13 -21
  8. package/bin/mlx-bun.server.mjs +4 -0
  9. package/bin/mlx-bun.upload.mjs +4 -0
  10. package/dist/web/app.js +9584 -0
  11. package/package.json +50 -53
  12. package/src/cli/ane-install.ts +76 -0
  13. package/src/cli/args.ts +179 -0
  14. package/src/cli/engine-entry.ts +68 -0
  15. package/src/cli/inference.ts +214 -0
  16. package/src/cli/job-entry.ts +97 -0
  17. package/src/cli/library-host.ts +117 -0
  18. package/src/cli/main.ts +57 -0
  19. package/src/cli/memory-engine.ts +163 -0
  20. package/src/cli/memory.ts +36 -0
  21. package/src/cli/mlx-lm-aliases.ts +309 -0
  22. package/src/cli/model-choice.ts +33 -0
  23. package/src/cli/model-selection.ts +84 -0
  24. package/src/cli/module-host.ts +11 -0
  25. package/src/cli/module-verbs.ts +50 -0
  26. package/src/cli/numerical-policy.ts +12 -0
  27. package/src/cli/perplexity.ts +89 -0
  28. package/src/cli/serve-host.ts +423 -0
  29. package/src/cli/serve-isolated.ts +217 -0
  30. package/src/cli/serve-options.ts +106 -0
  31. package/src/cli/serve-state.ts +205 -0
  32. package/src/cli/serve.ts +500 -0
  33. package/src/cli/served-model-host.ts +102 -0
  34. package/src/cli/server-entry.ts +148 -0
  35. package/src/cli/serving-unit.ts +177 -0
  36. package/src/{tui.ts → cli/terminal.ts} +13 -8
  37. package/src/cli/worker-adapters.ts +34 -0
  38. package/src/cli/worker-entry.ts +272 -0
  39. package/src/cli/worker-unit.ts +179 -0
  40. package/src/engine/adapter-operation.ts +26 -0
  41. package/src/engine/cache-services.ts +114 -0
  42. package/src/engine/completion.ts +39 -0
  43. package/src/engine/generation-gateway.ts +358 -0
  44. package/src/engine/index.ts +59 -0
  45. package/src/engine/kv-budget.ts +120 -0
  46. package/src/engine/media-preparation.ts +232 -0
  47. package/src/engine/model-binding.ts +24 -0
  48. package/src/engine/model-host.ts +513 -0
  49. package/src/engine/model-serving.ts +89 -0
  50. package/src/{serve → engine}/preparation.ts +10 -8
  51. package/src/{serve → engine}/session-completion-engine.ts +6 -6
  52. package/src/engine/telemetry.ts +109 -0
  53. package/src/hub/downloads.ts +116 -0
  54. package/src/jobs/db.ts +16 -7
  55. package/src/jobs/events.ts +29 -0
  56. package/src/jobs/executable.ts +5 -0
  57. package/src/jobs/host.ts +51 -0
  58. package/src/jobs/in-process.ts +55 -0
  59. package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
  60. package/src/jobs/runner.ts +150 -231
  61. package/src/jobs/service.ts +75 -0
  62. package/src/jobs/sse.ts +59 -23
  63. package/src/jobs/worker-memory.ts +46 -0
  64. package/src/jobs/worker-process.ts +163 -0
  65. package/src/jobs/worker-supervisor.ts +258 -0
  66. package/src/modules.ts +58 -0
  67. package/src/publishing/catalog-hub.ts +18 -0
  68. package/src/publishing/credentials.ts +31 -0
  69. package/src/publishing/upload.ts +37 -0
  70. package/src/residency/leased-adapters.ts +21 -0
  71. package/src/residency/local-records.ts +16 -0
  72. package/src/residency/model-residency.ts +405 -0
  73. package/src/residency/record-index.ts +28 -0
  74. package/src/residency/resident-estimate.ts +50 -0
  75. package/src/{anthropic.ts → server/anthropic.ts} +3 -81
  76. package/src/server/cache-routes.ts +36 -0
  77. package/src/{serve → server}/chat-request.ts +12 -10
  78. package/src/{serve → server}/chat-stage.ts +27 -22
  79. package/src/{client.ts → server/client.ts} +28 -2
  80. package/src/{serve → server}/completion-executor.ts +42 -42
  81. package/src/server/decision-body.ts +55 -0
  82. package/src/server/decision-routes.ts +134 -0
  83. package/src/server/decision-wire.ts +228 -0
  84. package/src/{serve → server}/discovery-routes.ts +42 -83
  85. package/src/server/generated-token-history.ts +57 -0
  86. package/src/{serve → server}/http.ts +25 -13
  87. package/src/{serve → server}/inference-request.ts +5 -5
  88. package/src/server/job-routes.ts +20 -0
  89. package/src/server/media-prompt.ts +63 -0
  90. package/src/server/memory-completion-client.ts +46 -0
  91. package/src/server/model-routes.ts +192 -0
  92. package/src/{serve → server}/openai-wire.ts +3 -2
  93. package/src/{serve → server}/pipeline.ts +1 -1
  94. package/src/server/prompt-builder.ts +13 -0
  95. package/src/server/prompt-contracts.ts +17 -0
  96. package/src/server/proxy-routes.ts +187 -0
  97. package/src/server/publishing-routes.ts +41 -0
  98. package/src/{serve → server}/request-plan.ts +8 -8
  99. package/src/{serve → server}/request-prep.ts +82 -68
  100. package/src/{serve → server}/responses-client.ts +30 -13
  101. package/src/{responses.ts → server/responses.ts} +12 -93
  102. package/src/server/routes.ts +205 -0
  103. package/src/server/start.ts +72 -0
  104. package/src/server/status-routes.ts +108 -0
  105. package/src/{serve → server}/text-completion-stage.ts +25 -4
  106. package/src/server/text-prompt.ts +12 -0
  107. package/src/{serve → server}/token-streams.ts +36 -38
  108. package/src/server/worker-routes.ts +311 -0
  109. package/src/storage/paths.ts +61 -0
  110. package/src/web/assets.ts +37 -0
  111. package/src/web/{src → browser}/api.ts +5 -23
  112. package/src/web/browser/chat-host.ts +17 -0
  113. package/src/web/browser/installed-panels.ts +6 -0
  114. package/src/web/browser/main.ts +55 -0
  115. package/src/web/browser/markdown.ts +279 -0
  116. package/src/web/browser/memory-panel.ts +9 -0
  117. package/src/web/browser/palette.ts +64 -0
  118. package/src/web/browser/protocol.ts +13 -0
  119. package/src/web/browser/shell.ts +467 -0
  120. package/src/web/{src → browser}/status.ts +32 -36
  121. package/src/web/build.ts +51 -0
  122. package/src/web/public/app.html +1054 -0
  123. package/src/web/{sw.js → public/sw.js} +4 -3
  124. package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
  125. package/src/web/{vendor → public/vendor}/hljs.js +2 -2
  126. package/THIRD_PARTY_LICENSES.md +0 -51
  127. package/docs/README.md +0 -94
  128. package/docs/archive/hlg-sampling.md +0 -350
  129. package/docs/archive/investigations/audio-input-plan.md +0 -356
  130. package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
  131. package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
  132. package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
  133. package/docs/archive/investigations/curve-bisector-routing.md +0 -123
  134. package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
  135. package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
  136. package/docs/archive/investigations/dspark-handoff.md +0 -224
  137. package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
  138. package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
  139. package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
  140. package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
  141. package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
  142. package/docs/archive/investigations/lab-build-journal.md +0 -157
  143. package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
  144. package/docs/archive/investigations/openjarvis-fit.md +0 -198
  145. package/docs/archive/investigations/openwiki-evaluation.md +0 -249
  146. package/docs/archive/investigations/optimization_plan.md +0 -346
  147. package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
  148. package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
  149. package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
  150. package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
  151. package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
  152. package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
  153. package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
  154. package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
  155. package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
  156. package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
  157. package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
  158. package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
  159. package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
  160. package/docs/archive/planning/chat-ui-vision.md +0 -50
  161. package/docs/archive/planning/chunk-task-roadmap.md +0 -77
  162. package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
  163. package/docs/archive/planning/journal.md +0 -316
  164. package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
  165. package/docs/archive/planning/pi-integration-review.md +0 -178
  166. package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
  167. package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
  168. package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
  169. package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
  170. package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
  171. package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
  172. package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
  173. package/docs/archive/planning/web-ui-pass-plan.md +0 -213
  174. package/docs/archive/planning/website-readme-pass-plan.md +0 -224
  175. package/docs/design/batching.md +0 -1841
  176. package/docs/design/decode-speed-program.md +0 -2349
  177. package/docs/design/dreaming-nightly-pipeline.md +0 -684
  178. package/docs/design/generic-model-support.md +0 -705
  179. package/docs/design/kv-cache.md +0 -824
  180. package/docs/design/orpo-training.md +0 -680
  181. package/docs/design/speculative-decoding.md +0 -1990
  182. package/docs/design/turboquant.md +0 -2212
  183. package/docs/design/unified-engine-frontier-plan.md +0 -1880
  184. package/docs/design/web-chat-redesign.md +0 -695
  185. package/docs/planning/PRODUCT_ROADMAP.md +0 -594
  186. package/docs/planning/ResearchTopics.md +0 -119
  187. package/docs/planning/release-notes-v0.4.0.md +0 -45
  188. package/docs/planning/video-series.md +0 -421
  189. package/docs/reference/benchmarks.md +0 -3613
  190. package/docs/reference/cli.md +0 -585
  191. package/docs/reference/distribution.md +0 -355
  192. package/docs/reference/environment.md +0 -398
  193. package/docs/reference/glossary.md +0 -39
  194. package/docs/reference/library-api.md +0 -674
  195. package/docs/reference/memory.md +0 -260
  196. package/docs/reference/models.md +0 -334
  197. package/docs/reference/server-api.md +0 -1775
  198. package/docs/reference/server-config.md +0 -846
  199. package/docs/reference/training.md +0 -462
  200. package/docs/reference/troubleshooting.md +0 -158
  201. package/src/admission-plan.ts +0 -18
  202. package/src/audio/conformer.ts +0 -647
  203. package/src/audio/decode.ts +0 -133
  204. package/src/audio/features.ts +0 -202
  205. package/src/audio/transcode.ts +0 -67
  206. package/src/backends/mlx/assistant-target.ts +0 -46
  207. package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
  208. package/src/backends/mlx/autoregressive.ts +0 -108
  209. package/src/backends/mlx/batch-group.ts +0 -1790
  210. package/src/backends/mlx/cache-attachment.ts +0 -67
  211. package/src/backends/mlx/cache-layout.ts +0 -31
  212. package/src/backends/mlx/checkpoint-state.ts +0 -53
  213. package/src/backends/mlx/continuation-persistence.ts +0 -106
  214. package/src/backends/mlx/continuation-request.ts +0 -43
  215. package/src/backends/mlx/continuation.ts +0 -77
  216. package/src/backends/mlx/diffusion.ts +0 -39
  217. package/src/backends/mlx/draft-target.ts +0 -32
  218. package/src/backends/mlx/evaluation.ts +0 -105
  219. package/src/backends/mlx/fill-append.ts +0 -45
  220. package/src/backends/mlx/full-kv-row-append.ts +0 -32
  221. package/src/backends/mlx/full-kv-row-donor.ts +0 -54
  222. package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
  223. package/src/backends/mlx/gateway-binding.ts +0 -130
  224. package/src/backends/mlx/graph.ts +0 -38
  225. package/src/backends/mlx/kv-codec.ts +0 -9
  226. package/src/backends/mlx/kv-maintenance.ts +0 -133
  227. package/src/backends/mlx/kv-tensor-rows.ts +0 -212
  228. package/src/backends/mlx/memory-client.ts +0 -189
  229. package/src/backends/mlx/methods.ts +0 -90
  230. package/src/backends/mlx/model-prompt.ts +0 -164
  231. package/src/backends/mlx/model-serving.ts +0 -79
  232. package/src/backends/mlx/prefill-cohort.ts +0 -89
  233. package/src/backends/mlx/prefill-rows.ts +0 -200
  234. package/src/backends/mlx/prefill.ts +0 -33
  235. package/src/backends/mlx/projected-context-rows.ts +0 -72
  236. package/src/backends/mlx/request-policy.ts +0 -20
  237. package/src/backends/mlx/request-state-policy.ts +0 -41
  238. package/src/backends/mlx/rollback.ts +0 -89
  239. package/src/backends/mlx/row-sampling.ts +0 -49
  240. package/src/backends/mlx/serial-executor.ts +0 -242
  241. package/src/backends/mlx/speculative-group.ts +0 -359
  242. package/src/backends/mlx/speculative-round.ts +0 -161
  243. package/src/backends/mlx/speculative.ts +0 -114
  244. package/src/backends/mlx/state-rows.ts +0 -104
  245. package/src/backends/mlx/state-views.ts +0 -46
  246. package/src/backends/mlx/target-layout-capability.ts +0 -35
  247. package/src/backends/mlx/text-engine.ts +0 -73
  248. package/src/backends/mlx/transitioning-kv-rows.ts +0 -165
  249. package/src/chat-template.ts +0 -382
  250. package/src/cli.ts +0 -3168
  251. package/src/config.ts +0 -399
  252. package/src/contracts/completion.ts +0 -16
  253. package/src/contracts/execution.ts +0 -60
  254. package/src/contracts/generation.ts +0 -103
  255. package/src/contracts/host.ts +0 -7
  256. package/src/contracts/pi-provider.ts +0 -30
  257. package/src/contracts/pi.ts +0 -259
  258. package/src/contracts/prefix-cache.ts +0 -23
  259. package/src/contracts/resources.ts +0 -25
  260. package/src/contracts/sampling.ts +0 -23
  261. package/src/contracts/scheduling.ts +0 -45
  262. package/src/contracts/tsconfig.json +0 -14
  263. package/src/dataset/generators.ts +0 -1087
  264. package/src/dataset/index.ts +0 -12
  265. package/src/dataset/job.ts +0 -81
  266. package/src/dataset/llm.ts +0 -71
  267. package/src/dataset/registry.ts +0 -528
  268. package/src/diffusion/diffusion-generate.ts +0 -507
  269. package/src/download.ts +0 -505
  270. package/src/embed.ts +0 -77
  271. package/src/engine/admission.ts +0 -59
  272. package/src/engine/cancellation.ts +0 -38
  273. package/src/engine/engine.ts +0 -56
  274. package/src/engine/execution-coordinator.ts +0 -70
  275. package/src/engine/execution-plan.ts +0 -53
  276. package/src/engine/resources.ts +0 -53
  277. package/src/engine/scheduler.ts +0 -72
  278. package/src/engine/session.ts +0 -247
  279. package/src/eval/README.md +0 -130
  280. package/src/eval/ifeval.ts +0 -81
  281. package/src/eval/kl-prompts.ts +0 -29
  282. package/src/eval/kl.ts +0 -590
  283. package/src/eval/perplexity.ts +0 -164
  284. package/src/eval/quality-db.ts +0 -92
  285. package/src/eval/reference-logits.ts +0 -207
  286. package/src/eval/runner.ts +0 -124
  287. package/src/eval/score.ts +0 -49
  288. package/src/eval/tasks/bfcl.ts +0 -535
  289. package/src/eval/tasks/gsm8k.ts +0 -98
  290. package/src/eval/tasks/hashhop.ts +0 -139
  291. package/src/eval/tasks/humaneval.ts +0 -275
  292. package/src/eval/tasks/ifeval.ts +0 -525
  293. package/src/eval/tasks/mmlu.ts +0 -157
  294. package/src/eval/ultrafeedback.ts +0 -60
  295. package/src/evaldb.ts +0 -199
  296. package/src/expert-io.ts +0 -393
  297. package/src/expert-offload-build.ts +0 -77
  298. package/src/expert-offload.ts +0 -76
  299. package/src/expert-residency.ts +0 -1134
  300. package/src/expert-usage.ts +0 -415
  301. package/src/fill/echo-index.ts +0 -188
  302. package/src/fill/fill-session.ts +0 -452
  303. package/src/fill/proposal.ts +0 -68
  304. package/src/fill/schema-rows.ts +0 -506
  305. package/src/fill/tool-boundary.ts +0 -103
  306. package/src/fit.ts +0 -249
  307. package/src/generate.ts +0 -1346
  308. package/src/grammar.ts +0 -591
  309. package/src/harness-pi.ts +0 -184
  310. package/src/hf-push.ts +0 -628
  311. package/src/hub-rest.ts +0 -242
  312. package/src/index.ts +0 -63
  313. package/src/inference/denoising.ts +0 -17
  314. package/src/inference/draft-acceptance.ts +0 -41
  315. package/src/inference/draft-output.ts +0 -49
  316. package/src/inference/graph.ts +0 -33
  317. package/src/inference/prefill.ts +0 -39
  318. package/src/inference/rollback.ts +0 -7
  319. package/src/jobs/index.ts +0 -31
  320. package/src/jobs/job-entry.ts +0 -107
  321. package/src/jobs/task-client.ts +0 -19
  322. package/src/jobs/types.ts +0 -2
  323. package/src/kv-scheme.ts +0 -257
  324. package/src/kv-store.ts +0 -1099
  325. package/src/lab/README.md +0 -18
  326. package/src/lab/curve/curve-designer.html +0 -303
  327. package/src/lab/curve/curve-sampler.ts +0 -122
  328. package/src/lab/expert-trace/expert-trace.ts +0 -69
  329. package/src/lab/paged-kv/paged-kv-rows.ts +0 -83
  330. package/src/lab/paged-kv/paged-kv.ts +0 -270
  331. package/src/library.ts +0 -40
  332. package/src/lora.ts +0 -408
  333. package/src/md-modules.d.ts +0 -49
  334. package/src/media-fetch.ts +0 -442
  335. package/src/memory/article.ts +0 -558
  336. package/src/memory/chunk-validation.ts +0 -150
  337. package/src/memory/chunk.ts +0 -318
  338. package/src/memory/cluster.ts +0 -351
  339. package/src/memory/coverage.ts +0 -134
  340. package/src/memory/crosslink.ts +0 -0
  341. package/src/memory/db.ts +0 -253
  342. package/src/memory/entity.ts +0 -203
  343. package/src/memory/gate.ts +0 -187
  344. package/src/memory/history.ts +0 -152
  345. package/src/memory/ingest.ts +0 -335
  346. package/src/memory/model.ts +0 -183
  347. package/src/memory/normalize.ts +0 -671
  348. package/src/memory/parse.ts +0 -22
  349. package/src/memory/pipeline.ts +0 -275
  350. package/src/memory/prompts.ts +0 -65
  351. package/src/memory/query.ts +0 -334
  352. package/src/memory/reconcile.ts +0 -734
  353. package/src/memory/redirect.ts +0 -108
  354. package/src/memory/reindex.ts +0 -325
  355. package/src/memory/resolve.ts +0 -307
  356. package/src/memory/rest.ts +0 -245
  357. package/src/memory/route.ts +0 -272
  358. package/src/memory/schedule.ts +0 -165
  359. package/src/memory/stages.ts +0 -777
  360. package/src/memory/synthesize.ts +0 -1638
  361. package/src/memory/tools.ts +0 -749
  362. package/src/memory/vault.ts +0 -1194
  363. package/src/memory/wikify.ts +0 -1064
  364. package/src/mlx/array.ts +0 -388
  365. package/src/mlx/autograd.ts +0 -304
  366. package/src/mlx/checkpoint.ts +0 -149
  367. package/src/mlx/compile.ts +0 -168
  368. package/src/mlx/custom-vjp.ts +0 -188
  369. package/src/mlx/ffi.ts +0 -512
  370. package/src/mlx/materialize.ts +0 -23
  371. package/src/mlx/metal-kernel.ts +0 -159
  372. package/src/mlx/ops.ts +0 -1021
  373. package/src/mlx/quantized-rows.ts +0 -36
  374. package/src/mlx/token-bitmask.ts +0 -52
  375. package/src/mlx/turboquant-kv-decode.ts +0 -82
  376. package/src/mlx/turboquant-kv-inverse.ts +0 -122
  377. package/src/mlx/turboquant-ops.ts +0 -861
  378. package/src/mlx/turboquant-tables.ts +0 -237
  379. package/src/mmap.ts +0 -90
  380. package/src/model/artifact-identity.ts +0 -73
  381. package/src/model/batched-kv.ts +0 -235
  382. package/src/model/batched-mask.ts +0 -279
  383. package/src/model/batched-quant.ts +0 -212
  384. package/src/model/batched-quantized-kv.ts +0 -144
  385. package/src/model/batched-rotating-quant.ts +0 -353
  386. package/src/model/batched-rotating-state.ts +0 -149
  387. package/src/model/batched-rotating.ts +0 -338
  388. package/src/model/batched-row-storage.ts +0 -179
  389. package/src/model/batched-ssm.ts +0 -52
  390. package/src/model/batched-turboquant-kv.ts +0 -93
  391. package/src/model/compiled-decode.ts +0 -712
  392. package/src/model/decoded-kv-donor.ts +0 -12
  393. package/src/model/delayed-quantized-kv.ts +0 -72
  394. package/src/model/delayed-rotating-quantized-kv.ts +0 -93
  395. package/src/model/delayed-turboquant-kv.ts +0 -55
  396. package/src/model/diffusion-gemma.ts +0 -1026
  397. package/src/model/factory.ts +0 -189
  398. package/src/model/fingerprint.ts +0 -33
  399. package/src/model/flash-attention.ts +0 -694
  400. package/src/model/full-prefill-padding.ts +0 -56
  401. package/src/model/full-prefill-row.ts +0 -105
  402. package/src/model/gemma4-base.ts +0 -2351
  403. package/src/model/gemma4.ts +0 -1315
  404. package/src/model/generated/gemma4-12b.ts +0 -429
  405. package/src/model/generated/gemma4-26b.ts +0 -448
  406. package/src/model/generated/gemma4-e4b.ts +0 -717
  407. package/src/model/generated/index.ts +0 -18
  408. package/src/model/glm52-atlas.ts +0 -511
  409. package/src/model/glm52-cache.ts +0 -792
  410. package/src/model/glm52-config.ts +0 -262
  411. package/src/model/glm52-container.ts +0 -305
  412. package/src/model/glm52-coupling.ts +0 -388
  413. package/src/model/glm52-dsa.ts +0 -606
  414. package/src/model/glm52-expert-layout.ts +0 -253
  415. package/src/model/glm52-layout.ts +0 -152
  416. package/src/model/glm52-memory.ts +0 -500
  417. package/src/model/glm52-mla.ts +0 -792
  418. package/src/model/glm52-moe.ts +0 -223
  419. package/src/model/glm52-pilot.ts +0 -537
  420. package/src/model/glm52-reference.ts +0 -445
  421. package/src/model/glm52-residency.ts +0 -557
  422. package/src/model/glm52-resident-weights.ts +0 -135
  423. package/src/model/glm52-streamed-experts.ts +0 -453
  424. package/src/model/glm52-streamed-metal.ts +0 -610
  425. package/src/model/glm52-weights.ts +0 -503
  426. package/src/model/glm52.ts +0 -1132
  427. package/src/model/implementation.ts +0 -69
  428. package/src/model/kv-attention-view.ts +0 -67
  429. package/src/model/minicpm5.ts +0 -376
  430. package/src/model/profile.ts +0 -443
  431. package/src/model/qwen-append.ts +0 -12
  432. package/src/model/qwen-conv.ts +0 -52
  433. package/src/model/qwen3-delta-state.ts +0 -74
  434. package/src/model/qwen3-delta.ts +0 -451
  435. package/src/model/qwen3-moe.ts +0 -404
  436. package/src/model/qwen3-mrope.ts +0 -252
  437. package/src/model/qwen3.ts +0 -277
  438. package/src/model/qwen3_5-checkpoint.ts +0 -164
  439. package/src/model/qwen3_5.ts +0 -819
  440. package/src/model/rotating-kv-donor.ts +0 -35
  441. package/src/model/rotating-kv-layout.ts +0 -262
  442. package/src/model/rotating-row-transaction.ts +0 -35
  443. package/src/model/speculative-rotating-kv.ts +0 -101
  444. package/src/model/ssm-prefill-padding.ts +0 -39
  445. package/src/model/support.ts +0 -102
  446. package/src/model/trellis-balanced-scatter.ts +0 -51
  447. package/src/model/trellis-linear.ts +0 -672
  448. package/src/model/trellis-shared-m.ts +0 -151
  449. package/src/model/trellis-shared-scatter.ts +0 -144
  450. package/src/model/trellis-splitk-prefill.ts +0 -158
  451. package/src/model/trellis-tiled-prefill.ts +0 -136
  452. package/src/model/trellis-vector-expand.ts +0 -62
  453. package/src/model/trellis-wide-prefill.ts +0 -94
  454. package/src/model/turboquant-codec.ts +0 -83
  455. package/src/model/universal/archs.ts +0 -332
  456. package/src/model/universal/dense.ts +0 -612
  457. package/src/model/universal/modules.ts +0 -170
  458. package/src/model/universal/rope.ts +0 -381
  459. package/src/native/expert_io.c +0 -523
  460. package/src/native/frame_extract.swift +0 -140
  461. package/src/native-pack.ts +0 -162
  462. package/src/pi-provider.ts +0 -53
  463. package/src/pi-session.ts +0 -81
  464. package/src/pi-terminal.ts +0 -274
  465. package/src/pi-web.ts +0 -1932
  466. package/src/preflight.ts +0 -158
  467. package/src/prompt-cache.ts +0 -460
  468. package/src/quantize/allocator.ts +0 -627
  469. package/src/quantize/atomic-output.ts +0 -38
  470. package/src/quantize/calibration.ts +0 -181
  471. package/src/quantize/config-writer.ts +0 -145
  472. package/src/quantize/index.ts +0 -84
  473. package/src/quantize/job.ts +0 -162
  474. package/src/quantize/quantizer.ts +0 -666
  475. package/src/quantize/rotate.ts +0 -603
  476. package/src/quantize/safetensors-writer.ts +0 -249
  477. package/src/quantize/sensitivity.ts +0 -216
  478. package/src/quantize/trellis.ts +0 -586
  479. package/src/quantize/weight-transform.ts +0 -292
  480. package/src/registry.ts +0 -602
  481. package/src/runtime-config.ts +0 -81
  482. package/src/safetensors.ts +0 -158
  483. package/src/sampler.ts +0 -1011
  484. package/src/serve/admin-routes.ts +0 -184
  485. package/src/serve/assets/curve-terrain.html +0 -105
  486. package/src/serve/assets/training-inference-map.html +0 -575
  487. package/src/serve/aux-routes.ts +0 -132
  488. package/src/serve/batch-scheduler.ts +0 -8
  489. package/src/serve/checkpoint-identity.ts +0 -64
  490. package/src/serve/generation-gateway.ts +0 -496
  491. package/src/serve/isolate.ts +0 -609
  492. package/src/serve/kv-budget.ts +0 -14
  493. package/src/serve/lab-routes.ts +0 -341
  494. package/src/serve/lane-registry.ts +0 -45
  495. package/src/serve/model-admin-routes.ts +0 -155
  496. package/src/serve/model-binding.ts +0 -43
  497. package/src/serve/model-host.ts +0 -511
  498. package/src/serve/parent-application.ts +0 -56
  499. package/src/serve/prompt-response-trace.ts +0 -147
  500. package/src/serve/response-lease.ts +0 -24
  501. package/src/serve/session-search.ts +0 -326
  502. package/src/serve/static-routes.ts +0 -101
  503. package/src/server.ts +0 -1314
  504. package/src/spec/assistant-rows.ts +0 -92
  505. package/src/spec/assistant-source.ts +0 -126
  506. package/src/spec/cache-identity.ts +0 -22
  507. package/src/spec/deepspec-rows.ts +0 -29
  508. package/src/spec/deepspec-source.ts +0 -145
  509. package/src/spec/dflash-source.ts +0 -149
  510. package/src/spec/drafter.ts +0 -304
  511. package/src/spec/dspark/ab-stats.ts +0 -136
  512. package/src/spec/dspark/calibration.ts +0 -112
  513. package/src/spec/dspark/data-dflash.ts +0 -156
  514. package/src/spec/dspark/data.ts +0 -201
  515. package/src/spec/dspark/deepspec-module.ts +0 -776
  516. package/src/spec/dspark/generate-dflash.ts +0 -245
  517. package/src/spec/dspark/generate.ts +0 -290
  518. package/src/spec/dspark/loader.ts +0 -23
  519. package/src/spec/dspark/loss.ts +0 -213
  520. package/src/spec/dspark/module-dflash.ts +0 -560
  521. package/src/spec/dspark/module.ts +0 -540
  522. package/src/spec/dspark/quantize-drafter.ts +0 -91
  523. package/src/spec/dspark/sample.ts +0 -112
  524. package/src/spec/generate.ts +0 -276
  525. package/src/spec/glm52-mtp-graph.ts +0 -29
  526. package/src/spec/glm52-mtp-rows.ts +0 -132
  527. package/src/spec/glm52-mtp-source.ts +0 -317
  528. package/src/spec/glm52-mtp-state.ts +0 -33
  529. package/src/spec/ngram-source.ts +0 -204
  530. package/src/spec/projected-draft-rows.ts +0 -73
  531. package/src/spec/qwen-mtp-module.ts +0 -171
  532. package/src/spec/qwen-mtp-rows.ts +0 -219
  533. package/src/spec/qwen-mtp-source.ts +0 -383
  534. package/src/spec/qwen-mtp-state.ts +0 -30
  535. package/src/spec/serve-loop.ts +0 -585
  536. package/src/spec/source.ts +0 -215
  537. package/src/spec/two-model-rows.ts +0 -61
  538. package/src/spec/two-model.ts +0 -113
  539. package/src/ssd-cache.ts +0 -372
  540. package/src/ssd-durability.ts +0 -211
  541. package/src/tokenizer.ts +0 -335
  542. package/src/tool-approvals.ts +0 -131
  543. package/src/tool-call.ts +0 -695
  544. package/src/train/dataset.ts +0 -484
  545. package/src/train/diffusion-lora.ts +0 -216
  546. package/src/train/export.ts +0 -32
  547. package/src/train/flash-cce.ts +0 -1234
  548. package/src/train/forward.ts +0 -266
  549. package/src/train/fuse.ts +0 -243
  550. package/src/train/index.ts +0 -46
  551. package/src/train/job.ts +0 -205
  552. package/src/train/lora-params.ts +0 -254
  553. package/src/train/loss.ts +0 -1591
  554. package/src/train/merge.ts +0 -195
  555. package/src/train/optimizer.ts +0 -183
  556. package/src/train/prefix-shared.ts +0 -490
  557. package/src/train/rank.ts +0 -137
  558. package/src/train/segmented.ts +0 -1828
  559. package/src/train/steel-qmm-header.ts +0 -1461
  560. package/src/train/trainer.ts +0 -1221
  561. package/src/train/watch.ts +0 -542
  562. package/src/vision/diffusion-vision.ts +0 -620
  563. package/src/vision/embedder.ts +0 -168
  564. package/src/vision/preprocess.ts +0 -247
  565. package/src/vision/prompt.ts +0 -373
  566. package/src/vision/qwen3vl-preprocess.ts +0 -409
  567. package/src/vision/qwen3vl-prompt.ts +0 -196
  568. package/src/vision/qwen3vl-tower.ts +0 -555
  569. package/src/vision/siglip.ts +0 -599
  570. package/src/vision/video-frames.ts +0 -148
  571. package/src/web/app.html +0 -2143
  572. package/src/web/app.js +0 -5703
  573. package/src/web/skills/memory/SKILL.md +0 -120
  574. package/src/web/skills/web-research/SKILL.md +0 -46
  575. package/src/web/skills.ts +0 -97
  576. package/src/web/src/adapters-panel.ts +0 -288
  577. package/src/web/src/assistant.ts +0 -451
  578. package/src/web/src/chat.ts +0 -1057
  579. package/src/web/src/composer.ts +0 -959
  580. package/src/web/src/dataset.ts +0 -132
  581. package/src/web/src/finetune.ts +0 -267
  582. package/src/web/src/hub.ts +0 -428
  583. package/src/web/src/main.ts +0 -98
  584. package/src/web/src/markdown.ts +0 -0
  585. package/src/web/src/memory-panel.ts +0 -572
  586. package/src/web/src/model-picker.ts +0 -159
  587. package/src/web/src/palette.ts +0 -346
  588. package/src/web/src/protocol.ts +0 -26
  589. package/src/web/src/quantize.ts +0 -167
  590. package/src/web/src/rag.ts +0 -274
  591. package/src/web/src/sessions.ts +0 -469
  592. package/src/web/src/shell.ts +0 -804
  593. package/src/web/src/ui-catalog.ts +0 -78
  594. package/src/web/vendor/README.md +0 -61
  595. package/src/web-assets.ts +0 -47
  596. package/src/web-tools.ts +0 -478
  597. package/src/weights.ts +0 -255
  598. /package/src/{serve → server}/completion-sink.ts +0 -0
  599. /package/src/web/{icon.svg → public/icon.svg} +0 -0
  600. /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
  601. /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
@@ -1,734 +0,0 @@
1
- // mlx-bun memory — RECONCILE pass (P7-T3): make an article internally consistent
2
- // with its latest-dated position AFTER a self-healing fold.
3
- //
4
- // A targeted PATCH resolves ONE claim toward the user's newest statement — it
5
- // flips the LEAD (the verdict) to the current value. But the rest of the article
6
- // can be left contradicting that verdict: an INFOBOX relationship field still
7
- // records the superseded value (`<preference_key>: <old value>` while the lead now
8
- // says `<new value>`), and a sibling BODY section still asserts the old take. This pass
9
- // runs AFTER the patch loop, scans those surfaces, and updates whatever the fold
10
- // made stale so the whole article ends consistent with the latest position.
11
- //
12
- // It is the same discipline as PATCH: every edit goes through NORMALIZE + the
13
- // conservative gate, and it is PROVENANCE-PRESERVING — a section rewrite keeps
14
- // every pre-existing `[^k]` marker (phrased as a change over time), so the gate's
15
- // citation-survival check passes instead of NO-OP'ing the fold. It is bounded
16
- // (only relationship infobox fields + body sections are touched; world-fact / spec
17
- // fields are left alone) and idempotent: a fully-consistent article is a NO-OP.
18
-
19
- import { readFile, writeFile } from "node:fs/promises";
20
- import { join } from "node:path";
21
-
22
- import { articleStructure, fenceLabel, isFenceLine, parseInfobox, parseLead } from "./article";
23
- import { articleTitle } from "./cluster";
24
- import type { MemoryStore } from "./db";
25
- import { gateEdit } from "./gate";
26
- import {
27
- buildContradictionClassifierPrompt,
28
- buildHistoryEntry,
29
- classifyContradiction,
30
- type ContradictionKind,
31
- markerNear,
32
- upsertHistoryEntry,
33
- } from "./history";
34
- import { callLocal, MAX_OUTPUT_TOKENS } from "./model";
35
- import { normalizeArticle } from "./normalize";
36
- import {
37
- articleFootnoteState,
38
- entityStem,
39
- PHYSICAL_SPEC_KEYS,
40
- referenceDatesByMarker,
41
- sanitizeLead,
42
- sanitizeSection,
43
- type SynthesisCall,
44
- } from "./synthesize";
45
- import { articlesDir, commitVault, extractSection, listArticles, slugifyHeading, vaultRoot } from "./vault";
46
-
47
- // ---- relationship-field classification -------------------------------------
48
-
49
- /** Infobox keys we NEVER touch — schema keys plus the physical spec keys (a
50
- * spec is a world-fact, not a stance that a verdict can supersede). */
51
- const RECONCILE_SKIP_KEYS = new Set(["type", "kind", "aliases", "categories"]);
52
-
53
- /** A relationship/opinion key whose value records the user's stance — the only
54
- * infobox fields a verdict change can make stale (favorite / opinion / status /
55
- * preference / current pick …). World-fact + spec keys are excluded above. */
56
- const RELATIONSHIP_KEY_RE =
57
- /(favou?rite|prefer|preference|pick|choice|chosen|opinion|verdict|status|stance|current|rating|sentiment|\btake\b|likes?|dislikes?|using|used_for|owns?|owned|acquired)/i;
58
-
59
- /** True when an infobox field is a relationship/stance fact (refreshable), not a
60
- * world-fact or physical spec (left untouched). */
61
- export function isRelationshipKey(key: string): boolean {
62
- const k = key.trim().toLowerCase();
63
- if (RECONCILE_SKIP_KEYS.has(k)) return false;
64
- if (PHYSICAL_SPEC_KEYS.has(k)) return false;
65
- return RELATIONSHIP_KEY_RE.test(k);
66
- }
67
-
68
- // ---- prompt builders (schematic — no parrot-able concrete values) ----------
69
-
70
- /** The model sentinel for "this fact already agrees with the summary". */
71
- export const RECONCILE_KEEP = "KEEP";
72
-
73
- /** Check ONE infobox relationship fact against the current verdict (the lead). The
74
- * model answers KEEP when it agrees, or the corrected VALUE (no key, no prose)
75
- * when it is stale. Schematic on purpose. */
76
- export function buildInfoboxReconcilePrompt(verdict: string, key: string, value: string): string {
77
- return (
78
- `You are checking ONE infobox fact against the article's current summary for internal consistency.\n\n` +
79
- `CURRENT SUMMARY (the user's latest position):\n${verdict}\n\n` +
80
- `INFOBOX FACT:\n${key}: ${value}\n\n` +
81
- `If the fact AGREES with the current summary, answer exactly ${RECONCILE_KEEP}. ` +
82
- `If it CONTRADICTS the summary (it records a preference, opinion, or status the summary has since changed), ` +
83
- `answer with the corrected VALUE only — the single value that matches the summary, no key, no prose, no quotes.`
84
- );
85
- }
86
-
87
- /** Detection (used only for a ground-truth field with no KNOWN superseded value):
88
- * does this passage state, as the user's CURRENT choice, a value that DIFFERS
89
- * from the established current value? Answer KEEP, or that different value only. */
90
- export function buildProseConflictPrompt(label: string, groundTruth: string, body: string): string {
91
- return (
92
- `ESTABLISHED CURRENT VALUE: ${groundTruth}\n\n` +
93
- `${label}:\n${body || "(empty)"}\n\n` +
94
- `Does this passage state, as the user's CURRENT / present choice, a value DIFFERENT from ${groundTruth}? ` +
95
- `If it states only ${groundTruth} as current (any other value appears solely as past history), answer exactly ${RECONCILE_KEEP}. ` +
96
- `Otherwise answer with that different current value ONLY — no key, no prose, no quotes.`
97
- );
98
- }
99
-
100
- /**
101
- * The CONSTRAINED consistency rewrite — the safety net that guarantees the prose
102
- * agrees with the infobox GROUND TRUTH. Both values are real article data (the
103
- * resolved current value + the value it replaced), not copyable example values:
104
- * we tell the model exactly which value is current and which is former, so the
105
- * small model only has to demote the named superseded value to past tense — far
106
- * more reliable than the open-ended "resolve toward latest" fold. Every existing
107
- * `[^k]` marker is preserved (provenance), framed as a change over time.
108
- */
109
- export function buildProseConsistencyPrompt(
110
- label: string,
111
- groundTruth: string,
112
- superseded: string,
113
- body: string,
114
- ): string {
115
- return (
116
- `You are correcting ONE passage of a personal-wiki article so it agrees with the article's ESTABLISHED CURRENT fact.\n\n` +
117
- `ESTABLISHED CURRENT VALUE: ${groundTruth}\n` +
118
- `FORMER (SUPERSEDED) VALUE: ${superseded}\n\n` +
119
- `${label}:\n${body || "(empty)"}\n\n` +
120
- `Rewrite the passage so it reads as a change OVER TIME: state ${superseded} ONLY in the past tense ` +
121
- `("previously ${superseded}", "originally ${superseded}", "${superseded} was"), and state ${groundTruth} as the user's CURRENT, present choice. ` +
122
- `${superseded} must NEVER appear as a current/present favourite, pick, or "the one I reach for". ` +
123
- `Keep EVERY existing [^k] citation marker: attach the ${superseded} marker to the past clause and the ${groundTruth} marker to the present clause, so no [^N] is lost. ` +
124
- `Prefer ONE flowing statement — "previously ${superseded}[^k]; now ${groundTruth} is my current pick[^n]". ` +
125
- `Output ONLY the rewritten passage prose — no heading, no list, no References section.`
126
- );
127
- }
128
-
129
- /**
130
- * The CURRENT-STATE-ONLY rewrite. Used for the LEAD (which must read as a clean
131
- * current-state summary — never scattered "previously …" caveats) and for a
132
- * factual-ERROR fix in any passage (the wrong value is removed as if it had never
133
- * been there). The passage is rewritten to state ONLY `groundTruth` as current and
134
- * to DROP `superseded` entirely. For an evolution the dropped value's trajectory
135
- * is recorded in the article's `## History` section instead; for an error it is
136
- * simply gone. Both values are real article data, not copyable example values.
137
- */
138
- export function buildCurrentStateOnlyPrompt(
139
- label: string,
140
- groundTruth: string,
141
- superseded: string,
142
- body: string,
143
- isError: boolean,
144
- ): string {
145
- const why = isError
146
- ? `The value ${superseded} was a MISTAKE — a wrong fact. Remove it entirely; it must NOT appear anywhere in the passage (no "previously", no history). ` +
147
- `You may drop ${superseded}'s citation marker.`
148
- : `Do NOT mention ${superseded} in this passage — its trajectory is recorded separately in the History section. ` +
149
- `Keep ${groundTruth}'s [^k] citation marker.`;
150
- return (
151
- `You are rewriting ONE passage of a personal-wiki article to state ONLY the user's CURRENT value.\n\n` +
152
- `CURRENT VALUE: ${groundTruth}\n` +
153
- `VALUE TO REMOVE: ${superseded}\n\n` +
154
- `${label}:\n${body || "(empty)"}\n\n` +
155
- `Rewrite the passage so it states ${groundTruth} as the user's current, present value, in the present tense. ` +
156
- `${why} ` +
157
- `Output ONLY the rewritten passage prose — no heading, no list, no References section.`
158
- );
159
- }
160
-
161
- /** Sharper retry instruction when the first rewrite still asserts the superseded
162
- * value as current. */
163
- export function proseRetryHint(groundTruth: string, superseded: string): string {
164
- return (
165
- `STILL WRONG: the passage must NOT claim ${superseded} is current, now, or a favourite. ` +
166
- `The ONLY current value is ${groundTruth}. Write ${superseded} strictly as a FORMER value ("previously ${superseded}[^k]"), ` +
167
- `and ${groundTruth} as the present one ("now ${groundTruth}[^n]"). Demote ${superseded} to the past tense.`
168
- );
169
- }
170
-
171
- // ---- pure helpers ----------------------------------------------------------
172
-
173
- /** Distinct `[^N]` markers in a block (definition lines excluded). */
174
- function markersIn(body: string): number[] {
175
- const out: number[] = [];
176
- const seen = new Set<number>();
177
- for (const line of body.split("\n")) {
178
- if (/^\[\^\d+\]:/.test(line.trim())) continue;
179
- for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
180
- const n = parseInt(m[1]!, 10);
181
- if (!seen.has(n)) {
182
- seen.add(n);
183
- out.push(n);
184
- }
185
- }
186
- }
187
- return out;
188
- }
189
-
190
- /** Escape a literal string for embedding in a RegExp. */
191
- function escapeRe(s: string): string {
192
- return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
193
- }
194
-
195
- /** Past-tense framing cues — when one governs the clause that mentions a value,
196
- * the value reads as HISTORY, not a present assertion. Deliberately EXCLUDES
197
- * "currently"/"now"/"today" (those are PRESENT, the very thing we flag). */
198
- const PAST_FRAME_RE =
199
- /\b(previous|origin|earlier|used to|no longer|formerly|initial|at first|back then|before|had been|until|supersed|was|were|moved on|switched)\b/i;
200
-
201
- /**
202
- * True when `value` is asserted as the user's CURRENT / present choice somewhere
203
- * in `text` and that clause is NOT framed as past. A "clause" is a span between
204
- * sentence / `;` breaks, so "previously <old>[^1]; now <new>[^2]" reads the <old>
205
- * clause as history (past-framed) and is NOT flagged. This is the deterministic
206
- * detector AND the post-rewrite acceptance check the retry hinges on. Pure.
207
- */
208
- export function assertsValueAsCurrent(text: string, value: string): boolean {
209
- const t = text.toLowerCase();
210
- const v = value.trim().toLowerCase();
211
- if (!v || !t.includes(v)) return false;
212
- const ev = escapeRe(v);
213
- const present = new RegExp(
214
- `(${ev}[^.;!?]{0,60}\\b(is|are|remain|stay|favou?rite|prefer|reach for|number-one|pick|go-to|current|now)\\b)|` +
215
- `(\\b(favou?rite|prefer|current(ly)?|now|pick|reach for|go-to|reach|consider)\\b[^.;!?]{0,60}${ev})`,
216
- );
217
- for (const clause of t.split(/[.;!?]+/)) {
218
- if (!clause.includes(v)) continue;
219
- if (!present.test(clause)) continue;
220
- if (PAST_FRAME_RE.test(clause)) continue; // framed as history → fine
221
- return true; // present-tense, not past-framed → stale
222
- }
223
- return false;
224
- }
225
-
226
- /** Collapse adjacent duplicate footnote markers (`[^2][^2]` or `[^2] [^2]` →
227
- * `[^2]`) — the lead artifact a fold can leave. Pure. */
228
- export function dedupeAdjacentMarkers(text: string): string {
229
- return text.replace(/(\[\^\d+\])(\s*\1)+/g, "$1");
230
- }
231
-
232
- /** Strip a model value reply to a clean single-line value (drop quotes, a stray
233
- * echoed `key:` prefix, surrounding markup). */
234
- function cleanValue(raw: string, key: string): string {
235
- let v = raw.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
236
- v = v.replace(new RegExp(`^${key}\\s*:`, "i"), "").trim();
237
- v = v.replace(/^[-*+]\s+/, "").replace(/^["'`]+|["'`.]+$/g, "").trim();
238
- return v;
239
- }
240
-
241
- /** Replace the value of one `key:` line inside the article's fenced ```info block.
242
- * Returns the article unchanged when the key is absent. Pure. */
243
- export function setInfoboxFieldValue(content: string, key: string, newValue: string): string {
244
- const lines = content.split(/\r?\n/);
245
- let inInfo = false;
246
- for (let i = 0; i < lines.length; i++) {
247
- const line = lines[i]!;
248
- if (isFenceLine(line)) {
249
- if (!inInfo && fenceLabel(line) === "info") inInfo = true;
250
- else if (inInfo) break; // closing fence — leave the info block
251
- continue;
252
- }
253
- if (inInfo) {
254
- const idx = line.indexOf(":");
255
- if (idx > 0 && line.slice(0, idx).trim() === key) {
256
- lines[i] = `${key}: ${newValue}`;
257
- return lines.join("\n");
258
- }
259
- }
260
- }
261
- return content;
262
- }
263
-
264
- // ---- the reconcile core (pure w.r.t. disk) ---------------------------------
265
-
266
- export interface ReconcileDeps {
267
- /** Model call; defaults to `callLocal("synthesis", …)`. */
268
- call?: SynthesisCall;
269
- /** Other article stems (wikilink resolution in NORMALIZE). */
270
- stems?: Set<string>;
271
- /** Article stem (H1/NORMALIZE); defaults to the entity stem of the H1 title. */
272
- stem?: string;
273
- }
274
-
275
- export interface ReconcileResult {
276
- content: string;
277
- changed: boolean;
278
- /** Infobox keys whose value was refreshed to the current verdict. */
279
- refreshedFields: string[];
280
- /** Passage labels rewritten toward the ground-truth value (`lead` + section anchors). */
281
- rewrittenSections: string[];
282
- /** True when NORMALIZE+gate vetoed the reconciled article (kept the original). */
283
- gateVetoed: boolean;
284
- reason: string | null;
285
- normalizeNotes: string[];
286
- /** Passage labels where, even after the retry, the small model could NOT demote
287
- * the superseded value to past tense (left as the gate-vetoed original). Logged
288
- * honestly — these are a base-model limit, not a silent success. */
289
- unresolved: string[];
290
- /** True when at least one passage rewrite needed the sharper retry. */
291
- retried: boolean;
292
- /** Trajectory lines written to the `## History` section for EVOLUTION
293
- * corrections (the user's position changing over time). Empty when every
294
- * contradiction was a factual error (silently overwritten, no History). */
295
- historyEntries: string[];
296
- }
297
-
298
- /** One prose-passage to reconcile: the LEAD or a body section. */
299
- interface Passage {
300
- /** Stable label for `rewrittenSections` / logs (`lead` or the section anchor). */
301
- label: string;
302
- /** Current passage body (no heading). */
303
- body: string;
304
- /** Sanitizer for a model rewrite (section vs lead). */
305
- sanitize: (raw: string) => string;
306
- /** Splice a rewritten body back into the article. */
307
- write: (article: string, newBody: string) => string;
308
- }
309
-
310
- /** Collect the reconcilable prose passages (lead + every body section, skipping
311
- * References / See also / Notes) with the helpers to read+write each. */
312
- function collectPassages(article: string): Passage[] {
313
- const out: Passage[] = [];
314
- const items = articleStructure(article);
315
-
316
- const lead = items.find((it) => it.kind === "lead");
317
- if (lead) {
318
- const lines = article.split("\n");
319
- const start = lead.startLine - 1;
320
- const end = lead.endLine - 1;
321
- const body = lines.slice(start, end + 1).join("\n").trim();
322
- if (body) {
323
- out.push({
324
- label: "lead",
325
- body,
326
- sanitize: sanitizeLead,
327
- write: (a, nb) => {
328
- const ls = a.split("\n");
329
- return [...ls.slice(0, start), nb, ...ls.slice(end + 1)].join("\n");
330
- },
331
- });
332
- }
333
- }
334
-
335
- for (const item of items) {
336
- if (item.kind !== "section" || !item.title) continue;
337
- const anchor = slugifyHeading(item.title);
338
- const block = extractSection(article, anchor);
339
- if (block == null) continue;
340
- const headingLine = block.split("\n")[0] ?? `## ${item.title}`;
341
- const body = block.split("\n").slice(1).join("\n").trim();
342
- if (!body) continue;
343
- const title = item.title;
344
- out.push({
345
- label: anchor,
346
- body,
347
- sanitize: (raw) => sanitizeSection(raw, title),
348
- write: (a, nb) => {
349
- const cur = extractSection(a, anchor);
350
- if (cur == null) return a;
351
- const idx = a.indexOf(cur);
352
- if (idx < 0) return a;
353
- return a.slice(0, idx) + `${headingLine}\n\n${nb}\n` + a.slice(idx + cur.length);
354
- },
355
- });
356
- }
357
- return out;
358
- }
359
-
360
- /** Outcome of one constrained passage rewrite. */
361
- interface PassageFix {
362
- body: string;
363
- changed: boolean;
364
- retried: boolean;
365
- /** True when the small model still left the superseded value as current AFTER
366
- * the retry (or dropped a marker) — kept the original; report honestly. */
367
- unresolved: boolean;
368
- }
369
-
370
- /**
371
- * Make ONE passage agree that `groundTruth` is current and `superseded` is former.
372
- * No-op when the passage does not assert `superseded` as current (idempotent).
373
- * Otherwise: constrained rewrite → deterministic check → ONE sharper retry; if it
374
- * still asserts the stale value as current OR would drop a `[^k]` marker, keep the
375
- * original and flag `unresolved`. Markers are deduped; provenance is preserved.
376
- */
377
- async function reconcileOnePassage(
378
- call: SynthesisCall,
379
- passage: Passage,
380
- groundTruth: string,
381
- superseded: string,
382
- ): Promise<PassageFix> {
383
- const keep: PassageFix = { body: passage.body, changed: false, retried: false, unresolved: false };
384
- if (!superseded || !groundTruth || superseded.toLowerCase() === groundTruth.toLowerCase()) return keep;
385
- if (!assertsValueAsCurrent(passage.body, superseded)) return keep; // already clean / absent
386
-
387
- const before = new Set(markersIn(passage.body));
388
- const attempt = async (prompt: string): Promise<string> => {
389
- const raw = await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS });
390
- return dedupeAdjacentMarkers(passage.sanitize(raw)).trim();
391
- };
392
- const accept = (x: string): boolean => {
393
- if (!x) return false;
394
- if (assertsValueAsCurrent(x, superseded)) return false; // still asserts stale value as current
395
- const after = new Set(markersIn(x));
396
- return ![...before].some((m) => !after.has(m)); // every marker preserved (provenance)
397
- };
398
-
399
- const base = buildProseConsistencyPrompt(passage.label, groundTruth, superseded, passage.body);
400
- let body = await attempt(base);
401
- if (accept(body)) return { body, changed: true, retried: false, unresolved: false };
402
-
403
- body = await attempt(`${base}\n\n${proseRetryHint(groundTruth, superseded)}`);
404
- if (accept(body)) return { body, changed: true, retried: true, unresolved: false };
405
-
406
- return { ...keep, retried: true, unresolved: true }; // base-model limit — keep original
407
- }
408
-
409
- /**
410
- * Make ONE passage state ONLY `groundTruth` as current and DROP `superseded`
411
- * entirely (the LEAD path, and the factual-ERROR path). No-op when the passage
412
- * does not mention `superseded`. Otherwise: constrained rewrite → deterministic
413
- * check (superseded gone, groundTruth present) → ONE sharper retry; if it still
414
- * mentions `superseded`, keep the original and flag `unresolved`. For an evolution
415
- * the dropped value's `[^k]` is preserved in the History section (added by the
416
- * caller); for an error it is allowed to fall away.
417
- */
418
- async function reconcileOnePassageCurrentState(
419
- call: SynthesisCall,
420
- passage: Passage,
421
- groundTruth: string,
422
- superseded: string,
423
- isError: boolean,
424
- ): Promise<PassageFix> {
425
- const keep: PassageFix = { body: passage.body, changed: false, retried: false, unresolved: false };
426
- if (!superseded || !groundTruth || superseded.toLowerCase() === groundTruth.toLowerCase()) return keep;
427
- if (!passage.body.toLowerCase().includes(superseded.toLowerCase())) return keep; // already clean
428
-
429
- const attempt = async (prompt: string): Promise<string> => {
430
- const raw = await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS });
431
- return dedupeAdjacentMarkers(passage.sanitize(raw)).trim();
432
- };
433
- const accept = (x: string): boolean => {
434
- if (!x) return false;
435
- if (x.toLowerCase().includes(superseded.toLowerCase())) return false; // value not removed
436
- return x.toLowerCase().includes(groundTruth.toLowerCase()); // current value present
437
- };
438
-
439
- const base = buildCurrentStateOnlyPrompt(passage.label, groundTruth, superseded, passage.body, isError);
440
- let body = await attempt(base);
441
- if (accept(body)) return { body, changed: true, retried: false, unresolved: false };
442
-
443
- body = await attempt(`${base}\n\n${proseRetryHint(groundTruth, superseded)}`);
444
- if (accept(body)) return { body, changed: true, retried: true, unresolved: false };
445
-
446
- return { ...keep, retried: true, unresolved: true };
447
- }
448
-
449
- /**
450
- * Reconcile ONE article to its INFOBOX GROUND TRUTH. The infobox relationship
451
- * value is the authority (it is resolved first, against the lead verdict): once
452
- * resolved, the LEAD and EVERY body section must AGREE with it. We refresh stale
453
- * infobox fields, then for each passage that still asserts a SUPERSEDED value as
454
- * current we run a CONSTRAINED rewrite ("make <ground-truth> current and <old>
455
- * past") with a deterministic check + one retry — far more reliable for the small
456
- * model than the open-ended fold. Every rewrite is provenance-preserving (keeps
457
- * its `[^k]` markers, dedupes duplicates). Runs NORMALIZE + the gate; a veto keeps
458
- * the original. A consistent article is a byte-identical NO-OP. Pure w.r.t. disk.
459
- */
460
- export async function reconcileContent(article: string, deps: ReconcileDeps = {}): Promise<ReconcileResult> {
461
- const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
462
- const title = articleTitle(article, deps.stem ?? "");
463
- const stem = deps.stem ?? entityStem(title);
464
- const stems = new Set(deps.stems ?? []);
465
- stems.add(stem);
466
-
467
- const noop: ReconcileResult = {
468
- content: article,
469
- changed: false,
470
- refreshedFields: [],
471
- rewrittenSections: [],
472
- gateVetoed: false,
473
- reason: null,
474
- normalizeNotes: [],
475
- unresolved: [],
476
- retried: false,
477
- historyEntries: [],
478
- };
479
-
480
- // The verdict — the latest-dated position — lives in the LEAD. With no lead
481
- // there is nothing to resolve the infobox against.
482
- const verdict = parseLead(article);
483
- if (!verdict || !verdict.trim()) return { ...noop, reason: "no lead verdict to reconcile against" };
484
-
485
- let next = article;
486
- const refreshedFields: string[] = [];
487
- const rewrittenSections: string[] = [];
488
- const unresolved: string[] = [];
489
- let retried = false;
490
-
491
- // Provenance lookups read from the ORIGINAL article (markers/hashes/dates are
492
- // stable there before any rewrite moves them around). Markers are found in the
493
- // PROSE only — the infobox value precedes the body, so scanning it would mis-
494
- // attribute a value to the lead's citation.
495
- const prose = article.replace(/```info[\s\S]*?```/i, "");
496
- const refDates = referenceDatesByMarker(article);
497
- const fnState = articleFootnoteState(article);
498
- const hashByMarker = new Map<number, string>();
499
- for (const [h, n] of fnState.byHash) hashByMarker.set(n, h);
500
-
501
- // (1) INFOBOX relationship fields — resolve any that contradict the verdict.
502
- // Each resolved field carries a GROUND-TRUTH value; a refreshed one also yields a
503
- // KNOWN superseded (old) value, the precise correction to propagate into the prose.
504
- // CLASSIFY each contradiction: a relationship/opinion/stance the user CHANGED over
505
- // time is an EVOLUTION (keep the trajectory in `## History`); a wrong world-fact /
506
- // spec is an ERROR (silently overwrite, prune the wrong value entirely).
507
- interface Correction {
508
- groundTruth: string;
509
- superseded: string;
510
- kind: ContradictionKind;
511
- earlierMarker: number | null;
512
- laterMarker: number | null;
513
- laterDate: string | null;
514
- }
515
- const corrections: Correction[] = [];
516
- const groundTruthOnly: string[] = []; // resolved values with no known old value (KEEP)
517
- const box = parseInfobox(article);
518
- if (box) {
519
- for (const f of box.fields) {
520
- if (!isRelationshipKey(f.key)) continue; // world-fact / spec → untouched
521
- const old = f.value.trim();
522
- const reply = await call(buildInfoboxReconcilePrompt(verdict, f.key, f.value), { maxTokens: MAX_OUTPUT_TOKENS });
523
- const cleaned = cleanValue(reply, f.key);
524
- if (cleaned && !/^keep$/i.test(cleaned) && cleaned !== old) {
525
- next = setInfoboxFieldValue(next, f.key, cleaned);
526
- refreshedFields.push(f.key);
527
- const kind = classifyContradiction(
528
- await call(buildContradictionClassifierPrompt(old, cleaned), { maxTokens: MAX_OUTPUT_TOKENS }),
529
- );
530
- const earlierMarker = markerNear(prose, old);
531
- const laterMarker = markerNear(prose, cleaned);
532
- corrections.push({
533
- groundTruth: cleaned,
534
- superseded: old,
535
- kind,
536
- earlierMarker,
537
- laterMarker,
538
- laterDate: laterMarker != null ? refDates.get(laterMarker) ?? null : null,
539
- });
540
- } else {
541
- groundTruthOnly.push(old); // already current — superseded value unknown
542
- }
543
- }
544
- }
545
-
546
- // (2) PROSE — the LEAD and every body section must agree with the ground truth.
547
- // The LEAD is held to a CLEAN CURRENT-STATE summary (no "previously …" caveats);
548
- // an ERROR correction strips the wrong value from EVERY passage; an EVOLUTION
549
- // keeps the body's in-context trajectory and consolidates it into `## History`.
550
- for (const passage of collectPassages(next)) {
551
- let body = passage.body;
552
- let touched = false;
553
- const isLead = passage.label === "lead";
554
-
555
- // (2a) KNOWN corrections (infobox refresh old→new): deterministic + reliable.
556
- for (const corr of corrections) {
557
- const live: Passage = { ...passage, body };
558
- const currentStateOnly = isLead || corr.kind === "error";
559
- const fix = currentStateOnly
560
- ? await reconcileOnePassageCurrentState(call, live, corr.groundTruth, corr.superseded, corr.kind === "error")
561
- : await reconcileOnePassage(call, live, corr.groundTruth, corr.superseded);
562
- retried ||= fix.retried;
563
- if (fix.unresolved) unresolved.push(passage.label);
564
- if (fix.changed) {
565
- body = fix.body;
566
- touched = true;
567
- }
568
- }
569
-
570
- // (2b) Ground-truth fields the infobox already had right (no known old value):
571
- // ask the model whether the prose names a DIFFERENT value as current, then run
572
- // the same constrained rewrite. Guarded by the deterministic check so a model
573
- // hallucination cannot trigger a needless rewrite.
574
- for (const gt of groundTruthOnly) {
575
- const detect = cleanValue(
576
- await call(buildProseConflictPrompt(passage.label, gt, body), { maxTokens: MAX_OUTPUT_TOKENS }),
577
- "",
578
- );
579
- if (!detect || /^keep$/i.test(detect) || detect.toLowerCase() === gt.toLowerCase()) continue;
580
- const live: Passage = { ...passage, body };
581
- const fix = await reconcileOnePassage(call, live, gt, detect);
582
- retried ||= fix.retried;
583
- if (fix.unresolved) unresolved.push(passage.label);
584
- if (fix.changed) {
585
- body = fix.body;
586
- touched = true;
587
- }
588
- }
589
-
590
- if (touched && body !== passage.body) {
591
- next = passage.write(next, body);
592
- rewrittenSections.push(passage.label);
593
- }
594
- }
595
-
596
- // (3) HISTORY — for EVOLUTION corrections, PRESERVE the trajectory in a
597
- // `## History` section (we keep the history). For ERROR corrections, the wrong
598
- // value's citation may have been pruned everywhere — allow the gate to drop it.
599
- const historyEntries: string[] = [];
600
- const allowDroppedHashes = new Set<string>();
601
- for (const corr of corrections) {
602
- if (corr.kind === "evolution") {
603
- const entry = buildHistoryEntry({
604
- earlier: corr.superseded,
605
- earlierMarker: corr.earlierMarker,
606
- later: corr.groundTruth,
607
- laterMarker: corr.laterMarker,
608
- laterDate: corr.laterDate,
609
- });
610
- const withHistory = upsertHistoryEntry(next, entry);
611
- if (withHistory !== next) {
612
- next = withHistory;
613
- historyEntries.push(entry);
614
- }
615
- } else if (corr.earlierMarker != null) {
616
- // A factual error: the wrong value (and only it) is allowed to lose its cite.
617
- const h = hashByMarker.get(corr.earlierMarker);
618
- if (h) allowDroppedHashes.add(h);
619
- }
620
- }
621
-
622
- if (!refreshedFields.length && !rewrittenSections.length && !historyEntries.length) {
623
- return { ...noop, unresolved, retried }; // already consistent (or unrepairable)
624
- }
625
-
626
- // NORMALIZE then the conservative gate over (original, reconciled).
627
- const norm = normalizeArticle(next, { stem, stems });
628
- if (norm.content === article) return { ...noop, normalizeNotes: norm.notes, unresolved, retried };
629
- const verdictGate = gateEdit(article, norm.content, { allowDroppedHashes });
630
- if (!verdictGate.ok) {
631
- return { ...noop, gateVetoed: true, reason: verdictGate.reason, normalizeNotes: norm.notes, unresolved, retried };
632
- }
633
-
634
- return {
635
- content: norm.content,
636
- changed: true,
637
- refreshedFields,
638
- rewrittenSections,
639
- gateVetoed: false,
640
- reason: null,
641
- normalizeNotes: norm.notes,
642
- unresolved,
643
- retried,
644
- historyEntries,
645
- };
646
- }
647
-
648
- // ---- effectful entry: reconcile a vault article + ledger -------------------
649
-
650
- export interface ReconcileArticleOpts {
651
- /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
652
- root?: string;
653
- /** Model-call override (tests inject a fake). */
654
- call?: SynthesisCall;
655
- /** Skip the git commit (tests). */
656
- commit?: boolean;
657
- now?: number;
658
- }
659
-
660
- export interface ReconcileArticleResult {
661
- stem: string;
662
- reconciled: boolean;
663
- refreshedFields: string[];
664
- rewrittenSections: string[];
665
- gateVetoed: boolean;
666
- reason: string | null;
667
- /** Passages the model could not demote even after the retry (honest reporting). */
668
- unresolved: string[];
669
- /** True when at least one rewrite needed the sharper retry. */
670
- retried: boolean;
671
- }
672
-
673
- /**
674
- * Reconcile a vault article on disk and persist it: read the file, run
675
- * {@link reconcileContent}, write back + record a `reconciled_articles` ledger row
676
- * when it changed, and commit (unless suppressed). A consistent article writes
677
- * nothing. Used by SYNTHESIZE after the PATCH loop for every article it touched.
678
- */
679
- export async function reconcileArticle(
680
- store: MemoryStore,
681
- stem: string,
682
- opts: ReconcileArticleOpts = {},
683
- ): Promise<ReconcileArticleResult> {
684
- const root = opts.root ?? vaultRoot();
685
- const base: ReconcileArticleResult = {
686
- stem,
687
- reconciled: false,
688
- refreshedFields: [],
689
- rewrittenSections: [],
690
- gateVetoed: false,
691
- reason: null,
692
- unresolved: [],
693
- retried: false,
694
- };
695
-
696
- const path = join(articlesDir(root), `${stem}.md`);
697
- let article: string;
698
- try {
699
- article = await readFile(path, "utf8");
700
- } catch {
701
- return { ...base, reason: `article not found: ${stem}` };
702
- }
703
-
704
- const stems = new Set(await listArticles(root));
705
- const res = await reconcileContent(article, { call: opts.call, stems, stem });
706
- if (!res.changed) {
707
- return { ...base, gateVetoed: res.gateVetoed, reason: res.reason, unresolved: res.unresolved, retried: res.retried };
708
- }
709
-
710
- await writeFile(path, res.content);
711
-
712
- const now = opts.now ?? Date.now();
713
- store.db
714
- .query(
715
- "INSERT OR REPLACE INTO reconciled_articles (article_stem, reconciled_at, refreshed, rewritten) VALUES (?,?,?,?)",
716
- )
717
- .run(stem, now, res.refreshedFields.join(","), res.rewrittenSections.join(","));
718
-
719
- if (opts.commit !== false) {
720
- await commitVault(
721
- root,
722
- `memory: reconcile ${stem} (${res.refreshedFields.length} field(s), ${res.rewrittenSections.length} section(s))`,
723
- );
724
- }
725
-
726
- return {
727
- ...base,
728
- reconciled: true,
729
- refreshedFields: res.refreshedFields,
730
- rewrittenSections: res.rewrittenSections,
731
- unresolved: res.unresolved,
732
- retried: res.retried,
733
- };
734
- }