mlx-bun 0.5.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +1449 -109
  2. package/THIRD_PARTY_NOTICES.md +882 -0
  3. package/bin/mlx-bun.convert.mjs +4 -0
  4. package/bin/mlx-bun.fuse.mjs +4 -0
  5. package/bin/mlx-bun.generate.mjs +4 -0
  6. package/bin/mlx-bun.lora.mjs +4 -0
  7. package/bin/mlx-bun.mjs +13 -21
  8. package/bin/mlx-bun.server.mjs +4 -0
  9. package/bin/mlx-bun.upload.mjs +4 -0
  10. package/dist/web/app.js +9584 -0
  11. package/package.json +50 -53
  12. package/src/cli/ane-install.ts +76 -0
  13. package/src/cli/args.ts +179 -0
  14. package/src/cli/engine-entry.ts +68 -0
  15. package/src/cli/inference.ts +214 -0
  16. package/src/cli/job-entry.ts +97 -0
  17. package/src/cli/library-host.ts +117 -0
  18. package/src/cli/main.ts +57 -0
  19. package/src/cli/memory-engine.ts +163 -0
  20. package/src/cli/memory.ts +36 -0
  21. package/src/cli/mlx-lm-aliases.ts +309 -0
  22. package/src/cli/model-choice.ts +33 -0
  23. package/src/cli/model-selection.ts +84 -0
  24. package/src/cli/module-host.ts +11 -0
  25. package/src/cli/module-verbs.ts +50 -0
  26. package/src/cli/numerical-policy.ts +12 -0
  27. package/src/cli/perplexity.ts +89 -0
  28. package/src/cli/serve-host.ts +423 -0
  29. package/src/cli/serve-isolated.ts +217 -0
  30. package/src/cli/serve-options.ts +106 -0
  31. package/src/cli/serve-state.ts +205 -0
  32. package/src/cli/serve.ts +500 -0
  33. package/src/cli/served-model-host.ts +102 -0
  34. package/src/cli/server-entry.ts +148 -0
  35. package/src/cli/serving-unit.ts +177 -0
  36. package/src/{tui.ts → cli/terminal.ts} +13 -8
  37. package/src/cli/worker-adapters.ts +34 -0
  38. package/src/cli/worker-entry.ts +272 -0
  39. package/src/cli/worker-unit.ts +179 -0
  40. package/src/engine/adapter-operation.ts +26 -0
  41. package/src/engine/cache-services.ts +114 -0
  42. package/src/engine/completion.ts +39 -0
  43. package/src/engine/generation-gateway.ts +358 -0
  44. package/src/engine/index.ts +59 -0
  45. package/src/engine/kv-budget.ts +120 -0
  46. package/src/engine/media-preparation.ts +232 -0
  47. package/src/engine/model-binding.ts +24 -0
  48. package/src/engine/model-host.ts +513 -0
  49. package/src/engine/model-serving.ts +89 -0
  50. package/src/{serve → engine}/preparation.ts +7 -5
  51. package/src/{serve → engine}/session-completion-engine.ts +6 -6
  52. package/src/engine/telemetry.ts +109 -0
  53. package/src/hub/downloads.ts +116 -0
  54. package/src/jobs/db.ts +16 -7
  55. package/src/jobs/events.ts +29 -0
  56. package/src/jobs/executable.ts +5 -0
  57. package/src/jobs/host.ts +51 -0
  58. package/src/jobs/in-process.ts +55 -0
  59. package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
  60. package/src/jobs/runner.ts +150 -231
  61. package/src/jobs/service.ts +75 -0
  62. package/src/jobs/sse.ts +59 -23
  63. package/src/jobs/worker-memory.ts +46 -0
  64. package/src/jobs/worker-process.ts +163 -0
  65. package/src/jobs/worker-supervisor.ts +258 -0
  66. package/src/modules.ts +58 -0
  67. package/src/publishing/catalog-hub.ts +18 -0
  68. package/src/publishing/credentials.ts +31 -0
  69. package/src/publishing/upload.ts +37 -0
  70. package/src/residency/leased-adapters.ts +21 -0
  71. package/src/residency/local-records.ts +16 -0
  72. package/src/residency/model-residency.ts +405 -0
  73. package/src/residency/record-index.ts +28 -0
  74. package/src/residency/resident-estimate.ts +50 -0
  75. package/src/{anthropic.ts → server/anthropic.ts} +3 -81
  76. package/src/server/cache-routes.ts +36 -0
  77. package/src/{serve → server}/chat-request.ts +10 -10
  78. package/src/{serve → server}/chat-stage.ts +14 -14
  79. package/src/{client.ts → server/client.ts} +28 -2
  80. package/src/{serve → server}/completion-executor.ts +33 -41
  81. package/src/server/decision-body.ts +55 -0
  82. package/src/server/decision-routes.ts +134 -0
  83. package/src/server/decision-wire.ts +228 -0
  84. package/src/{serve → server}/discovery-routes.ts +25 -82
  85. package/src/{serve → server}/http.ts +25 -13
  86. package/src/{serve → server}/inference-request.ts +5 -5
  87. package/src/server/job-routes.ts +20 -0
  88. package/src/server/media-prompt.ts +63 -0
  89. package/src/server/memory-completion-client.ts +46 -0
  90. package/src/server/model-routes.ts +192 -0
  91. package/src/{serve → server}/openai-wire.ts +3 -2
  92. package/src/{serve → server}/pipeline.ts +1 -1
  93. package/src/server/prompt-builder.ts +13 -0
  94. package/src/server/prompt-contracts.ts +17 -0
  95. package/src/server/proxy-routes.ts +187 -0
  96. package/src/server/publishing-routes.ts +41 -0
  97. package/src/{serve → server}/request-plan.ts +6 -8
  98. package/src/{serve → server}/request-prep.ts +47 -50
  99. package/src/{serve → server}/responses-client.ts +30 -13
  100. package/src/{responses.ts → server/responses.ts} +12 -93
  101. package/src/server/routes.ts +205 -0
  102. package/src/server/start.ts +72 -0
  103. package/src/server/status-routes.ts +108 -0
  104. package/src/{serve → server}/text-completion-stage.ts +25 -4
  105. package/src/server/text-prompt.ts +12 -0
  106. package/src/{serve → server}/token-streams.ts +36 -38
  107. package/src/server/worker-routes.ts +311 -0
  108. package/src/storage/paths.ts +61 -0
  109. package/src/web/assets.ts +37 -0
  110. package/src/web/{src → browser}/api.ts +5 -23
  111. package/src/web/browser/chat-host.ts +17 -0
  112. package/src/web/browser/installed-panels.ts +6 -0
  113. package/src/web/browser/main.ts +55 -0
  114. package/src/web/browser/markdown.ts +279 -0
  115. package/src/web/browser/memory-panel.ts +9 -0
  116. package/src/web/browser/palette.ts +64 -0
  117. package/src/web/browser/protocol.ts +13 -0
  118. package/src/web/browser/shell.ts +467 -0
  119. package/src/web/{src → browser}/status.ts +32 -36
  120. package/src/web/build.ts +51 -0
  121. package/src/web/public/app.html +1054 -0
  122. package/src/web/{sw.js → public/sw.js} +4 -3
  123. package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
  124. package/src/web/{vendor → public/vendor}/hljs.js +2 -2
  125. package/THIRD_PARTY_LICENSES.md +0 -51
  126. package/docs/README.md +0 -107
  127. package/docs/archive/hlg-sampling.md +0 -350
  128. package/docs/archive/investigations/audio-input-plan.md +0 -356
  129. package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +0 -3617
  130. package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
  131. package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
  132. package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
  133. package/docs/archive/investigations/curve-bisector-routing.md +0 -123
  134. package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
  135. package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
  136. package/docs/archive/investigations/dspark-handoff.md +0 -224
  137. package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
  138. package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
  139. package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
  140. package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
  141. package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
  142. package/docs/archive/investigations/lab-build-journal.md +0 -157
  143. package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
  144. package/docs/archive/investigations/openjarvis-fit.md +0 -198
  145. package/docs/archive/investigations/openwiki-evaluation.md +0 -249
  146. package/docs/archive/investigations/optimization_plan.md +0 -346
  147. package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
  148. package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
  149. package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
  150. package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
  151. package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
  152. package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
  153. package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
  154. package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
  155. package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
  156. package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
  157. package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
  158. package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
  159. package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
  160. package/docs/archive/planning/chat-ui-vision.md +0 -50
  161. package/docs/archive/planning/chunk-task-roadmap.md +0 -77
  162. package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
  163. package/docs/archive/planning/journal.md +0 -316
  164. package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
  165. package/docs/archive/planning/pi-integration-review.md +0 -178
  166. package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
  167. package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
  168. package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
  169. package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
  170. package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
  171. package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
  172. package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
  173. package/docs/archive/planning/release-notes-v0.4.0.md +0 -45
  174. package/docs/archive/planning/release-notes-v0.5.0.md +0 -37
  175. package/docs/archive/planning/web-ui-pass-plan.md +0 -213
  176. package/docs/archive/planning/website-readme-pass-plan.md +0 -224
  177. package/docs/design/batching.md +0 -2054
  178. package/docs/design/decode-speed-program.md +0 -2404
  179. package/docs/design/dreaming-nightly-pipeline.md +0 -684
  180. package/docs/design/generic-model-support.md +0 -821
  181. package/docs/design/kv-cache.md +0 -993
  182. package/docs/design/orpo-training.md +0 -680
  183. package/docs/design/speculative-decoding.md +0 -2129
  184. package/docs/design/turboquant.md +0 -2225
  185. package/docs/design/unified-engine-frontier-plan.md +0 -1987
  186. package/docs/design/web-chat-redesign.md +0 -695
  187. package/docs/planning/PRODUCT_ROADMAP.md +0 -593
  188. package/docs/planning/ResearchTopics.md +0 -119
  189. package/docs/planning/public-readiness-handoff.md +0 -244
  190. package/docs/planning/public-readiness.md +0 -149
  191. package/docs/planning/video-series.md +0 -421
  192. package/docs/reference/benchmarks.md +0 -3587
  193. package/docs/reference/cli.md +0 -646
  194. package/docs/reference/distribution.md +0 -380
  195. package/docs/reference/environment.md +0 -494
  196. package/docs/reference/glossary.md +0 -39
  197. package/docs/reference/library-api.md +0 -688
  198. package/docs/reference/memory.md +0 -260
  199. package/docs/reference/models.md +0 -346
  200. package/docs/reference/server-api.md +0 -1945
  201. package/docs/reference/server-config.md +0 -973
  202. package/docs/reference/training.md +0 -462
  203. package/docs/reference/troubleshooting.md +0 -158
  204. package/src/admission-plan.ts +0 -18
  205. package/src/audio/audiotoolbox.ts +0 -112
  206. package/src/audio/conformer.ts +0 -655
  207. package/src/audio/decode.ts +0 -133
  208. package/src/audio/features.ts +0 -202
  209. package/src/audio/mic-capture.ts +0 -98
  210. package/src/audio/silero-vad.ts +0 -379
  211. package/src/audio/transcode.ts +0 -67
  212. package/src/audio/whisper-decode.ts +0 -774
  213. package/src/audio/whisper-fast.ts +0 -732
  214. package/src/audio/whisper-format.ts +0 -73
  215. package/src/audio/whisper-mel.ts +0 -185
  216. package/src/audio/whisper-timing.ts +0 -274
  217. package/src/audio/whisper-tokenizer.ts +0 -239
  218. package/src/audio/whisper-transcribe.ts +0 -410
  219. package/src/backends/mlx/assistant-target.ts +0 -46
  220. package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
  221. package/src/backends/mlx/autoregressive.ts +0 -129
  222. package/src/backends/mlx/batch-group.ts +0 -1874
  223. package/src/backends/mlx/cache-attachment.ts +0 -67
  224. package/src/backends/mlx/cache-layout.ts +0 -31
  225. package/src/backends/mlx/checkpoint-state.ts +0 -53
  226. package/src/backends/mlx/continuation-persistence.ts +0 -107
  227. package/src/backends/mlx/continuation-request.ts +0 -43
  228. package/src/backends/mlx/continuation.ts +0 -77
  229. package/src/backends/mlx/diffusion.ts +0 -39
  230. package/src/backends/mlx/draft-target.ts +0 -32
  231. package/src/backends/mlx/encoder-cache.ts +0 -25
  232. package/src/backends/mlx/evaluation.ts +0 -105
  233. package/src/backends/mlx/fill-append.ts +0 -58
  234. package/src/backends/mlx/fill-group.ts +0 -497
  235. package/src/backends/mlx/full-kv-row-append.ts +0 -32
  236. package/src/backends/mlx/full-kv-row-donor.ts +0 -54
  237. package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
  238. package/src/backends/mlx/gateway-binding.ts +0 -174
  239. package/src/backends/mlx/graph.ts +0 -38
  240. package/src/backends/mlx/kv-codec.ts +0 -9
  241. package/src/backends/mlx/kv-maintenance.ts +0 -161
  242. package/src/backends/mlx/kv-tensor-rows.ts +0 -212
  243. package/src/backends/mlx/memory-client.ts +0 -189
  244. package/src/backends/mlx/methods.ts +0 -90
  245. package/src/backends/mlx/mixed-iteration.ts +0 -62
  246. package/src/backends/mlx/model-prompt.ts +0 -178
  247. package/src/backends/mlx/model-serving.ts +0 -80
  248. package/src/backends/mlx/paged-cache-codec.ts +0 -27
  249. package/src/backends/mlx/prefill-cohort.ts +0 -100
  250. package/src/backends/mlx/prefill-policy.ts +0 -26
  251. package/src/backends/mlx/prefill-rows.ts +0 -244
  252. package/src/backends/mlx/prefill.ts +0 -33
  253. package/src/backends/mlx/projected-context-rows.ts +0 -72
  254. package/src/backends/mlx/prompt-input.ts +0 -29
  255. package/src/backends/mlx/qwen-prompt-input.ts +0 -40
  256. package/src/backends/mlx/request-policy.ts +0 -20
  257. package/src/backends/mlx/request-state-policy.ts +0 -54
  258. package/src/backends/mlx/rollback.ts +0 -92
  259. package/src/backends/mlx/row-sampling.ts +0 -49
  260. package/src/backends/mlx/serial-executor.ts +0 -251
  261. package/src/backends/mlx/speculative-group.ts +0 -438
  262. package/src/backends/mlx/speculative-round.ts +0 -161
  263. package/src/backends/mlx/speculative.ts +0 -131
  264. package/src/backends/mlx/state-rows.ts +0 -104
  265. package/src/backends/mlx/state-views.ts +0 -46
  266. package/src/backends/mlx/target-layout-capability.ts +0 -35
  267. package/src/backends/mlx/text-engine.ts +0 -73
  268. package/src/backends/mlx/transitioning-kv-rows.ts +0 -173
  269. package/src/chat-template.ts +0 -382
  270. package/src/cli.ts +0 -3490
  271. package/src/config.ts +0 -399
  272. package/src/contracts/completion.ts +0 -16
  273. package/src/contracts/execution.ts +0 -70
  274. package/src/contracts/generation.ts +0 -103
  275. package/src/contracts/host.ts +0 -7
  276. package/src/contracts/object-cache.ts +0 -8
  277. package/src/contracts/pi-provider.ts +0 -30
  278. package/src/contracts/pi.ts +0 -261
  279. package/src/contracts/prefix-cache.ts +0 -33
  280. package/src/contracts/resources.ts +0 -25
  281. package/src/contracts/sampling.ts +0 -23
  282. package/src/contracts/scheduling.ts +0 -54
  283. package/src/contracts/tsconfig.json +0 -14
  284. package/src/dataset/generators.ts +0 -1087
  285. package/src/dataset/index.ts +0 -12
  286. package/src/dataset/job.ts +0 -81
  287. package/src/dataset/llm.ts +0 -71
  288. package/src/dataset/registry.ts +0 -528
  289. package/src/diffusion/diffusion-generate.ts +0 -507
  290. package/src/download.ts +0 -505
  291. package/src/embed.ts +0 -77
  292. package/src/engine/admission.ts +0 -59
  293. package/src/engine/cancellation.ts +0 -38
  294. package/src/engine/engine.ts +0 -56
  295. package/src/engine/execution-coordinator.ts +0 -70
  296. package/src/engine/execution-plan.ts +0 -64
  297. package/src/engine/execution-tasks.ts +0 -44
  298. package/src/engine/namespaced-cache.ts +0 -13
  299. package/src/engine/resources.ts +0 -53
  300. package/src/engine/scheduler.ts +0 -92
  301. package/src/engine/session.ts +0 -247
  302. package/src/eval/README.md +0 -130
  303. package/src/eval/ifeval.ts +0 -81
  304. package/src/eval/kl-prompts.ts +0 -29
  305. package/src/eval/kl.ts +0 -590
  306. package/src/eval/perplexity.ts +0 -164
  307. package/src/eval/quality-db.ts +0 -92
  308. package/src/eval/reference-logits.ts +0 -207
  309. package/src/eval/runner.ts +0 -124
  310. package/src/eval/score.ts +0 -49
  311. package/src/eval/tasks/bfcl.ts +0 -535
  312. package/src/eval/tasks/gsm8k.ts +0 -98
  313. package/src/eval/tasks/hashhop.ts +0 -139
  314. package/src/eval/tasks/humaneval.ts +0 -275
  315. package/src/eval/tasks/ifeval.ts +0 -525
  316. package/src/eval/tasks/mmlu.ts +0 -157
  317. package/src/eval/ultrafeedback.ts +0 -60
  318. package/src/evaldb.ts +0 -199
  319. package/src/expert-io.ts +0 -393
  320. package/src/expert-offload-build.ts +0 -77
  321. package/src/expert-offload.ts +0 -76
  322. package/src/expert-residency.ts +0 -1134
  323. package/src/expert-usage.ts +0 -415
  324. package/src/fill/echo-index.ts +0 -188
  325. package/src/fill/fill-session.ts +0 -463
  326. package/src/fill/proposal.ts +0 -68
  327. package/src/fill/schema-rows.ts +0 -506
  328. package/src/fill/tool-boundary.ts +0 -103
  329. package/src/fit.ts +0 -249
  330. package/src/generate.ts +0 -1352
  331. package/src/grammar.ts +0 -604
  332. package/src/harness-pi.ts +0 -184
  333. package/src/hf-push.ts +0 -628
  334. package/src/hub-rest.ts +0 -242
  335. package/src/index.ts +0 -63
  336. package/src/inference/denoising.ts +0 -17
  337. package/src/inference/draft-acceptance.ts +0 -41
  338. package/src/inference/draft-output.ts +0 -49
  339. package/src/inference/graph.ts +0 -33
  340. package/src/inference/prefill.ts +0 -45
  341. package/src/inference/rollback.ts +0 -7
  342. package/src/jobs/index.ts +0 -31
  343. package/src/jobs/job-entry.ts +0 -107
  344. package/src/jobs/task-client.ts +0 -19
  345. package/src/jobs/types.ts +0 -2
  346. package/src/kv-scheme.ts +0 -279
  347. package/src/kv-store.ts +0 -1135
  348. package/src/lab/README.md +0 -22
  349. package/src/lab/curve/curve-designer.html +0 -303
  350. package/src/lab/curve/curve-sampler.ts +0 -122
  351. package/src/lab/expert-trace/expert-trace.ts +0 -69
  352. package/src/lab/paged-kv/paged-attention.ts +0 -117
  353. package/src/lab/paged-kv/paged-kv-rows.ts +0 -111
  354. package/src/lab/paged-kv/paged-kv.ts +0 -336
  355. package/src/library.ts +0 -40
  356. package/src/lora.ts +0 -408
  357. package/src/md-modules.d.ts +0 -54
  358. package/src/media-fetch.ts +0 -442
  359. package/src/memory/article.ts +0 -558
  360. package/src/memory/chunk-validation.ts +0 -150
  361. package/src/memory/chunk.ts +0 -318
  362. package/src/memory/cluster.ts +0 -351
  363. package/src/memory/coverage.ts +0 -134
  364. package/src/memory/crosslink.ts +0 -0
  365. package/src/memory/db.ts +0 -253
  366. package/src/memory/entity.ts +0 -203
  367. package/src/memory/gate.ts +0 -187
  368. package/src/memory/history.ts +0 -152
  369. package/src/memory/ingest.ts +0 -335
  370. package/src/memory/model.ts +0 -183
  371. package/src/memory/normalize.ts +0 -671
  372. package/src/memory/parse.ts +0 -22
  373. package/src/memory/pipeline.ts +0 -275
  374. package/src/memory/prompts.ts +0 -65
  375. package/src/memory/query.ts +0 -334
  376. package/src/memory/reconcile.ts +0 -734
  377. package/src/memory/redirect.ts +0 -108
  378. package/src/memory/reindex.ts +0 -325
  379. package/src/memory/resolve.ts +0 -307
  380. package/src/memory/rest.ts +0 -245
  381. package/src/memory/route.ts +0 -272
  382. package/src/memory/schedule.ts +0 -165
  383. package/src/memory/stages.ts +0 -777
  384. package/src/memory/synthesize.ts +0 -1638
  385. package/src/memory/tools.ts +0 -749
  386. package/src/memory/vault.ts +0 -1194
  387. package/src/memory/wikify.ts +0 -1064
  388. package/src/mlx/array.ts +0 -425
  389. package/src/mlx/autograd.ts +0 -304
  390. package/src/mlx/checkpoint.ts +0 -149
  391. package/src/mlx/compile.ts +0 -168
  392. package/src/mlx/custom-vjp.ts +0 -188
  393. package/src/mlx/ffi.ts +0 -531
  394. package/src/mlx/materialize.ts +0 -23
  395. package/src/mlx/metal-kernel.ts +0 -159
  396. package/src/mlx/normalized-argmax.ts +0 -116
  397. package/src/mlx/ops.ts +0 -1108
  398. package/src/mlx/quantized-rows.ts +0 -36
  399. package/src/mlx/token-bitmask.ts +0 -52
  400. package/src/mlx/turboquant-kv-decode.ts +0 -82
  401. package/src/mlx/turboquant-kv-inverse.ts +0 -122
  402. package/src/mlx/turboquant-ops.ts +0 -861
  403. package/src/mlx/turboquant-tables.ts +0 -237
  404. package/src/mmap.ts +0 -90
  405. package/src/model/artifact-identity.ts +0 -73
  406. package/src/model/batched-kv.ts +0 -235
  407. package/src/model/batched-mask.ts +0 -279
  408. package/src/model/batched-quant.ts +0 -212
  409. package/src/model/batched-quantized-kv.ts +0 -144
  410. package/src/model/batched-rotating-quant.ts +0 -353
  411. package/src/model/batched-rotating-state.ts +0 -149
  412. package/src/model/batched-rotating.ts +0 -338
  413. package/src/model/batched-row-storage.ts +0 -179
  414. package/src/model/batched-ssm.ts +0 -52
  415. package/src/model/batched-turboquant-kv.ts +0 -94
  416. package/src/model/compiled-decode.ts +0 -712
  417. package/src/model/decoded-kv-donor.ts +0 -12
  418. package/src/model/delayed-quantized-kv.ts +0 -72
  419. package/src/model/delayed-rotating-quantized-kv.ts +0 -93
  420. package/src/model/delayed-turboquant-kv.ts +0 -57
  421. package/src/model/diffusion-gemma.ts +0 -1026
  422. package/src/model/factory.ts +0 -215
  423. package/src/model/fingerprint.ts +0 -45
  424. package/src/model/flash-attention.ts +0 -694
  425. package/src/model/full-prefill-padding.ts +0 -56
  426. package/src/model/full-prefill-row.ts +0 -105
  427. package/src/model/gemma4-base.ts +0 -2370
  428. package/src/model/gemma4.ts +0 -1395
  429. package/src/model/generated/gemma4-12b.ts +0 -429
  430. package/src/model/generated/gemma4-26b.ts +0 -448
  431. package/src/model/generated/gemma4-e4b.ts +0 -717
  432. package/src/model/generated/index.ts +0 -18
  433. package/src/model/glm52-atlas.ts +0 -511
  434. package/src/model/glm52-cache.ts +0 -792
  435. package/src/model/glm52-config.ts +0 -262
  436. package/src/model/glm52-container.ts +0 -305
  437. package/src/model/glm52-coupling.ts +0 -388
  438. package/src/model/glm52-dsa.ts +0 -606
  439. package/src/model/glm52-expert-layout.ts +0 -253
  440. package/src/model/glm52-layout.ts +0 -152
  441. package/src/model/glm52-memory.ts +0 -500
  442. package/src/model/glm52-mla.ts +0 -792
  443. package/src/model/glm52-moe.ts +0 -223
  444. package/src/model/glm52-pilot.ts +0 -537
  445. package/src/model/glm52-reference.ts +0 -445
  446. package/src/model/glm52-residency.ts +0 -557
  447. package/src/model/glm52-resident-weights.ts +0 -135
  448. package/src/model/glm52-streamed-experts.ts +0 -453
  449. package/src/model/glm52-streamed-metal.ts +0 -610
  450. package/src/model/glm52-weights.ts +0 -503
  451. package/src/model/glm52.ts +0 -1132
  452. package/src/model/implementation.ts +0 -69
  453. package/src/model/kv-attention-view.ts +0 -70
  454. package/src/model/minicpm5.ts +0 -376
  455. package/src/model/profile.ts +0 -468
  456. package/src/model/quantized-append-attention.ts +0 -31
  457. package/src/model/qwen-append.ts +0 -12
  458. package/src/model/qwen-conv.ts +0 -52
  459. package/src/model/qwen3-delta-state.ts +0 -74
  460. package/src/model/qwen3-delta.ts +0 -451
  461. package/src/model/qwen3-moe.ts +0 -404
  462. package/src/model/qwen3-mrope.ts +0 -250
  463. package/src/model/qwen3.ts +0 -277
  464. package/src/model/qwen3_5-checkpoint.ts +0 -164
  465. package/src/model/qwen3_5.ts +0 -881
  466. package/src/model/rotating-kv-donor.ts +0 -35
  467. package/src/model/rotating-kv-layout.ts +0 -262
  468. package/src/model/rotating-row-transaction.ts +0 -35
  469. package/src/model/speculative-rotating-kv.ts +0 -101
  470. package/src/model/ssm-prefill-padding.ts +0 -39
  471. package/src/model/support.ts +0 -110
  472. package/src/model/token-groups.ts +0 -66
  473. package/src/model/trellis-balanced-scatter.ts +0 -51
  474. package/src/model/trellis-linear.ts +0 -672
  475. package/src/model/trellis-shared-m.ts +0 -151
  476. package/src/model/trellis-shared-scatter.ts +0 -144
  477. package/src/model/trellis-splitk-prefill.ts +0 -158
  478. package/src/model/trellis-tiled-prefill.ts +0 -136
  479. package/src/model/trellis-vector-expand.ts +0 -62
  480. package/src/model/trellis-wide-prefill.ts +0 -94
  481. package/src/model/turboquant-codec.ts +0 -89
  482. package/src/model/universal/archs.ts +0 -332
  483. package/src/model/universal/dense.ts +0 -612
  484. package/src/model/universal/modules.ts +0 -170
  485. package/src/model/universal/rope.ts +0 -381
  486. package/src/model/whisper.ts +0 -435
  487. package/src/native/expert_io.c +0 -523
  488. package/src/native/frame_extract.swift +0 -140
  489. package/src/native/mic_capture.swift +0 -92
  490. package/src/native-pack.ts +0 -163
  491. package/src/pi-provider.ts +0 -53
  492. package/src/pi-session.ts +0 -81
  493. package/src/pi-terminal.ts +0 -274
  494. package/src/pi-web.ts +0 -1940
  495. package/src/preflight.ts +0 -158
  496. package/src/prompt-cache.ts +0 -682
  497. package/src/quantize/allocator.ts +0 -627
  498. package/src/quantize/atomic-output.ts +0 -38
  499. package/src/quantize/calibration.ts +0 -181
  500. package/src/quantize/config-writer.ts +0 -145
  501. package/src/quantize/index.ts +0 -84
  502. package/src/quantize/job.ts +0 -162
  503. package/src/quantize/quantizer.ts +0 -666
  504. package/src/quantize/rotate.ts +0 -603
  505. package/src/quantize/safetensors-writer.ts +0 -249
  506. package/src/quantize/sensitivity.ts +0 -216
  507. package/src/quantize/trellis.ts +0 -586
  508. package/src/quantize/weight-transform.ts +0 -292
  509. package/src/registry.ts +0 -602
  510. package/src/runtime-config.ts +0 -77
  511. package/src/safetensors.ts +0 -165
  512. package/src/sampler.ts +0 -1012
  513. package/src/serve/admin-routes.ts +0 -184
  514. package/src/serve/assets/curve-terrain.html +0 -105
  515. package/src/serve/assets/training-inference-map.html +0 -575
  516. package/src/serve/audio-routes.ts +0 -323
  517. package/src/serve/aux-routes.ts +0 -132
  518. package/src/serve/batch-scheduler.ts +0 -8
  519. package/src/serve/checkpoint-identity.ts +0 -64
  520. package/src/serve/generation-gateway.ts +0 -537
  521. package/src/serve/isolate.ts +0 -609
  522. package/src/serve/kv-budget.ts +0 -14
  523. package/src/serve/lab-routes.ts +0 -341
  524. package/src/serve/lane-registry.ts +0 -45
  525. package/src/serve/model-admin-routes.ts +0 -155
  526. package/src/serve/model-binding.ts +0 -49
  527. package/src/serve/model-host.ts +0 -511
  528. package/src/serve/parent-application.ts +0 -56
  529. package/src/serve/prompt-response-trace.ts +0 -161
  530. package/src/serve/response-lease.ts +0 -24
  531. package/src/serve/session-search.ts +0 -326
  532. package/src/serve/static-routes.ts +0 -101
  533. package/src/serve/transcription-server.ts +0 -64
  534. package/src/serve/transcription-service.ts +0 -502
  535. package/src/server.ts +0 -1282
  536. package/src/spec/assistant-rows.ts +0 -93
  537. package/src/spec/assistant-source.ts +0 -126
  538. package/src/spec/cache-identity.ts +0 -22
  539. package/src/spec/deepspec-rows.ts +0 -29
  540. package/src/spec/deepspec-source.ts +0 -145
  541. package/src/spec/dflash-source.ts +0 -149
  542. package/src/spec/drafter.ts +0 -304
  543. package/src/spec/dspark/ab-stats.ts +0 -136
  544. package/src/spec/dspark/calibration.ts +0 -112
  545. package/src/spec/dspark/data-dflash.ts +0 -156
  546. package/src/spec/dspark/data.ts +0 -201
  547. package/src/spec/dspark/deepspec-module.ts +0 -776
  548. package/src/spec/dspark/generate-dflash.ts +0 -245
  549. package/src/spec/dspark/generate.ts +0 -290
  550. package/src/spec/dspark/loader.ts +0 -23
  551. package/src/spec/dspark/loss.ts +0 -213
  552. package/src/spec/dspark/module-dflash.ts +0 -560
  553. package/src/spec/dspark/module.ts +0 -540
  554. package/src/spec/dspark/quantize-drafter.ts +0 -91
  555. package/src/spec/dspark/sample.ts +0 -112
  556. package/src/spec/generate.ts +0 -276
  557. package/src/spec/glm52-mtp-graph.ts +0 -29
  558. package/src/spec/glm52-mtp-rows.ts +0 -132
  559. package/src/spec/glm52-mtp-source.ts +0 -318
  560. package/src/spec/glm52-mtp-state.ts +0 -33
  561. package/src/spec/ngram-source.ts +0 -226
  562. package/src/spec/projected-draft-rows.ts +0 -73
  563. package/src/spec/qwen-mtp-module.ts +0 -171
  564. package/src/spec/qwen-mtp-rows.ts +0 -230
  565. package/src/spec/qwen-mtp-source.ts +0 -386
  566. package/src/spec/qwen-mtp-state.ts +0 -30
  567. package/src/spec/serve-loop.ts +0 -587
  568. package/src/spec/source.ts +0 -228
  569. package/src/spec/two-model-rows.ts +0 -62
  570. package/src/spec/two-model.ts +0 -113
  571. package/src/ssd-cache.ts +0 -431
  572. package/src/ssd-durability.ts +0 -209
  573. package/src/storage/host-buffer.ts +0 -24
  574. package/src/storage/kv-writer.ts +0 -83
  575. package/src/storage/kv-writer.worker.js +0 -264
  576. package/src/storage/retention-policy.ts +0 -37
  577. package/src/tiered-prompt-cache.ts +0 -45
  578. package/src/tokenizer.ts +0 -335
  579. package/src/tool-approvals.ts +0 -131
  580. package/src/tool-call.ts +0 -695
  581. package/src/train/dataset.ts +0 -484
  582. package/src/train/diffusion-lora.ts +0 -216
  583. package/src/train/export.ts +0 -32
  584. package/src/train/flash-cce.ts +0 -1234
  585. package/src/train/forward.ts +0 -266
  586. package/src/train/fuse.ts +0 -243
  587. package/src/train/index.ts +0 -46
  588. package/src/train/job.ts +0 -205
  589. package/src/train/lora-params.ts +0 -254
  590. package/src/train/loss.ts +0 -1591
  591. package/src/train/merge.ts +0 -195
  592. package/src/train/optimizer.ts +0 -183
  593. package/src/train/prefix-shared.ts +0 -490
  594. package/src/train/rank.ts +0 -137
  595. package/src/train/segmented.ts +0 -1828
  596. package/src/train/steel-qmm-header.ts +0 -1461
  597. package/src/train/trainer.ts +0 -1221
  598. package/src/train/watch.ts +0 -542
  599. package/src/vision/diffusion-vision.ts +0 -620
  600. package/src/vision/embedder.ts +0 -176
  601. package/src/vision/preprocess.ts +0 -247
  602. package/src/vision/prompt.ts +0 -409
  603. package/src/vision/qwen3vl-preprocess.ts +0 -409
  604. package/src/vision/qwen3vl-prompt.ts +0 -233
  605. package/src/vision/qwen3vl-tower.ts +0 -566
  606. package/src/vision/siglip.ts +0 -607
  607. package/src/vision/video-frames.ts +0 -148
  608. package/src/web/app.html +0 -2155
  609. package/src/web/app.js +0 -5863
  610. package/src/web/skills/memory/SKILL.md +0 -120
  611. package/src/web/skills/web-research/SKILL.md +0 -46
  612. package/src/web/skills.ts +0 -97
  613. package/src/web/src/adapters-panel.ts +0 -288
  614. package/src/web/src/assistant.ts +0 -451
  615. package/src/web/src/chat.ts +0 -1060
  616. package/src/web/src/composer.ts +0 -962
  617. package/src/web/src/dataset.ts +0 -132
  618. package/src/web/src/finetune.ts +0 -267
  619. package/src/web/src/hub.ts +0 -428
  620. package/src/web/src/main.ts +0 -98
  621. package/src/web/src/markdown.ts +0 -0
  622. package/src/web/src/memory-panel.ts +0 -572
  623. package/src/web/src/model-picker.ts +0 -159
  624. package/src/web/src/palette.ts +0 -346
  625. package/src/web/src/protocol.ts +0 -26
  626. package/src/web/src/quantize.ts +0 -167
  627. package/src/web/src/rag.ts +0 -274
  628. package/src/web/src/sessions.ts +0 -469
  629. package/src/web/src/shell.ts +0 -804
  630. package/src/web/src/ui-catalog.ts +0 -78
  631. package/src/web/src/voice.ts +0 -141
  632. package/src/web/vendor/README.md +0 -61
  633. package/src/web-assets.ts +0 -47
  634. package/src/web-tools.ts +0 -478
  635. package/src/weights.ts +0 -255
  636. /package/src/{serve → server}/completion-sink.ts +0 -0
  637. /package/src/{serve → server}/generated-token-history.ts +0 -0
  638. /package/src/web/{icon.svg → public/icon.svg} +0 -0
  639. /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
  640. /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
@@ -1,734 +0,0 @@
1
- // mlx-bun memory — RECONCILE pass (P7-T3): make an article internally consistent
2
- // with its latest-dated position AFTER a self-healing fold.
3
- //
4
- // A targeted PATCH resolves ONE claim toward the user's newest statement — it
5
- // flips the LEAD (the verdict) to the current value. But the rest of the article
6
- // can be left contradicting that verdict: an INFOBOX relationship field still
7
- // records the superseded value (`<preference_key>: <old value>` while the lead now
8
- // says `<new value>`), and a sibling BODY section still asserts the old take. This pass
9
- // runs AFTER the patch loop, scans those surfaces, and updates whatever the fold
10
- // made stale so the whole article ends consistent with the latest position.
11
- //
12
- // It is the same discipline as PATCH: every edit goes through NORMALIZE + the
13
- // conservative gate, and it is PROVENANCE-PRESERVING — a section rewrite keeps
14
- // every pre-existing `[^k]` marker (phrased as a change over time), so the gate's
15
- // citation-survival check passes instead of NO-OP'ing the fold. It is bounded
16
- // (only relationship infobox fields + body sections are touched; world-fact / spec
17
- // fields are left alone) and idempotent: a fully-consistent article is a NO-OP.
18
-
19
- import { readFile, writeFile } from "node:fs/promises";
20
- import { join } from "node:path";
21
-
22
- import { articleStructure, fenceLabel, isFenceLine, parseInfobox, parseLead } from "./article";
23
- import { articleTitle } from "./cluster";
24
- import type { MemoryStore } from "./db";
25
- import { gateEdit } from "./gate";
26
- import {
27
- buildContradictionClassifierPrompt,
28
- buildHistoryEntry,
29
- classifyContradiction,
30
- type ContradictionKind,
31
- markerNear,
32
- upsertHistoryEntry,
33
- } from "./history";
34
- import { callLocal, MAX_OUTPUT_TOKENS } from "./model";
35
- import { normalizeArticle } from "./normalize";
36
- import {
37
- articleFootnoteState,
38
- entityStem,
39
- PHYSICAL_SPEC_KEYS,
40
- referenceDatesByMarker,
41
- sanitizeLead,
42
- sanitizeSection,
43
- type SynthesisCall,
44
- } from "./synthesize";
45
- import { articlesDir, commitVault, extractSection, listArticles, slugifyHeading, vaultRoot } from "./vault";
46
-
47
- // ---- relationship-field classification -------------------------------------
48
-
49
- /** Infobox keys we NEVER touch — schema keys plus the physical spec keys (a
50
- * spec is a world-fact, not a stance that a verdict can supersede). */
51
- const RECONCILE_SKIP_KEYS = new Set(["type", "kind", "aliases", "categories"]);
52
-
53
- /** A relationship/opinion key whose value records the user's stance — the only
54
- * infobox fields a verdict change can make stale (favorite / opinion / status /
55
- * preference / current pick …). World-fact + spec keys are excluded above. */
56
- const RELATIONSHIP_KEY_RE =
57
- /(favou?rite|prefer|preference|pick|choice|chosen|opinion|verdict|status|stance|current|rating|sentiment|\btake\b|likes?|dislikes?|using|used_for|owns?|owned|acquired)/i;
58
-
59
- /** True when an infobox field is a relationship/stance fact (refreshable), not a
60
- * world-fact or physical spec (left untouched). */
61
- export function isRelationshipKey(key: string): boolean {
62
- const k = key.trim().toLowerCase();
63
- if (RECONCILE_SKIP_KEYS.has(k)) return false;
64
- if (PHYSICAL_SPEC_KEYS.has(k)) return false;
65
- return RELATIONSHIP_KEY_RE.test(k);
66
- }
67
-
68
- // ---- prompt builders (schematic — no parrot-able concrete values) ----------
69
-
70
- /** The model sentinel for "this fact already agrees with the summary". */
71
- export const RECONCILE_KEEP = "KEEP";
72
-
73
- /** Check ONE infobox relationship fact against the current verdict (the lead). The
74
- * model answers KEEP when it agrees, or the corrected VALUE (no key, no prose)
75
- * when it is stale. Schematic on purpose. */
76
- export function buildInfoboxReconcilePrompt(verdict: string, key: string, value: string): string {
77
- return (
78
- `You are checking ONE infobox fact against the article's current summary for internal consistency.\n\n` +
79
- `CURRENT SUMMARY (the user's latest position):\n${verdict}\n\n` +
80
- `INFOBOX FACT:\n${key}: ${value}\n\n` +
81
- `If the fact AGREES with the current summary, answer exactly ${RECONCILE_KEEP}. ` +
82
- `If it CONTRADICTS the summary (it records a preference, opinion, or status the summary has since changed), ` +
83
- `answer with the corrected VALUE only — the single value that matches the summary, no key, no prose, no quotes.`
84
- );
85
- }
86
-
87
- /** Detection (used only for a ground-truth field with no KNOWN superseded value):
88
- * does this passage state, as the user's CURRENT choice, a value that DIFFERS
89
- * from the established current value? Answer KEEP, or that different value only. */
90
- export function buildProseConflictPrompt(label: string, groundTruth: string, body: string): string {
91
- return (
92
- `ESTABLISHED CURRENT VALUE: ${groundTruth}\n\n` +
93
- `${label}:\n${body || "(empty)"}\n\n` +
94
- `Does this passage state, as the user's CURRENT / present choice, a value DIFFERENT from ${groundTruth}? ` +
95
- `If it states only ${groundTruth} as current (any other value appears solely as past history), answer exactly ${RECONCILE_KEEP}. ` +
96
- `Otherwise answer with that different current value ONLY — no key, no prose, no quotes.`
97
- );
98
- }
99
-
100
- /**
101
- * The CONSTRAINED consistency rewrite — the safety net that guarantees the prose
102
- * agrees with the infobox GROUND TRUTH. Both values are real article data (the
103
- * resolved current value + the value it replaced), not copyable example values:
104
- * we tell the model exactly which value is current and which is former, so the
105
- * small model only has to demote the named superseded value to past tense — far
106
- * more reliable than the open-ended "resolve toward latest" fold. Every existing
107
- * `[^k]` marker is preserved (provenance), framed as a change over time.
108
- */
109
- export function buildProseConsistencyPrompt(
110
- label: string,
111
- groundTruth: string,
112
- superseded: string,
113
- body: string,
114
- ): string {
115
- return (
116
- `You are correcting ONE passage of a personal-wiki article so it agrees with the article's ESTABLISHED CURRENT fact.\n\n` +
117
- `ESTABLISHED CURRENT VALUE: ${groundTruth}\n` +
118
- `FORMER (SUPERSEDED) VALUE: ${superseded}\n\n` +
119
- `${label}:\n${body || "(empty)"}\n\n` +
120
- `Rewrite the passage so it reads as a change OVER TIME: state ${superseded} ONLY in the past tense ` +
121
- `("previously ${superseded}", "originally ${superseded}", "${superseded} was"), and state ${groundTruth} as the user's CURRENT, present choice. ` +
122
- `${superseded} must NEVER appear as a current/present favourite, pick, or "the one I reach for". ` +
123
- `Keep EVERY existing [^k] citation marker: attach the ${superseded} marker to the past clause and the ${groundTruth} marker to the present clause, so no [^N] is lost. ` +
124
- `Prefer ONE flowing statement — "previously ${superseded}[^k]; now ${groundTruth} is my current pick[^n]". ` +
125
- `Output ONLY the rewritten passage prose — no heading, no list, no References section.`
126
- );
127
- }
128
-
129
- /**
130
- * The CURRENT-STATE-ONLY rewrite. Used for the LEAD (which must read as a clean
131
- * current-state summary — never scattered "previously …" caveats) and for a
132
- * factual-ERROR fix in any passage (the wrong value is removed as if it had never
133
- * been there). The passage is rewritten to state ONLY `groundTruth` as current and
134
- * to DROP `superseded` entirely. For an evolution the dropped value's trajectory
135
- * is recorded in the article's `## History` section instead; for an error it is
136
- * simply gone. Both values are real article data, not copyable example values.
137
- */
138
- export function buildCurrentStateOnlyPrompt(
139
- label: string,
140
- groundTruth: string,
141
- superseded: string,
142
- body: string,
143
- isError: boolean,
144
- ): string {
145
- const why = isError
146
- ? `The value ${superseded} was a MISTAKE — a wrong fact. Remove it entirely; it must NOT appear anywhere in the passage (no "previously", no history). ` +
147
- `You may drop ${superseded}'s citation marker.`
148
- : `Do NOT mention ${superseded} in this passage — its trajectory is recorded separately in the History section. ` +
149
- `Keep ${groundTruth}'s [^k] citation marker.`;
150
- return (
151
- `You are rewriting ONE passage of a personal-wiki article to state ONLY the user's CURRENT value.\n\n` +
152
- `CURRENT VALUE: ${groundTruth}\n` +
153
- `VALUE TO REMOVE: ${superseded}\n\n` +
154
- `${label}:\n${body || "(empty)"}\n\n` +
155
- `Rewrite the passage so it states ${groundTruth} as the user's current, present value, in the present tense. ` +
156
- `${why} ` +
157
- `Output ONLY the rewritten passage prose — no heading, no list, no References section.`
158
- );
159
- }
160
-
161
- /** Sharper retry instruction when the first rewrite still asserts the superseded
162
- * value as current. */
163
- export function proseRetryHint(groundTruth: string, superseded: string): string {
164
- return (
165
- `STILL WRONG: the passage must NOT claim ${superseded} is current, now, or a favourite. ` +
166
- `The ONLY current value is ${groundTruth}. Write ${superseded} strictly as a FORMER value ("previously ${superseded}[^k]"), ` +
167
- `and ${groundTruth} as the present one ("now ${groundTruth}[^n]"). Demote ${superseded} to the past tense.`
168
- );
169
- }
170
-
171
- // ---- pure helpers ----------------------------------------------------------
172
-
173
- /** Distinct `[^N]` markers in a block (definition lines excluded). */
174
- function markersIn(body: string): number[] {
175
- const out: number[] = [];
176
- const seen = new Set<number>();
177
- for (const line of body.split("\n")) {
178
- if (/^\[\^\d+\]:/.test(line.trim())) continue;
179
- for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
180
- const n = parseInt(m[1]!, 10);
181
- if (!seen.has(n)) {
182
- seen.add(n);
183
- out.push(n);
184
- }
185
- }
186
- }
187
- return out;
188
- }
189
-
190
- /** Escape a literal string for embedding in a RegExp. */
191
- function escapeRe(s: string): string {
192
- return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
193
- }
194
-
195
- /** Past-tense framing cues — when one governs the clause that mentions a value,
196
- * the value reads as HISTORY, not a present assertion. Deliberately EXCLUDES
197
- * "currently"/"now"/"today" (those are PRESENT, the very thing we flag). */
198
- const PAST_FRAME_RE =
199
- /\b(previous|origin|earlier|used to|no longer|formerly|initial|at first|back then|before|had been|until|supersed|was|were|moved on|switched)\b/i;
200
-
201
- /**
202
- * True when `value` is asserted as the user's CURRENT / present choice somewhere
203
- * in `text` and that clause is NOT framed as past. A "clause" is a span between
204
- * sentence / `;` breaks, so "previously <old>[^1]; now <new>[^2]" reads the <old>
205
- * clause as history (past-framed) and is NOT flagged. This is the deterministic
206
- * detector AND the post-rewrite acceptance check the retry hinges on. Pure.
207
- */
208
- export function assertsValueAsCurrent(text: string, value: string): boolean {
209
- const t = text.toLowerCase();
210
- const v = value.trim().toLowerCase();
211
- if (!v || !t.includes(v)) return false;
212
- const ev = escapeRe(v);
213
- const present = new RegExp(
214
- `(${ev}[^.;!?]{0,60}\\b(is|are|remain|stay|favou?rite|prefer|reach for|number-one|pick|go-to|current|now)\\b)|` +
215
- `(\\b(favou?rite|prefer|current(ly)?|now|pick|reach for|go-to|reach|consider)\\b[^.;!?]{0,60}${ev})`,
216
- );
217
- for (const clause of t.split(/[.;!?]+/)) {
218
- if (!clause.includes(v)) continue;
219
- if (!present.test(clause)) continue;
220
- if (PAST_FRAME_RE.test(clause)) continue; // framed as history → fine
221
- return true; // present-tense, not past-framed → stale
222
- }
223
- return false;
224
- }
225
-
226
- /** Collapse adjacent duplicate footnote markers (`[^2][^2]` or `[^2] [^2]` →
227
- * `[^2]`) — the lead artifact a fold can leave. Pure. */
228
- export function dedupeAdjacentMarkers(text: string): string {
229
- return text.replace(/(\[\^\d+\])(\s*\1)+/g, "$1");
230
- }
231
-
232
- /** Strip a model value reply to a clean single-line value (drop quotes, a stray
233
- * echoed `key:` prefix, surrounding markup). */
234
- function cleanValue(raw: string, key: string): string {
235
- let v = raw.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
236
- v = v.replace(new RegExp(`^${key}\\s*:`, "i"), "").trim();
237
- v = v.replace(/^[-*+]\s+/, "").replace(/^["'`]+|["'`.]+$/g, "").trim();
238
- return v;
239
- }
240
-
241
- /** Replace the value of one `key:` line inside the article's fenced ```info block.
242
- * Returns the article unchanged when the key is absent. Pure. */
243
- export function setInfoboxFieldValue(content: string, key: string, newValue: string): string {
244
- const lines = content.split(/\r?\n/);
245
- let inInfo = false;
246
- for (let i = 0; i < lines.length; i++) {
247
- const line = lines[i]!;
248
- if (isFenceLine(line)) {
249
- if (!inInfo && fenceLabel(line) === "info") inInfo = true;
250
- else if (inInfo) break; // closing fence — leave the info block
251
- continue;
252
- }
253
- if (inInfo) {
254
- const idx = line.indexOf(":");
255
- if (idx > 0 && line.slice(0, idx).trim() === key) {
256
- lines[i] = `${key}: ${newValue}`;
257
- return lines.join("\n");
258
- }
259
- }
260
- }
261
- return content;
262
- }
263
-
264
- // ---- the reconcile core (pure w.r.t. disk) ---------------------------------
265
-
266
- export interface ReconcileDeps {
267
- /** Model call; defaults to `callLocal("synthesis", …)`. */
268
- call?: SynthesisCall;
269
- /** Other article stems (wikilink resolution in NORMALIZE). */
270
- stems?: Set<string>;
271
- /** Article stem (H1/NORMALIZE); defaults to the entity stem of the H1 title. */
272
- stem?: string;
273
- }
274
-
275
- export interface ReconcileResult {
276
- content: string;
277
- changed: boolean;
278
- /** Infobox keys whose value was refreshed to the current verdict. */
279
- refreshedFields: string[];
280
- /** Passage labels rewritten toward the ground-truth value (`lead` + section anchors). */
281
- rewrittenSections: string[];
282
- /** True when NORMALIZE+gate vetoed the reconciled article (kept the original). */
283
- gateVetoed: boolean;
284
- reason: string | null;
285
- normalizeNotes: string[];
286
- /** Passage labels where, even after the retry, the small model could NOT demote
287
- * the superseded value to past tense (left as the gate-vetoed original). Logged
288
- * honestly — these are a base-model limit, not a silent success. */
289
- unresolved: string[];
290
- /** True when at least one passage rewrite needed the sharper retry. */
291
- retried: boolean;
292
- /** Trajectory lines written to the `## History` section for EVOLUTION
293
- * corrections (the user's position changing over time). Empty when every
294
- * contradiction was a factual error (silently overwritten, no History). */
295
- historyEntries: string[];
296
- }
297
-
298
- /** One prose-passage to reconcile: the LEAD or a body section. */
299
- interface Passage {
300
- /** Stable label for `rewrittenSections` / logs (`lead` or the section anchor). */
301
- label: string;
302
- /** Current passage body (no heading). */
303
- body: string;
304
- /** Sanitizer for a model rewrite (section vs lead). */
305
- sanitize: (raw: string) => string;
306
- /** Splice a rewritten body back into the article. */
307
- write: (article: string, newBody: string) => string;
308
- }
309
-
310
- /** Collect the reconcilable prose passages (lead + every body section, skipping
311
- * References / See also / Notes) with the helpers to read+write each. */
312
- function collectPassages(article: string): Passage[] {
313
- const out: Passage[] = [];
314
- const items = articleStructure(article);
315
-
316
- const lead = items.find((it) => it.kind === "lead");
317
- if (lead) {
318
- const lines = article.split("\n");
319
- const start = lead.startLine - 1;
320
- const end = lead.endLine - 1;
321
- const body = lines.slice(start, end + 1).join("\n").trim();
322
- if (body) {
323
- out.push({
324
- label: "lead",
325
- body,
326
- sanitize: sanitizeLead,
327
- write: (a, nb) => {
328
- const ls = a.split("\n");
329
- return [...ls.slice(0, start), nb, ...ls.slice(end + 1)].join("\n");
330
- },
331
- });
332
- }
333
- }
334
-
335
- for (const item of items) {
336
- if (item.kind !== "section" || !item.title) continue;
337
- const anchor = slugifyHeading(item.title);
338
- const block = extractSection(article, anchor);
339
- if (block == null) continue;
340
- const headingLine = block.split("\n")[0] ?? `## ${item.title}`;
341
- const body = block.split("\n").slice(1).join("\n").trim();
342
- if (!body) continue;
343
- const title = item.title;
344
- out.push({
345
- label: anchor,
346
- body,
347
- sanitize: (raw) => sanitizeSection(raw, title),
348
- write: (a, nb) => {
349
- const cur = extractSection(a, anchor);
350
- if (cur == null) return a;
351
- const idx = a.indexOf(cur);
352
- if (idx < 0) return a;
353
- return a.slice(0, idx) + `${headingLine}\n\n${nb}\n` + a.slice(idx + cur.length);
354
- },
355
- });
356
- }
357
- return out;
358
- }
359
-
360
- /** Outcome of one constrained passage rewrite. */
361
- interface PassageFix {
362
- body: string;
363
- changed: boolean;
364
- retried: boolean;
365
- /** True when the small model still left the superseded value as current AFTER
366
- * the retry (or dropped a marker) — kept the original; report honestly. */
367
- unresolved: boolean;
368
- }
369
-
370
- /**
371
- * Make ONE passage agree that `groundTruth` is current and `superseded` is former.
372
- * No-op when the passage does not assert `superseded` as current (idempotent).
373
- * Otherwise: constrained rewrite → deterministic check → ONE sharper retry; if it
374
- * still asserts the stale value as current OR would drop a `[^k]` marker, keep the
375
- * original and flag `unresolved`. Markers are deduped; provenance is preserved.
376
- */
377
- async function reconcileOnePassage(
378
- call: SynthesisCall,
379
- passage: Passage,
380
- groundTruth: string,
381
- superseded: string,
382
- ): Promise<PassageFix> {
383
- const keep: PassageFix = { body: passage.body, changed: false, retried: false, unresolved: false };
384
- if (!superseded || !groundTruth || superseded.toLowerCase() === groundTruth.toLowerCase()) return keep;
385
- if (!assertsValueAsCurrent(passage.body, superseded)) return keep; // already clean / absent
386
-
387
- const before = new Set(markersIn(passage.body));
388
- const attempt = async (prompt: string): Promise<string> => {
389
- const raw = await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS });
390
- return dedupeAdjacentMarkers(passage.sanitize(raw)).trim();
391
- };
392
- const accept = (x: string): boolean => {
393
- if (!x) return false;
394
- if (assertsValueAsCurrent(x, superseded)) return false; // still asserts stale value as current
395
- const after = new Set(markersIn(x));
396
- return ![...before].some((m) => !after.has(m)); // every marker preserved (provenance)
397
- };
398
-
399
- const base = buildProseConsistencyPrompt(passage.label, groundTruth, superseded, passage.body);
400
- let body = await attempt(base);
401
- if (accept(body)) return { body, changed: true, retried: false, unresolved: false };
402
-
403
- body = await attempt(`${base}\n\n${proseRetryHint(groundTruth, superseded)}`);
404
- if (accept(body)) return { body, changed: true, retried: true, unresolved: false };
405
-
406
- return { ...keep, retried: true, unresolved: true }; // base-model limit — keep original
407
- }
408
-
409
- /**
410
- * Make ONE passage state ONLY `groundTruth` as current and DROP `superseded`
411
- * entirely (the LEAD path, and the factual-ERROR path). No-op when the passage
412
- * does not mention `superseded`. Otherwise: constrained rewrite → deterministic
413
- * check (superseded gone, groundTruth present) → ONE sharper retry; if it still
414
- * mentions `superseded`, keep the original and flag `unresolved`. For an evolution
415
- * the dropped value's `[^k]` is preserved in the History section (added by the
416
- * caller); for an error it is allowed to fall away.
417
- */
418
- async function reconcileOnePassageCurrentState(
419
- call: SynthesisCall,
420
- passage: Passage,
421
- groundTruth: string,
422
- superseded: string,
423
- isError: boolean,
424
- ): Promise<PassageFix> {
425
- const keep: PassageFix = { body: passage.body, changed: false, retried: false, unresolved: false };
426
- if (!superseded || !groundTruth || superseded.toLowerCase() === groundTruth.toLowerCase()) return keep;
427
- if (!passage.body.toLowerCase().includes(superseded.toLowerCase())) return keep; // already clean
428
-
429
- const attempt = async (prompt: string): Promise<string> => {
430
- const raw = await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS });
431
- return dedupeAdjacentMarkers(passage.sanitize(raw)).trim();
432
- };
433
- const accept = (x: string): boolean => {
434
- if (!x) return false;
435
- if (x.toLowerCase().includes(superseded.toLowerCase())) return false; // value not removed
436
- return x.toLowerCase().includes(groundTruth.toLowerCase()); // current value present
437
- };
438
-
439
- const base = buildCurrentStateOnlyPrompt(passage.label, groundTruth, superseded, passage.body, isError);
440
- let body = await attempt(base);
441
- if (accept(body)) return { body, changed: true, retried: false, unresolved: false };
442
-
443
- body = await attempt(`${base}\n\n${proseRetryHint(groundTruth, superseded)}`);
444
- if (accept(body)) return { body, changed: true, retried: true, unresolved: false };
445
-
446
- return { ...keep, retried: true, unresolved: true };
447
- }
448
-
449
- /**
450
- * Reconcile ONE article to its INFOBOX GROUND TRUTH. The infobox relationship
451
- * value is the authority (it is resolved first, against the lead verdict): once
452
- * resolved, the LEAD and EVERY body section must AGREE with it. We refresh stale
453
- * infobox fields, then for each passage that still asserts a SUPERSEDED value as
454
- * current we run a CONSTRAINED rewrite ("make <ground-truth> current and <old>
455
- * past") with a deterministic check + one retry — far more reliable for the small
456
- * model than the open-ended fold. Every rewrite is provenance-preserving (keeps
457
- * its `[^k]` markers, dedupes duplicates). Runs NORMALIZE + the gate; a veto keeps
458
- * the original. A consistent article is a byte-identical NO-OP. Pure w.r.t. disk.
459
- */
460
- export async function reconcileContent(article: string, deps: ReconcileDeps = {}): Promise<ReconcileResult> {
461
- const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
462
- const title = articleTitle(article, deps.stem ?? "");
463
- const stem = deps.stem ?? entityStem(title);
464
- const stems = new Set(deps.stems ?? []);
465
- stems.add(stem);
466
-
467
- const noop: ReconcileResult = {
468
- content: article,
469
- changed: false,
470
- refreshedFields: [],
471
- rewrittenSections: [],
472
- gateVetoed: false,
473
- reason: null,
474
- normalizeNotes: [],
475
- unresolved: [],
476
- retried: false,
477
- historyEntries: [],
478
- };
479
-
480
- // The verdict — the latest-dated position — lives in the LEAD. With no lead
481
- // there is nothing to resolve the infobox against.
482
- const verdict = parseLead(article);
483
- if (!verdict || !verdict.trim()) return { ...noop, reason: "no lead verdict to reconcile against" };
484
-
485
- let next = article;
486
- const refreshedFields: string[] = [];
487
- const rewrittenSections: string[] = [];
488
- const unresolved: string[] = [];
489
- let retried = false;
490
-
491
- // Provenance lookups read from the ORIGINAL article (markers/hashes/dates are
492
- // stable there before any rewrite moves them around). Markers are found in the
493
- // PROSE only — the infobox value precedes the body, so scanning it would mis-
494
- // attribute a value to the lead's citation.
495
- const prose = article.replace(/```info[\s\S]*?```/i, "");
496
- const refDates = referenceDatesByMarker(article);
497
- const fnState = articleFootnoteState(article);
498
- const hashByMarker = new Map<number, string>();
499
- for (const [h, n] of fnState.byHash) hashByMarker.set(n, h);
500
-
501
- // (1) INFOBOX relationship fields — resolve any that contradict the verdict.
502
- // Each resolved field carries a GROUND-TRUTH value; a refreshed one also yields a
503
- // KNOWN superseded (old) value, the precise correction to propagate into the prose.
504
- // CLASSIFY each contradiction: a relationship/opinion/stance the user CHANGED over
505
- // time is an EVOLUTION (keep the trajectory in `## History`); a wrong world-fact /
506
- // spec is an ERROR (silently overwrite, prune the wrong value entirely).
507
- interface Correction {
508
- groundTruth: string;
509
- superseded: string;
510
- kind: ContradictionKind;
511
- earlierMarker: number | null;
512
- laterMarker: number | null;
513
- laterDate: string | null;
514
- }
515
- const corrections: Correction[] = [];
516
- const groundTruthOnly: string[] = []; // resolved values with no known old value (KEEP)
517
- const box = parseInfobox(article);
518
- if (box) {
519
- for (const f of box.fields) {
520
- if (!isRelationshipKey(f.key)) continue; // world-fact / spec → untouched
521
- const old = f.value.trim();
522
- const reply = await call(buildInfoboxReconcilePrompt(verdict, f.key, f.value), { maxTokens: MAX_OUTPUT_TOKENS });
523
- const cleaned = cleanValue(reply, f.key);
524
- if (cleaned && !/^keep$/i.test(cleaned) && cleaned !== old) {
525
- next = setInfoboxFieldValue(next, f.key, cleaned);
526
- refreshedFields.push(f.key);
527
- const kind = classifyContradiction(
528
- await call(buildContradictionClassifierPrompt(old, cleaned), { maxTokens: MAX_OUTPUT_TOKENS }),
529
- );
530
- const earlierMarker = markerNear(prose, old);
531
- const laterMarker = markerNear(prose, cleaned);
532
- corrections.push({
533
- groundTruth: cleaned,
534
- superseded: old,
535
- kind,
536
- earlierMarker,
537
- laterMarker,
538
- laterDate: laterMarker != null ? refDates.get(laterMarker) ?? null : null,
539
- });
540
- } else {
541
- groundTruthOnly.push(old); // already current — superseded value unknown
542
- }
543
- }
544
- }
545
-
546
- // (2) PROSE — the LEAD and every body section must agree with the ground truth.
547
- // The LEAD is held to a CLEAN CURRENT-STATE summary (no "previously …" caveats);
548
- // an ERROR correction strips the wrong value from EVERY passage; an EVOLUTION
549
- // keeps the body's in-context trajectory and consolidates it into `## History`.
550
- for (const passage of collectPassages(next)) {
551
- let body = passage.body;
552
- let touched = false;
553
- const isLead = passage.label === "lead";
554
-
555
- // (2a) KNOWN corrections (infobox refresh old→new): deterministic + reliable.
556
- for (const corr of corrections) {
557
- const live: Passage = { ...passage, body };
558
- const currentStateOnly = isLead || corr.kind === "error";
559
- const fix = currentStateOnly
560
- ? await reconcileOnePassageCurrentState(call, live, corr.groundTruth, corr.superseded, corr.kind === "error")
561
- : await reconcileOnePassage(call, live, corr.groundTruth, corr.superseded);
562
- retried ||= fix.retried;
563
- if (fix.unresolved) unresolved.push(passage.label);
564
- if (fix.changed) {
565
- body = fix.body;
566
- touched = true;
567
- }
568
- }
569
-
570
- // (2b) Ground-truth fields the infobox already had right (no known old value):
571
- // ask the model whether the prose names a DIFFERENT value as current, then run
572
- // the same constrained rewrite. Guarded by the deterministic check so a model
573
- // hallucination cannot trigger a needless rewrite.
574
- for (const gt of groundTruthOnly) {
575
- const detect = cleanValue(
576
- await call(buildProseConflictPrompt(passage.label, gt, body), { maxTokens: MAX_OUTPUT_TOKENS }),
577
- "",
578
- );
579
- if (!detect || /^keep$/i.test(detect) || detect.toLowerCase() === gt.toLowerCase()) continue;
580
- const live: Passage = { ...passage, body };
581
- const fix = await reconcileOnePassage(call, live, gt, detect);
582
- retried ||= fix.retried;
583
- if (fix.unresolved) unresolved.push(passage.label);
584
- if (fix.changed) {
585
- body = fix.body;
586
- touched = true;
587
- }
588
- }
589
-
590
- if (touched && body !== passage.body) {
591
- next = passage.write(next, body);
592
- rewrittenSections.push(passage.label);
593
- }
594
- }
595
-
596
- // (3) HISTORY — for EVOLUTION corrections, PRESERVE the trajectory in a
597
- // `## History` section (we keep the history). For ERROR corrections, the wrong
598
- // value's citation may have been pruned everywhere — allow the gate to drop it.
599
- const historyEntries: string[] = [];
600
- const allowDroppedHashes = new Set<string>();
601
- for (const corr of corrections) {
602
- if (corr.kind === "evolution") {
603
- const entry = buildHistoryEntry({
604
- earlier: corr.superseded,
605
- earlierMarker: corr.earlierMarker,
606
- later: corr.groundTruth,
607
- laterMarker: corr.laterMarker,
608
- laterDate: corr.laterDate,
609
- });
610
- const withHistory = upsertHistoryEntry(next, entry);
611
- if (withHistory !== next) {
612
- next = withHistory;
613
- historyEntries.push(entry);
614
- }
615
- } else if (corr.earlierMarker != null) {
616
- // A factual error: the wrong value (and only it) is allowed to lose its cite.
617
- const h = hashByMarker.get(corr.earlierMarker);
618
- if (h) allowDroppedHashes.add(h);
619
- }
620
- }
621
-
622
- if (!refreshedFields.length && !rewrittenSections.length && !historyEntries.length) {
623
- return { ...noop, unresolved, retried }; // already consistent (or unrepairable)
624
- }
625
-
626
- // NORMALIZE then the conservative gate over (original, reconciled).
627
- const norm = normalizeArticle(next, { stem, stems });
628
- if (norm.content === article) return { ...noop, normalizeNotes: norm.notes, unresolved, retried };
629
- const verdictGate = gateEdit(article, norm.content, { allowDroppedHashes });
630
- if (!verdictGate.ok) {
631
- return { ...noop, gateVetoed: true, reason: verdictGate.reason, normalizeNotes: norm.notes, unresolved, retried };
632
- }
633
-
634
- return {
635
- content: norm.content,
636
- changed: true,
637
- refreshedFields,
638
- rewrittenSections,
639
- gateVetoed: false,
640
- reason: null,
641
- normalizeNotes: norm.notes,
642
- unresolved,
643
- retried,
644
- historyEntries,
645
- };
646
- }
647
-
648
- // ---- effectful entry: reconcile a vault article + ledger -------------------
649
-
650
- export interface ReconcileArticleOpts {
651
- /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
652
- root?: string;
653
- /** Model-call override (tests inject a fake). */
654
- call?: SynthesisCall;
655
- /** Skip the git commit (tests). */
656
- commit?: boolean;
657
- now?: number;
658
- }
659
-
660
- export interface ReconcileArticleResult {
661
- stem: string;
662
- reconciled: boolean;
663
- refreshedFields: string[];
664
- rewrittenSections: string[];
665
- gateVetoed: boolean;
666
- reason: string | null;
667
- /** Passages the model could not demote even after the retry (honest reporting). */
668
- unresolved: string[];
669
- /** True when at least one rewrite needed the sharper retry. */
670
- retried: boolean;
671
- }
672
-
673
- /**
674
- * Reconcile a vault article on disk and persist it: read the file, run
675
- * {@link reconcileContent}, write back + record a `reconciled_articles` ledger row
676
- * when it changed, and commit (unless suppressed). A consistent article writes
677
- * nothing. Used by SYNTHESIZE after the PATCH loop for every article it touched.
678
- */
679
- export async function reconcileArticle(
680
- store: MemoryStore,
681
- stem: string,
682
- opts: ReconcileArticleOpts = {},
683
- ): Promise<ReconcileArticleResult> {
684
- const root = opts.root ?? vaultRoot();
685
- const base: ReconcileArticleResult = {
686
- stem,
687
- reconciled: false,
688
- refreshedFields: [],
689
- rewrittenSections: [],
690
- gateVetoed: false,
691
- reason: null,
692
- unresolved: [],
693
- retried: false,
694
- };
695
-
696
- const path = join(articlesDir(root), `${stem}.md`);
697
- let article: string;
698
- try {
699
- article = await readFile(path, "utf8");
700
- } catch {
701
- return { ...base, reason: `article not found: ${stem}` };
702
- }
703
-
704
- const stems = new Set(await listArticles(root));
705
- const res = await reconcileContent(article, { call: opts.call, stems, stem });
706
- if (!res.changed) {
707
- return { ...base, gateVetoed: res.gateVetoed, reason: res.reason, unresolved: res.unresolved, retried: res.retried };
708
- }
709
-
710
- await writeFile(path, res.content);
711
-
712
- const now = opts.now ?? Date.now();
713
- store.db
714
- .query(
715
- "INSERT OR REPLACE INTO reconciled_articles (article_stem, reconciled_at, refreshed, rewritten) VALUES (?,?,?,?)",
716
- )
717
- .run(stem, now, res.refreshedFields.join(","), res.rewrittenSections.join(","));
718
-
719
- if (opts.commit !== false) {
720
- await commitVault(
721
- root,
722
- `memory: reconcile ${stem} (${res.refreshedFields.length} field(s), ${res.rewrittenSections.length} section(s))`,
723
- );
724
- }
725
-
726
- return {
727
- ...base,
728
- reconciled: true,
729
- refreshedFields: res.refreshedFields,
730
- rewrittenSections: res.rewrittenSections,
731
- unresolved: res.unresolved,
732
- retried: res.retried,
733
- };
734
- }