akm-cli 0.9.0-beta.9 → 0.9.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (381) hide show
  1. package/CHANGELOG.md +715 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/help/help-improve.md +9 -6
  6. package/dist/assets/hints/cli-hints-full.md +6 -5
  7. package/dist/assets/profiles/default.json +9 -4
  8. package/dist/assets/profiles/frequent.json +1 -1
  9. package/dist/assets/profiles/memory-focus.json +1 -1
  10. package/dist/assets/profiles/proactive-maintenance.json +25 -0
  11. package/dist/assets/profiles/quick.json +1 -1
  12. package/dist/assets/profiles/recombine-only.json +21 -0
  13. package/dist/assets/profiles/reflect-distill.json +30 -0
  14. package/dist/assets/profiles/synthesize.json +15 -0
  15. package/dist/assets/profiles/thorough.json +1 -1
  16. package/dist/assets/prompts/consolidate-system.md +23 -0
  17. package/dist/assets/prompts/contradiction-judge.md +33 -0
  18. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  19. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  20. package/dist/assets/prompts/extract-session.md +11 -3
  21. package/dist/assets/prompts/graph-extract-system.md +1 -0
  22. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  23. package/dist/assets/prompts/memory-infer-system.md +1 -0
  24. package/dist/assets/prompts/memory-infer-user.md +5 -0
  25. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  26. package/dist/assets/prompts/procedural-system.md +44 -0
  27. package/dist/assets/prompts/recombine-system.md +40 -0
  28. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  29. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  30. package/dist/assets/prompts/workflow-unit-preamble.md +26 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  34. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  35. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  36. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  37. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  38. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  39. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  40. package/dist/assets/templates/html/health.html +281 -111
  41. package/dist/assets/wiki/ingest-workflow-template.md +45 -16
  42. package/dist/assets/wiki/schema-template.md +4 -4
  43. package/dist/cli/clack.js +56 -0
  44. package/dist/cli/config-migrate.js +7 -1
  45. package/dist/cli/confirm.js +1 -1
  46. package/dist/cli/parse-args.js +46 -1
  47. package/dist/cli/shared.js +28 -0
  48. package/dist/cli.js +25 -21
  49. package/dist/commands/agent/agent-dispatch.js +3 -2
  50. package/dist/commands/agent/agent-support.js +0 -7
  51. package/dist/commands/agent/contribute-cli.js +26 -7
  52. package/dist/commands/config-cli.js +26 -13
  53. package/dist/commands/env/child-env.js +47 -0
  54. package/dist/commands/env/env-binding.js +95 -0
  55. package/dist/commands/env/env-cli.js +228 -292
  56. package/dist/commands/env/env.js +14 -67
  57. package/dist/commands/env/secret-cli.js +140 -138
  58. package/dist/commands/feedback-cli.js +156 -155
  59. package/dist/commands/graph/graph-cli.js +5 -13
  60. package/dist/commands/graph/graph.js +3 -3
  61. package/dist/commands/health/advisories.js +151 -0
  62. package/dist/commands/health/checks.js +103 -16
  63. package/dist/commands/health/html-report.js +447 -81
  64. package/dist/commands/health/improve-metrics.js +771 -0
  65. package/dist/commands/health/llm-usage.js +65 -0
  66. package/dist/commands/health/md-report.js +103 -0
  67. package/dist/commands/health/metrics.js +278 -0
  68. package/dist/commands/health/stash-exposure.js +46 -0
  69. package/dist/commands/health/surfaces.js +216 -0
  70. package/dist/commands/health/task-runs.js +135 -0
  71. package/dist/commands/health/types.js +26 -0
  72. package/dist/commands/health/windows.js +195 -0
  73. package/dist/commands/health.js +91 -1091
  74. package/dist/commands/improve/anti-collapse.js +170 -0
  75. package/dist/commands/improve/calibration.js +161 -0
  76. package/dist/commands/improve/collapse-detector.js +421 -0
  77. package/dist/commands/improve/consolidate/chunking.js +141 -0
  78. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  79. package/dist/commands/improve/consolidate/merge.js +145 -0
  80. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  81. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  82. package/dist/commands/improve/consolidate.js +1295 -1277
  83. package/dist/commands/improve/dedup.js +482 -0
  84. package/dist/commands/improve/distill/content-repair.js +202 -0
  85. package/dist/commands/improve/distill/promote-memory.js +229 -0
  86. package/dist/commands/improve/distill/quality-gate.js +236 -0
  87. package/dist/commands/improve/distill-guards.js +127 -0
  88. package/dist/commands/improve/distill-promotion-policy.js +826 -167
  89. package/dist/commands/improve/distill.js +228 -605
  90. package/dist/commands/improve/eligibility.js +434 -0
  91. package/dist/commands/improve/encoding-salience.js +205 -0
  92. package/dist/commands/improve/extract-cli.js +179 -59
  93. package/dist/commands/improve/extract-prompt.js +54 -3
  94. package/dist/commands/improve/extract-watch.js +140 -0
  95. package/dist/commands/improve/extract.js +409 -43
  96. package/dist/commands/improve/feedback-valence.js +54 -0
  97. package/dist/commands/improve/hot-probation.js +45 -0
  98. package/dist/commands/improve/improve-auto-accept.js +157 -10
  99. package/dist/commands/improve/improve-cli.js +115 -73
  100. package/dist/commands/improve/improve-profiles.js +28 -8
  101. package/dist/commands/improve/improve-result-file.js +15 -25
  102. package/dist/commands/improve/improve-session.js +58 -0
  103. package/dist/commands/improve/improve.js +485 -2764
  104. package/dist/commands/improve/locks.js +154 -0
  105. package/dist/commands/improve/loop-stages.js +1100 -0
  106. package/dist/commands/improve/memory/memory-belief.js +14 -15
  107. package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
  108. package/dist/commands/improve/memory/memory-improve.js +27 -27
  109. package/dist/commands/improve/outcome-loop.js +270 -0
  110. package/dist/commands/improve/preparation.js +2002 -0
  111. package/dist/commands/improve/proactive-maintenance.js +37 -35
  112. package/dist/commands/improve/procedural.js +398 -0
  113. package/dist/commands/improve/recombine.js +818 -0
  114. package/dist/commands/improve/reflect-noise.js +0 -0
  115. package/dist/commands/improve/reflect.js +206 -45
  116. package/dist/commands/improve/salience.js +455 -0
  117. package/dist/commands/improve/schema-similarity-gate.js +168 -0
  118. package/dist/commands/improve/shared.js +51 -0
  119. package/dist/commands/improve/triage.js +93 -0
  120. package/dist/commands/lint/agent-linter.js +19 -24
  121. package/dist/commands/lint/base-linter.js +173 -60
  122. package/dist/commands/lint/command-linter.js +19 -24
  123. package/dist/commands/lint/env-key-rules.js +38 -1
  124. package/dist/commands/lint/fact-linter.js +39 -0
  125. package/dist/commands/lint/index.js +31 -13
  126. package/dist/commands/lint/memory-linter.js +1 -1
  127. package/dist/commands/lint/registry.js +7 -2
  128. package/dist/commands/lint/task-linter.js +3 -3
  129. package/dist/commands/lint/workflow-linter.js +26 -1
  130. package/dist/commands/observability-cli.js +4 -4
  131. package/dist/commands/proposal/drain-policies.js +13 -4
  132. package/dist/commands/proposal/drain.js +45 -51
  133. package/dist/commands/proposal/legacy-import.js +115 -0
  134. package/dist/commands/proposal/proposal-cli.js +24 -34
  135. package/dist/commands/proposal/proposal.js +2 -1
  136. package/dist/commands/proposal/propose.js +8 -3
  137. package/dist/commands/proposal/repository.js +829 -0
  138. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  139. package/dist/commands/proposal/validators/proposals.js +93 -895
  140. package/dist/commands/read/curate.js +410 -111
  141. package/dist/commands/read/knowledge.js +10 -3
  142. package/dist/commands/read/remember-cli.js +133 -138
  143. package/dist/commands/read/search-cli.js +15 -8
  144. package/dist/commands/read/search.js +22 -11
  145. package/dist/commands/read/show.js +106 -14
  146. package/dist/commands/registry-cli.js +76 -87
  147. package/dist/commands/remember.js +11 -12
  148. package/dist/commands/sources/add-cli.js +91 -95
  149. package/dist/commands/sources/history.js +1 -1
  150. package/dist/commands/sources/init.js +66 -18
  151. package/dist/commands/sources/installed-stashes.js +11 -3
  152. package/dist/commands/sources/migration-help.js +7 -4
  153. package/dist/commands/sources/schema-repair.js +44 -46
  154. package/dist/commands/sources/self-update.js +2 -2
  155. package/dist/commands/sources/source-add.js +7 -3
  156. package/dist/commands/sources/sources-cli.js +3 -3
  157. package/dist/commands/sources/stash-cli.js +19 -39
  158. package/dist/commands/sources/stash-skeleton.js +57 -8
  159. package/dist/commands/tasks/default-tasks.js +15 -2
  160. package/dist/commands/tasks/tasks-cli.js +20 -29
  161. package/dist/commands/tasks/tasks.js +39 -11
  162. package/dist/commands/wiki-cli.js +23 -38
  163. package/dist/commands/workflow-cli.js +291 -13
  164. package/dist/core/asset/asset-registry.js +3 -1
  165. package/dist/core/asset/asset-spec.js +79 -5
  166. package/dist/core/asset/frontmatter.js +188 -167
  167. package/dist/core/asset/markdown.js +8 -0
  168. package/dist/core/authoring-rules.js +92 -0
  169. package/dist/core/common.js +4 -23
  170. package/dist/core/concurrent.js +10 -1
  171. package/dist/core/config/config-io.js +10 -1
  172. package/dist/core/config/config-migration.js +18 -40
  173. package/dist/core/config/config-schema.js +403 -62
  174. package/dist/core/config/config-types.js +3 -3
  175. package/dist/core/config/config.js +67 -22
  176. package/dist/core/deep-merge.js +38 -0
  177. package/dist/core/errors.js +1 -0
  178. package/dist/core/eval/rank-metrics.js +113 -0
  179. package/dist/core/events.js +4 -7
  180. package/dist/core/improve-types.js +47 -8
  181. package/dist/core/json-schema.js +142 -0
  182. package/dist/core/logs-db.js +14 -75
  183. package/dist/core/parse.js +36 -16
  184. package/dist/core/paths.js +18 -18
  185. package/dist/core/standards/resolve-standards-context.js +87 -0
  186. package/dist/core/standards/resolve-stash-standards.js +99 -0
  187. package/dist/core/standards/resolve-type-conventions.js +66 -0
  188. package/dist/core/state/migrations.js +770 -0
  189. package/dist/core/state-db.js +132 -1126
  190. package/dist/core/structured.js +69 -0
  191. package/dist/core/time.js +53 -0
  192. package/dist/core/warn.js +21 -0
  193. package/dist/core/write-source.js +37 -0
  194. package/dist/indexer/db/db.js +261 -770
  195. package/dist/indexer/db/entry-mapper.js +41 -0
  196. package/dist/indexer/db/graph-db.js +129 -86
  197. package/dist/indexer/db/llm-cache.js +2 -2
  198. package/dist/indexer/db/schema.js +516 -0
  199. package/dist/indexer/ensure-index.js +36 -92
  200. package/dist/indexer/feedback/utility-policy.js +75 -0
  201. package/dist/indexer/graph/graph-boost.js +51 -41
  202. package/dist/indexer/graph/graph-extraction.js +207 -4
  203. package/dist/indexer/index-writer-lock.js +18 -11
  204. package/dist/indexer/index-written-assets.js +105 -0
  205. package/dist/indexer/indexer.js +182 -204
  206. package/dist/indexer/passes/dir-staleness.js +114 -0
  207. package/dist/indexer/passes/memory-inference.js +13 -5
  208. package/dist/indexer/passes/metadata.js +20 -0
  209. package/dist/indexer/read-preflight.js +23 -0
  210. package/dist/indexer/search/db-search.js +89 -13
  211. package/dist/indexer/search/fts-query.js +51 -0
  212. package/dist/indexer/search/ranking-contributors.js +95 -9
  213. package/dist/indexer/search/ranking.js +79 -3
  214. package/dist/indexer/search/search-fields.js +6 -0
  215. package/dist/indexer/search/search-source.js +32 -21
  216. package/dist/indexer/search/semantic-status.js +4 -0
  217. package/dist/indexer/walk/matchers.js +48 -0
  218. package/dist/indexer/walk/walker.js +21 -13
  219. package/dist/integrations/agent/builders.js +41 -13
  220. package/dist/integrations/agent/config.js +20 -59
  221. package/dist/integrations/agent/detect.js +9 -0
  222. package/dist/integrations/agent/index.js +3 -19
  223. package/dist/integrations/agent/model-aliases.js +16 -2
  224. package/dist/integrations/agent/profiles.js +79 -6
  225. package/dist/integrations/agent/prompts.js +75 -9
  226. package/dist/integrations/agent/runner-dispatch.js +83 -0
  227. package/dist/integrations/agent/runner.js +13 -9
  228. package/dist/integrations/agent/spawn.js +206 -81
  229. package/dist/integrations/harnesses/aider/agent-builder.js +113 -0
  230. package/dist/integrations/harnesses/aider/index.js +58 -0
  231. package/dist/integrations/harnesses/aider/result-extractor.js +53 -0
  232. package/dist/integrations/harnesses/amazonq/agent-builder.js +153 -0
  233. package/dist/integrations/harnesses/amazonq/index.js +59 -0
  234. package/dist/integrations/harnesses/amazonq/result-extractor.js +48 -0
  235. package/dist/integrations/harnesses/claude/agent-builder.js +46 -7
  236. package/dist/integrations/harnesses/claude/index.js +27 -23
  237. package/dist/integrations/harnesses/claude/result-extractor.js +52 -0
  238. package/dist/integrations/harnesses/claude/session-log.js +10 -0
  239. package/dist/integrations/harnesses/codex/agent-builder.js +137 -0
  240. package/dist/integrations/harnesses/codex/index.js +63 -0
  241. package/dist/integrations/harnesses/codex/result-extractor.js +73 -0
  242. package/dist/integrations/harnesses/copilot/agent-builder.js +122 -0
  243. package/dist/integrations/harnesses/copilot/index.js +60 -0
  244. package/dist/integrations/harnesses/copilot/result-extractor.js +151 -0
  245. package/dist/integrations/harnesses/gemini/agent-builder.js +121 -0
  246. package/dist/integrations/harnesses/gemini/index.js +60 -0
  247. package/dist/integrations/harnesses/gemini/result-extractor.js +121 -0
  248. package/dist/integrations/harnesses/index.js +28 -7
  249. package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
  250. package/dist/integrations/harnesses/opencode/index.js +17 -16
  251. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  252. package/dist/integrations/harnesses/opencode-sdk/harness.js +65 -0
  253. package/dist/integrations/harnesses/opencode-sdk/index.js +10 -34
  254. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +642 -71
  255. package/dist/integrations/harnesses/openhands/agent-builder.js +126 -0
  256. package/dist/integrations/harnesses/openhands/index.js +58 -0
  257. package/dist/integrations/harnesses/openhands/result-extractor.js +103 -0
  258. package/dist/integrations/harnesses/pi/agent-builder.js +104 -0
  259. package/dist/integrations/harnesses/pi/index.js +58 -0
  260. package/dist/integrations/harnesses/pi/result-extractor.js +135 -0
  261. package/dist/integrations/harnesses/types.js +8 -0
  262. package/dist/integrations/session-logs/index.js +40 -11
  263. package/dist/llm/call-ai.js +2 -2
  264. package/dist/llm/client.js +34 -11
  265. package/dist/llm/embedder.js +67 -4
  266. package/dist/llm/embedders/cache.js +3 -1
  267. package/dist/llm/embedders/deterministic.js +66 -0
  268. package/dist/llm/embedders/local.js +73 -3
  269. package/dist/llm/feature-gate.js +16 -15
  270. package/dist/llm/graph-extract.js +67 -44
  271. package/dist/llm/memory-infer-impl.js +138 -0
  272. package/dist/llm/memory-infer.js +1 -127
  273. package/dist/llm/metadata-enhance.js +44 -31
  274. package/dist/llm/structured-call.js +49 -0
  275. package/dist/migrate-storage-node.mjs +8 -0
  276. package/dist/output/context.js +5 -5
  277. package/dist/output/renderers.js +87 -15
  278. package/dist/output/shapes/curate.js +14 -2
  279. package/dist/output/shapes/helpers.js +0 -3
  280. package/dist/output/shapes/passthrough.js +6 -1
  281. package/dist/output/text/helpers.js +241 -2
  282. package/dist/output/text/workflow.js +4 -1
  283. package/dist/registry/providers/skills-sh.js +21 -147
  284. package/dist/registry/providers/static-index.js +15 -157
  285. package/dist/registry/resolve.js +27 -9
  286. package/dist/runtime.js +25 -1
  287. package/dist/schemas/akm-config.json +14225 -0
  288. package/dist/schemas/akm-workflow.json +328 -0
  289. package/dist/scripts/migrate-storage.js +2743 -8390
  290. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +1652 -607
  291. package/dist/setup/detect.js +9 -0
  292. package/dist/setup/legacy-config.js +106 -0
  293. package/dist/setup/prompt.js +57 -0
  294. package/dist/setup/providers.js +14 -0
  295. package/dist/setup/registry-stash-loader.js +12 -0
  296. package/dist/setup/semantic-assets.js +124 -0
  297. package/dist/setup/setup.js +52 -1614
  298. package/dist/setup/steps/connection.js +734 -0
  299. package/dist/setup/steps/output.js +31 -0
  300. package/dist/setup/steps/platforms.js +124 -0
  301. package/dist/setup/steps/semantic.js +27 -0
  302. package/dist/setup/steps/sources.js +222 -0
  303. package/dist/setup/steps/stashdir.js +42 -0
  304. package/dist/setup/steps/tasks.js +152 -0
  305. package/dist/sources/include.js +6 -2
  306. package/dist/sources/providers/filesystem.js +0 -1
  307. package/dist/sources/providers/git-install.js +210 -0
  308. package/dist/sources/providers/git-provider.js +234 -0
  309. package/dist/sources/providers/git-stash.js +248 -0
  310. package/dist/sources/providers/git.js +10 -661
  311. package/dist/sources/providers/npm.js +2 -6
  312. package/dist/sources/providers/provider-utils.js +13 -7
  313. package/dist/sources/providers/sync-from-ref.js +9 -1
  314. package/dist/sources/providers/website.js +9 -5
  315. package/dist/sources/website-ingest.js +187 -29
  316. package/dist/sources/wiki-fetchers/registry.js +53 -0
  317. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  318. package/dist/storage/database.js +45 -10
  319. package/dist/storage/managed-db.js +82 -0
  320. package/dist/storage/repositories/canaries-repository.js +107 -0
  321. package/dist/storage/repositories/consolidation-repository.js +38 -0
  322. package/dist/storage/repositories/embeddings-repository.js +72 -0
  323. package/dist/storage/repositories/events-repository.js +187 -0
  324. package/dist/storage/repositories/extract-sessions-repository.js +96 -0
  325. package/dist/storage/repositories/improve-runs-repository.js +146 -0
  326. package/dist/storage/repositories/index-db.js +14 -8
  327. package/dist/storage/repositories/proposals-repository.js +220 -0
  328. package/dist/storage/repositories/recombine-repository.js +213 -0
  329. package/dist/storage/repositories/registry-cache.js +93 -0
  330. package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
  331. package/dist/storage/repositories/task-history-repository.js +93 -0
  332. package/dist/storage/repositories/workflow-runs-repository.js +189 -1
  333. package/dist/storage/sqlite-pragmas.js +146 -0
  334. package/dist/tasks/backends/cron.js +1 -1
  335. package/dist/tasks/backends/index.js +9 -0
  336. package/dist/tasks/backends/launchd.js +1 -1
  337. package/dist/tasks/backends/schtasks.js +1 -1
  338. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  339. package/dist/tasks/runner.js +15 -13
  340. package/dist/text-import-hook.mjs +1 -1
  341. package/dist/wiki/wiki.js +52 -11
  342. package/dist/workflows/authoring/authoring.js +123 -10
  343. package/dist/workflows/authoring/workflow-program-template.yaml +31 -0
  344. package/dist/workflows/cli.js +5 -0
  345. package/dist/workflows/db.js +138 -4
  346. package/dist/workflows/exec/brief.js +484 -0
  347. package/dist/workflows/exec/native-executor.js +975 -0
  348. package/dist/workflows/exec/param-secrets.js +115 -0
  349. package/dist/workflows/exec/report.js +1295 -0
  350. package/dist/workflows/exec/run-workflow.js +596 -0
  351. package/dist/workflows/exec/scheduler.js +100 -0
  352. package/dist/workflows/exec/step-work.js +1156 -0
  353. package/dist/workflows/exec/unit-writer.js +23 -0
  354. package/dist/workflows/exec/watch.js +116 -0
  355. package/dist/workflows/exec/worktree.js +171 -0
  356. package/dist/workflows/ir/compile.js +388 -0
  357. package/dist/workflows/ir/params.js +54 -0
  358. package/dist/workflows/ir/plan-hash.js +33 -0
  359. package/dist/workflows/ir/schema.js +4 -0
  360. package/dist/workflows/parser.js +3 -1
  361. package/dist/workflows/program/expressions.js +369 -0
  362. package/dist/workflows/program/parser.js +760 -0
  363. package/dist/workflows/program/project.js +105 -0
  364. package/dist/workflows/program/schema.js +54 -0
  365. package/dist/workflows/renderer.js +82 -5
  366. package/dist/workflows/runtime/agent-identity.js +59 -14
  367. package/dist/workflows/runtime/runs.js +248 -153
  368. package/dist/workflows/runtime/unit-checkin.js +45 -0
  369. package/dist/workflows/runtime/workflow-asset-loader.js +188 -0
  370. package/dist/workflows/validate-summary.js +26 -10
  371. package/dist/workflows/validator.js +1 -1
  372. package/docs/README.md +69 -18
  373. package/docs/data-and-telemetry.md +7 -5
  374. package/docs/migration/release-notes/0.7.0.md +1 -1
  375. package/docs/migration/release-notes/0.9.0-beta.60.md +19 -0
  376. package/docs/migration/release-notes/0.9.0.md +39 -0
  377. package/package.json +10 -10
  378. package/dist/assets/tasks/core/update-stashes.yml +0 -4
  379. package/dist/commands/db-cli.js +0 -23
  380. package/dist/indexer/db/db-backup.js +0 -376
  381. package/dist/indexer/passes/staleness-detect.js +0 -488
@@ -5,52 +5,54 @@ import { createHash } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
7
  import readline from "node:readline";
8
- import { parse as yamlParse } from "yaml";
8
+ import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
9
9
  import { parseAssetRef } from "../../core/asset/asset-ref.js";
10
10
  import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
11
11
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
12
12
  import { resolveStashDir, timestampForFilename } from "../../core/common.js";
13
- import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
13
+ import { getDefaultLlmConfig, getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
14
14
  import { ConfigError } from "../../core/errors.js";
15
- import { appendEvent } from "../../core/events.js";
15
+ // Note: appendEvent import removed (WS-3a: archive TTL machinery retired)
16
16
  import { parseEmbeddedJsonResponse } from "../../core/parse.js";
17
+ import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
17
18
  import { detectTruncatedDescription } from "../../core/text-truncation.js";
18
- import { hasHotCaptureMode, hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
19
- import { createProposal, isProposalSkipped, listProposals } from "../proposal/validators/proposals.js";
19
+ import { DURATION_UNITS, parseDuration } from "../../core/time.js";
20
+ import { createProposal, isProposalSkipped, listProposals } from "../proposal/repository.js";
21
+ import { hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
22
+ import { checkGenerationGuard, checkLexicalDiversity, checkMergeInformationFloor, computeMergedGeneration, readAssetGeneration, } from "./anti-collapse.js";
23
+ import { cacheHash, runDeterministicDedup, stripFrontmatterBody } from "./dedup.js";
24
+ import { shouldSkipHotProbationInLlm } from "./hot-probation.js";
20
25
  import { writeContradictEdge } from "./memory/memory-belief.js";
21
26
  // Re-export the moved helpers so existing test imports continue to resolve.
22
27
  export { hasSupersededStatus, validateProposalFrontmatter };
28
+ import { openStateDatabase, withStateDb } from "../../core/state-db.js";
23
29
  import { warn } from "../../core/warn.js";
24
30
  import { commitWriteTargetBoundary, deleteAssetFromSource, resolveWriteTarget, writeAssetToSource, } from "../../core/write-source.js";
25
31
  import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../../indexer/db/db.js";
26
32
  import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
27
33
  import { chatCompletion } from "../../llm/client.js";
28
- import { cosineSimilarity, embedBatch } from "../../llm/embedder.js";
34
+ import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
29
35
  import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
36
+ import { getConsolidationJudgedMap, upsertConsolidationJudged, } from "../../storage/repositories/consolidation-repository.js";
37
+ import { getBodyEmbeddings, upsertBodyEmbeddings } from "../../storage/repositories/embeddings-repository.js";
38
+ // Chunk sizing + per-chunk prompt assembly live in ./consolidate/chunking.
39
+ // Imported for internal use by the orchestrator and re-exported for importers.
40
+ import { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
41
+ export { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
42
+ // LLM-output sanitization (pure string/frontmatter transforms) lives in
43
+ // ./consolidate/sanitize. Imported for internal use + re-exported for importers.
44
+ import { normalizeUpdatedField, sanitizeMergedContent } from "./consolidate/sanitize.js";
45
+ export { normalizeUpdatedField, sanitizeMergedContent, stripOuterCodeFence } from "./consolidate/sanitize.js";
46
+ // Eligibility / safety predicates live in ./consolidate/eligibility. Imported
47
+ // for internal guard use; the two public predicates are re-exported.
48
+ import { consolidateGuardStatus, isConsolidationEligibleMemoryName, isHotCapturedMemory, } from "./consolidate/eligibility.js";
49
+ export { isConsolidationEligibleMemoryName, isHotCapturedMemory } from "./consolidate/eligibility.js";
50
+ // Plan parsing / merging (pure op-reconciliation algebra) lives in
51
+ // ./consolidate/merge. Imported for internal use; mergePlans re-exported.
52
+ import { isValidOp, mergePlans } from "./consolidate/merge.js";
53
+ export { mergePlans } from "./consolidate/merge.js";
30
54
  // ── Prompts ─────────────────────────────────────────────────────────────────
31
- const CONSOLIDATE_SYSTEM_PROMPT = `You are the akm consolidate assistant analyzing memory assets.
32
-
33
- Rules:
34
- 1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
35
- 2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
36
- 3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
37
- 4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") → mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
38
- 5. KEEP: Memory is unique and current → omit from output.
39
-
40
- Return ONLY JSON (no prose, no code fences):
41
- {
42
- "operations": [
43
- { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
44
- { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
45
- { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
46
- { "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
47
- ],
48
- "warnings": ["<optional concerns>"]
49
- }
50
-
51
- For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
52
-
53
- When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
55
+ const CONSOLIDATE_SYSTEM_PROMPT = consolidateSystemPrompt;
54
56
  /**
55
57
  * JSON Schema for structured consolidate plans (PR 1 of the asset-writers
56
58
  * decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
@@ -136,140 +138,17 @@ export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
136
138
  },
137
139
  },
138
140
  };
139
- export function isConsolidationEligibleMemoryName(name) {
140
- return !name.endsWith(".derived");
141
- }
142
- /**
143
- * Returns true when the memory file has `captureMode: hot` in its frontmatter.
144
- *
145
- * Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
146
- * The consolidate LLM is forbidden from deleting or auto-merging them — the
147
- * user wrote them on purpose and only the user can decide to retire them.
148
- *
149
- * Reads the file once per check; consolidate runs against ~10 memories per
150
- * chunk so the IO cost is trivial. Returns false on any read/parse error
151
- * (fail-safe: an unparseable file is treated as not-hot, but the broader
152
- * consolidate flow already guards against unparseable memories elsewhere).
153
- *
154
- * Defends against four observed defect classes (see
155
- * `memory:akm-improve-critical-review-2026-05-20`):
156
- * - LLM marks a memory contradicted then deletes (dangling contradictedBy)
157
- * - LLM merges two unrelated memories sharing a topic keyword
158
- * - LLM judges a recent durable design memo as "redundant"
159
- * - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
160
- */
161
- export function isHotCapturedMemory(filePath) {
162
- try {
163
- if (!fs.existsSync(filePath))
164
- return false;
165
- const content = fs.readFileSync(filePath, "utf8");
166
- const parsed = parseFrontmatter(content);
167
- return hasHotCaptureMode(parsed.data);
168
- }
169
- catch {
170
- return false;
171
- }
172
- }
173
- function consolidateGuardStatus(filePath) {
174
- if (!fs.existsSync(filePath))
175
- return "missing";
176
- let content;
177
- try {
178
- content = fs.readFileSync(filePath, "utf8");
179
- }
180
- catch {
181
- return "unparseable";
182
- }
183
- let parsed;
184
- try {
185
- parsed = parseFrontmatter(content);
186
- }
187
- catch {
188
- return "unparseable";
189
- }
190
- const data = parsed.data;
191
- if (!data || Object.keys(data).length === 0)
192
- return "unparseable";
193
- return hasHotCaptureMode(data) ? "hot" : "safe";
194
- }
195
- // ── Chunk sizing ─────────────────────────────────────────────────────────────
196
- /**
197
- * Conservative chars-per-token estimate used when computing prompt budgets.
198
- * English text averages roughly 4 chars/token for most LLM tokenizers. We use
199
- * 3 to stay conservative (shorter tokens = more tokens per char).
200
- */
201
- const CHARS_PER_TOKEN = 3;
202
- /**
203
- * Overhead budget reserved for the system prompt, chunk header lines, and per-
204
- * memory metadata lines (name, description, tags, separator). Measured at
205
- * roughly 600 chars for the system prompt + ~100 chars of header + ~50 chars
206
- * per memory × chunk size. We round up to 2 000 tokens to leave room for the
207
- * model's own output.
208
- */
209
- const PROMPT_OVERHEAD_TOKENS = 2_000;
210
- /**
211
- * Default effective token budget used when the default LLM profile's
212
- * `contextLength` is not set. This is intentionally conservative (4 096)
213
- * rather than being set to the model's actual context window, because:
214
- *
215
- * - When the agent path is used, the agent CLI (e.g. opencode)
216
- * prepends its own large system prompt + conversation history before
217
- * forwarding to the model. That overhead easily consumes 30K+ tokens on
218
- * a model with a 16K context window, leaving very little room for
219
- * chunk content.
220
- * - When the HTTP path is used (an LLM profile is selected), only the akm
221
- * system prompt and user prompt are sent, so the budget can be set to the
222
- * model's actual context length via profiles.llm[defaults.llm].contextLength.
223
- *
224
- * Set profiles.llm[defaults.llm].contextLength in your config file to the
225
- * model's actual context window to allow larger chunks on the HTTP path.
226
- */
227
- export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
228
- /**
229
- * Given the model's context window and the per-memory body truncation limit,
230
- * return the maximum number of memories that can safely fit in one chunk
231
- * without the prompt overflowing the context window.
232
- *
233
- * The formula is:
234
- * usableTokens = contextLength - PROMPT_OVERHEAD_TOKENS
235
- * tokensPerMemory = ceil(bodyTruncation / CHARS_PER_TOKEN)
236
- * chunkSize = floor(usableTokens / tokensPerMemory)
237
- *
238
- * Result is clamped between 1 and 50 to avoid degenerate values.
239
- *
240
- * @param contextLength - Model context window in tokens.
241
- * @param bodyTruncation - Max chars per memory body included in the prompt.
242
- * @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
243
- */
244
- export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
245
- const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
246
- const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
247
- const raw = Math.floor(usableTokens / tokensPerMemory);
248
- return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
249
- }
250
- // ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
251
- /**
252
- * Re-order memories so that similar ones are placed adjacent to each other
253
- * before the memories are sliced into chunks. This ensures high-similarity
254
- * memories land in the same LLM context window, allowing the consolidate
255
- * model to detect and merge duplicates that would otherwise be split across
256
- * chunks and survive indefinitely.
257
- *
258
- * Algorithm: greedy nearest-neighbour chain starting from the first memory.
259
- * Each step selects the unused memory with the highest cosine similarity to
260
- * the last-placed memory. O(n²) — acceptable for the expected N < 200.
261
- *
262
- * mem0 arXiv:2504.19413 — every candidate compared against whole store.
263
- * A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
264
- *
265
- * Returns the original order unchanged when:
266
- * - The embedding config is not present.
267
- * - Embedding requests fail (fail-open).
268
- * - There are fewer than 3 memories (no benefit to reordering).
269
- */
270
- async function clusterMemoriesBySimilarity(memories, config) {
141
+ async function clusterMemoriesBySimilarity(memories, config, stateDb) {
142
+ const noTelemetry = { embedMs: 0, cacheHits: 0, cacheMisses: 0 };
271
143
  if (memories.length < 3 || !config.embedding)
272
- return memories;
144
+ return { ordered: memories, embedTelemetry: noTelemetry };
145
+ // WS-3a: cluster uses description+tags as the embedding input (NOT the raw
146
+ // body) — this is intentionally different from the dedup/body cache because
147
+ // the clustering goal is semantic grouping, not dedup twin detection.
148
+ // The body_embeddings cache is keyed by cacheHash(body); clustering inputs
149
+ // are keyed by cacheHash(description+tags text). Re-use the same table with
150
+ // a distinct hash so the two lookup sets never collide.
151
+ const modelId = resolveEmbeddingModelId(config.embedding);
273
152
  const texts = memories.map((m) => {
274
153
  const parts = [];
275
154
  if (m.description)
@@ -278,16 +157,91 @@ async function clusterMemoriesBySimilarity(memories, config) {
278
157
  parts.push(m.tags.join(" "));
279
158
  return parts.join(". ") || m.name;
280
159
  });
281
- let embeddings = null;
282
- try {
283
- embeddings = await embedBatch(texts, config.embedding);
160
+ // Compute content hashes for the cluster texts (not bodies — different input).
161
+ const contentHashes = texts.map((t) => createHash("sha256").update(t, "utf8").digest("hex"));
162
+ // WS-5: track embed cache hits/misses for perf telemetry.
163
+ let embedMs = 0;
164
+ let cacheHits = 0;
165
+ let cacheMisses = 0;
166
+ let cachedVecs = new Map();
167
+ if (stateDb) {
168
+ try {
169
+ cachedVecs = getBodyEmbeddings(stateDb, contentHashes, modelId);
170
+ }
171
+ catch {
172
+ // Fail open.
173
+ cachedVecs = new Map();
174
+ }
284
175
  }
285
- catch {
286
- // Fail open: embedding failures degrade gracefully to original order.
287
- return memories;
176
+ const missIndices = [];
177
+ const missTexts = [];
178
+ for (let i = 0; i < texts.length; i++) {
179
+ if (!cachedVecs.has(contentHashes[i])) {
180
+ missIndices.push(i);
181
+ missTexts.push(texts[i]);
182
+ cacheMisses++;
183
+ }
184
+ else {
185
+ cacheHits++;
186
+ }
187
+ }
188
+ let missVecs = [];
189
+ if (missTexts.length > 0) {
190
+ const embedStart = Date.now();
191
+ try {
192
+ missVecs = await embedBatch(missTexts, config.embedding);
193
+ }
194
+ catch {
195
+ // Fail open: embedding failures degrade gracefully to original order.
196
+ return { ordered: memories, embedTelemetry: { embedMs, cacheHits, cacheMisses } };
197
+ }
198
+ finally {
199
+ embedMs += Date.now() - embedStart;
200
+ }
201
+ // Upsert newly computed vectors into the cache.
202
+ if (stateDb && missVecs.length === missTexts.length) {
203
+ try {
204
+ const toUpsert = missIndices.map((idx, pos) => ({
205
+ contentHash: contentHashes[idx],
206
+ embedding: missVecs[pos],
207
+ modelId,
208
+ }));
209
+ upsertBodyEmbeddings(stateDb, toUpsert);
210
+ }
211
+ catch {
212
+ // Fail open: cache write errors are non-fatal.
213
+ }
214
+ }
215
+ }
216
+ // Assemble the full embedding array in memories order.
217
+ let embeddings = null;
218
+ {
219
+ const assembled = [];
220
+ let ok = true;
221
+ for (let i = 0; i < memories.length; i++) {
222
+ const hash = contentHashes[i];
223
+ const cached = cachedVecs.get(hash);
224
+ if (cached) {
225
+ assembled.push(cached);
226
+ continue;
227
+ }
228
+ const missPos = missIndices.indexOf(i);
229
+ const vec = missPos >= 0 ? missVecs[missPos] : undefined;
230
+ if (vec) {
231
+ assembled.push(vec);
232
+ }
233
+ else {
234
+ ok = false;
235
+ break;
236
+ }
237
+ }
238
+ if (ok && assembled.length === memories.length) {
239
+ embeddings = assembled;
240
+ }
288
241
  }
242
+ const embedTelemetry = { embedMs, cacheHits, cacheMisses };
289
243
  if (!embeddings || embeddings.length !== memories.length)
290
- return memories;
244
+ return { ordered: memories, embedTelemetry };
291
245
  // Greedy nearest-neighbour chain.
292
246
  const used = new Array(memories.length).fill(false);
293
247
  const ordered = [];
@@ -313,87 +267,16 @@ async function clusterMemoriesBySimilarity(memories, config) {
313
267
  used[bestIdx] = true;
314
268
  current = bestIdx;
315
269
  }
316
- return ordered;
270
+ return { ordered, embedTelemetry };
317
271
  }
318
272
  // ── Chunk helpers ────────────────────────────────────────────────────────────
319
- /**
320
- * Build the per-chunk user prompt fed to the consolidate LLM.
321
- *
322
- * Each memory is annotated with two flags that drive the system-prompt
323
- * rules at lines 181-186:
324
- * - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
325
- * forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
326
- * stack before this annotation.
327
- * - `(already queued)` — the memory's body hash matches a pending
328
- * consolidate proposal; system prompt rule 3 forbids proposing
329
- * promote/merge/contradict. ~107/4h before this annotation.
330
- *
331
- * Both annotations are visible to the LLM. `pendingProposalBodyHashes`
332
- * is precomputed once per run by `loadPendingConsolidateProposalHashes`
333
- * so the cost stays O(memories) inside the chunk loop.
334
- */
335
- export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
336
- const start = memories[0] ? `memory:${memories[0].name}` : "";
337
- const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
338
- const annotationsByIndex = [];
339
- const hotRefs = [];
340
- for (const m of memories) {
341
- let body = "";
342
- try {
343
- body = fs.readFileSync(m.filePath, "utf8");
344
- }
345
- catch {
346
- body = "(unreadable)";
347
- }
348
- const parsed = parseFrontmatter(body);
349
- const isHot = parsed.data.captureMode === "hot";
350
- const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
351
- const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
352
- annotationsByIndex.push({ isHot, isAlreadyQueued, body });
353
- if (isHot)
354
- hotRefs.push(`memory:${m.name}`);
355
- }
356
- const lines = [
357
- `Source: ${sourceName}`,
358
- `Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
359
- "",
360
- ];
361
- // Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
362
- // op-words like "promote", "merge", "contradict" so the model doesn't
363
- // accidentally treat the warning as a hint to use that op elsewhere
364
- // (variant B leaked the word "contradict" into the control sample
365
- // during the diagnostic).
366
- if (hotRefs.length > 0) {
367
- lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
368
- for (const ref of hotRefs)
369
- lines.push(` - ${ref}`);
370
- lines.push("");
371
- }
372
- for (let i = 0; i < memories.length; i++) {
373
- const m = memories[i];
374
- const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
375
- const annotations = [];
376
- if (isHot)
377
- annotations.push("captureMode: hot");
378
- if (isAlreadyQueued)
379
- annotations.push("already queued");
380
- const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
381
- lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
382
- lines.push(`Description: ${m.description || "(none)"}`);
383
- lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
384
- lines.push("---");
385
- lines.push(body.slice(0, bodyTruncation));
386
- lines.push("");
387
- }
388
- return lines.join("\n");
389
- }
390
273
  /**
391
274
  * Precompute body-hashes of all currently-pending consolidate proposals so
392
275
  * the per-chunk prompt can annotate memories whose body would just produce
393
- * a deterministic `dedup_pending_proposal` skip. Hash domain matches the
394
- * dedup site at ~line 1510 (sha256 over the post-frontmatter content,
395
- * trimmed). Empty set on any read/parse error — fail-safe to "annotate
396
- * nothing" so the LLM still proposes, just slightly more wastefully.
276
+ * a deterministic `dedup_pending_proposal` skip. Uses `cacheHash` (case-
277
+ * preserving stripped body) the same domain used by the body-embedding
278
+ * cache and `computeMemoryContentHash`. Empty set on any read/parse error
279
+ * — fail-safe to "annotate nothing" so the LLM still proposes.
397
280
  */
398
281
  function loadPendingConsolidateProposalHashes(stashDir) {
399
282
  const hashes = new Set();
@@ -401,8 +284,7 @@ function loadPendingConsolidateProposalHashes(stashDir) {
401
284
  const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
402
285
  for (const p of pending) {
403
286
  try {
404
- const body = parseFrontmatter(p.payload.content).content.trim();
405
- hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
287
+ hashes.add(cacheHash(p.payload.content));
406
288
  }
407
289
  catch {
408
290
  // skip malformed payloads — they can't dedup anyway
@@ -414,148 +296,6 @@ function loadPendingConsolidateProposalHashes(stashDir) {
414
296
  }
415
297
  return hashes;
416
298
  }
417
- function isValidOp(op) {
418
- if (typeof op !== "object" || op === null)
419
- return false;
420
- const o = op;
421
- if (o.op === "merge") {
422
- return typeof o.primary === "string" && Array.isArray(o.secondaries);
423
- }
424
- if (o.op === "delete") {
425
- return typeof o.ref === "string";
426
- }
427
- if (o.op === "promote") {
428
- return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
429
- }
430
- if (o.op === "contradict") {
431
- return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
432
- }
433
- return false;
434
- }
435
- export function mergePlans(chunks, knownRefs) {
436
- const mergeOps = new Map();
437
- const deleteOps = new Map();
438
- const promoteOps = new Map();
439
- // C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
440
- const contradictOps = new Map();
441
- const warnings = [];
442
- for (const chunk of chunks) {
443
- for (const op of chunk) {
444
- if (op.op === "merge") {
445
- // Drop ops whose primary the LLM hallucinated (not in the loaded memory
446
- // pool). Without this guard, a hallucinated primary flows all the way to
447
- // Phase B where !memoryByRef.has(primary) fires and charges every real
448
- // secondary with merge_primary_missing — masking LLM hallucinations as
449
- // filter regressions in health metrics.
450
- if (knownRefs && !knownRefs.has(op.primary)) {
451
- warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
452
- // Use a dedicated skip reason so dashboards can distinguish
453
- // hallucinated primaries from stale-DB regressions.
454
- // Secondaries are real refs; they are NOT charged here — they remain
455
- // available for other ops to claim.
456
- continue;
457
- }
458
- // Filter hallucinated secondaries while preserving real ones.
459
- let mergeOp = op;
460
- if (knownRefs) {
461
- const filteredSecondaries = op.secondaries.filter((sec) => {
462
- if (!knownRefs.has(sec)) {
463
- warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
464
- return false;
465
- }
466
- return true;
467
- });
468
- if (filteredSecondaries.length !== op.secondaries.length) {
469
- mergeOp = { ...op, secondaries: filteredSecondaries };
470
- }
471
- }
472
- // merge wins over delete
473
- if (deleteOps.has(mergeOp.primary)) {
474
- deleteOps.delete(mergeOp.primary);
475
- }
476
- for (const sec of mergeOp.secondaries) {
477
- if (deleteOps.has(sec))
478
- deleteOps.delete(sec);
479
- }
480
- mergeOps.set(mergeOp.primary, mergeOp);
481
- }
482
- else if (op.op === "delete") {
483
- // merge and promote both win over delete. A promote is non-destructive
484
- // (creates a proposal) but the source memory is counted in `promoted`;
485
- // if a delete also fires, the ref lands in both `promoted` and
486
- // `skipReasons`, breaking the invariant by +1.
487
- if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
488
- deleteOps.set(op.ref, op);
489
- }
490
- }
491
- else if (op.op === "promote") {
492
- // C-2 / #381: when both a promote and a merge target the same ref,
493
- // queue the promote FIRST rather than discarding it. The promote op
494
- // routes through createProposal (the human-gated proposal queue), so
495
- // it is non-destructive. The merge follows after the proposal is
496
- // created. This preserves the human reviewer's ability to inspect the
497
- // promotion before the source memory is merged/deleted.
498
- // AGM K*8 — retain the maximally informative consistent subset.
499
- promoteOps.set(op.ref, op);
500
- }
501
- else if (op.op === "contradict") {
502
- // Deduplicate by ref+contradictedByRef pair.
503
- const key = `${op.ref}|${op.contradictedByRef}`;
504
- if (!contradictOps.has(key)) {
505
- contradictOps.set(key, op);
506
- }
507
- }
508
- }
509
- }
510
- // Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
511
- //
512
- // 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
513
- // for secondaries that were already in deleteOps, but misses the case
514
- // where the delete chunk came first. A full sweep here fixes both orders.
515
- //
516
- // 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
517
- // only the first (insertion-order) retains it. Without this, a successful
518
- // merge credits A to mergedSecondaries and a later merge's emitMerge-
519
- // FailureSkips also charges A to skipReasons — double-counting A while
520
- // processed has it only once.
521
- //
522
- // 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
523
- // a secondary in another, remove A from the secondary list. Both merges
524
- // would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
525
- // breaking the invariant the same way.
526
- // Also remove delete ops for any ref claimed by a promote op (handles the
527
- // case where the delete chunk appeared before the promote chunk).
528
- for (const ref of promoteOps.keys()) {
529
- deleteOps.delete(ref);
530
- }
531
- const claimedSecondaries = new Set();
532
- for (const mergeOp of mergeOps.values()) {
533
- deleteOps.delete(mergeOp.primary);
534
- mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
535
- if (mergeOps.has(sec)) {
536
- warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
537
- return false;
538
- }
539
- if (claimedSecondaries.has(sec)) {
540
- warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
541
- return false;
542
- }
543
- claimedSecondaries.add(sec);
544
- deleteOps.delete(sec);
545
- return true;
546
- });
547
- }
548
- // C-2 / #381: promote ops are ordered BEFORE merge ops so that the
549
- // human-gated proposal queue entry is created before any destructive merge.
550
- // Phase B processes ops in array order, so promote executes first.
551
- const ops = [
552
- ...promoteOps.values(),
553
- ...mergeOps.values(),
554
- ...deleteOps.values(),
555
- ...contradictOps.values(),
556
- ];
557
- return { ops, warnings };
558
- }
559
299
  function getJournalPath(stashDir) {
560
300
  return path.join(stashDir, ".akm", "consolidate-journal.json");
561
301
  }
@@ -672,6 +412,32 @@ function backupFile(filePath, backupDir, name) {
672
412
  // best-effort
673
413
  }
674
414
  }
415
+ // ── WS-3b: Generation frontmatter injection ───────────────────────────────────
416
+ /**
417
+ * Inject `generation` and `source_refs` into merged content.
418
+ * generation = max(sourceGenerations) + 1.
419
+ * source_refs = UNION of the provided provenance refs (participants + their
420
+ * cited sources) with anything already present in the merged frontmatter —
421
+ * R5 §4.2: the old set-if-absent behavior dropped second-generation
422
+ * provenance whenever the LLM emitted its own (partial) source_refs.
423
+ * Fails open — returns original content if frontmatter can't be parsed.
424
+ */
425
+ function injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceRefs) {
426
+ try {
427
+ const parsed = parseFrontmatter(mergedContent);
428
+ const existingFm = parsed.data;
429
+ const existingRefs = Array.isArray(existingFm.source_refs) ? existingFm.source_refs.map(String) : [];
430
+ const updatedFm = {
431
+ ...existingFm,
432
+ generation: computeMergedGeneration(sourceGenerations),
433
+ source_refs: [...new Set([...existingRefs, ...provenanceRefs])],
434
+ };
435
+ return assembleAssetFromString(serializeFrontmatter(updatedFm), parsed.content);
436
+ }
437
+ catch {
438
+ return mergedContent; // fail open
439
+ }
440
+ }
675
441
  // ── Archive helper (P1-B: soft-invalidation) ─────────────────────────────────
676
442
  /**
677
443
  * Move a memory asset to `.akm/archive/` with `status: superseded` frontmatter
@@ -742,8 +508,8 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
742
508
  * silent 400s from LM Studio). The investigation lives at
743
509
  * `/tmp/akm-health-investigations/consolidation-no-op.md`.
744
510
  */
745
- function resolveConsolidateLlmConfig(config) {
746
- const consolidateProcess = config.profiles?.improve?.default?.processes?.consolidate;
511
+ function resolveConsolidateLlmConfig(config, activeProfile) {
512
+ const consolidateProcess = getImproveProcessConfig(config, "consolidate", activeProfile);
747
513
  const runnerSpec = resolveImproveProcessRunnerFromProfile(consolidateProcess, config);
748
514
  if (runnerSpec && runnerIsLlm(runnerSpec)) {
749
515
  return runnerSpec.connection;
@@ -752,6 +518,58 @@ function resolveConsolidateLlmConfig(config) {
752
518
  // fall back to the default LLM profile rather than disabling the pass.
753
519
  return getDefaultLlmConfig(config);
754
520
  }
521
+ // ── Judged-state cache (#581) ────────────────────────────────────────────────
522
+ /**
523
+ * Stable content hash for a memory file used by the judged-state cache (#581)
524
+ * and the body-embedding cache (WS-3a). Uses `cacheHash` from dedup.ts
525
+ * (sha256 of the case-preserving stripped body) plus the sorted `tags` list,
526
+ * so semantic-metadata drift re-enters the judge while cosmetic frontmatter
527
+ * touches (`updated:`, `inferenceProcessed:`) still hash identically and never
528
+ * force a needless re-judge. Returns `undefined` on any read/parse error so
529
+ * callers fail open (treat the memory as un-cached → it stays in the LLM pool).
530
+ */
531
+ function computeMemoryContentHash(filePath) {
532
+ try {
533
+ const raw = fs.readFileSync(filePath, "utf8");
534
+ let tagSuffix = "";
535
+ try {
536
+ const { data } = parseFrontmatter(raw);
537
+ const tags = Array.isArray(data?.tags) ? data.tags.map(String).sort() : [];
538
+ if (tags.length > 0)
539
+ tagSuffix = `\n\u0000tags:${tags.join(",")}`;
540
+ }
541
+ catch {
542
+ // Unparseable frontmatter → body-only hash (prior behaviour).
543
+ }
544
+ return cacheHash(raw + tagSuffix);
545
+ }
546
+ catch {
547
+ return undefined;
548
+ }
549
+ }
550
+ /**
551
+ * Build a {@link ConsolidateResult} from partial overrides, filling the envelope
552
+ * defaults (schemaVersion / ok / shape + the zeroed counters). Collapses the
553
+ * ~7 near-identical result literals that previously appeared verbatim at every
554
+ * early-return site and the final return of `akmConsolidateInner`. Callers pass
555
+ * only the fields that differ from the all-zero, ok, non-preview baseline.
556
+ */
557
+ export function makeConsolidateResult(overrides) {
558
+ return {
559
+ schemaVersion: 1,
560
+ ok: true,
561
+ shape: "consolidate-result",
562
+ dryRun: false,
563
+ previewOnly: false,
564
+ processed: 0,
565
+ merged: 0,
566
+ deleted: 0,
567
+ promoted: [],
568
+ contradicted: 0,
569
+ warnings: [],
570
+ ...overrides,
571
+ };
572
+ }
755
573
  // ── Main entry point ─────────────────────────────────────────────────────────
756
574
  export async function akmConsolidate(opts = {}) {
757
575
  const startMs = Date.now();
@@ -762,24 +580,75 @@ export async function akmConsolidate(opts = {}) {
762
580
  const config = opts.config ?? loadConfig();
763
581
  const stashDir = opts.stashDir ?? resolveStashDir();
764
582
  if (!isLlmFeatureEnabled(config, "memory_consolidation")) {
765
- return {
766
- schemaVersion: 1,
767
- ok: true,
768
- shape: "consolidate-result",
583
+ return makeConsolidateResult({
769
584
  dryRun: opts.dryRun ?? false,
770
- previewOnly: false,
771
585
  target: opts.target ?? stashDir,
772
- processed: 0,
773
- merged: 0,
774
- deleted: 0,
775
- promoted: [],
776
- contradicted: 0,
777
- warnings: [],
778
586
  durationMs: Date.now() - startMs,
779
- };
587
+ });
780
588
  }
781
589
  const warnings = [];
782
590
  checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
591
+ // WS-3a: open one state.db handle shared by the body-embedding cache (dedup
592
+ // + cluster) and the judged-state cache. All callers in the function body
593
+ // receive this handle; it is closed in the `finally` block below.
594
+ // Fail-open: any open error leaves it `undefined` and all cache paths skip.
595
+ let sharedStateDb;
596
+ try {
597
+ sharedStateDb = openStateDatabase();
598
+ }
599
+ catch {
600
+ // State DB unavailable → skip the embedding cache for this run.
601
+ }
602
+ try {
603
+ return await akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb);
604
+ }
605
+ finally {
606
+ sharedStateDb?.close();
607
+ }
608
+ }
609
+ /** Fresh, zeroed accounting accumulators for one consolidate run. */
610
+ function createConsolidateAccounting() {
611
+ const acc = {
612
+ judgedNoAction: 0,
613
+ failedChunkMemories: 0,
614
+ totalChunksFailed: 0,
615
+ skipReasons: [],
616
+ skipReasonByRef: new Map(),
617
+ judgedNoActionRefs: new Set(),
618
+ pushSkipReason: () => { },
619
+ };
620
+ acc.pushSkipReason = (op, ref, reason) => {
621
+ // 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
622
+ // to judgedNoAction in its own chunk (a different chunk proposed an op
623
+ // for it that is now being rejected here), promote it from the
624
+ // judgedNoAction bucket into the more specific skipReason bucket.
625
+ // Preserves the invariant: processed == actioned + judgedNoAction +
626
+ // Σ(skipReasons) + failedChunkMemories.
627
+ if (acc.judgedNoActionRefs.delete(ref))
628
+ acc.judgedNoAction--;
629
+ const existing = acc.skipReasonByRef.get(ref);
630
+ if (existing) {
631
+ // Already counted once for accounting. Append the extra skip to the
632
+ // ref's grouped entry for observability without adding a new array
633
+ // entry (which would break the accounting invariant).
634
+ existing.skips.push({ op, reason });
635
+ return;
636
+ }
637
+ const entry = { ref, skips: [{ op, reason }] };
638
+ acc.skipReasonByRef.set(ref, entry);
639
+ acc.skipReasons.push(entry);
640
+ };
641
+ return acc;
642
+ }
643
+ /**
644
+ * Pass 1 — narrow the memory pool before any LLM work: drop stale DB entries,
645
+ * partition hot-probation assets, run the deterministic dedup pre-pass, apply
646
+ * incremental-since and judged-state-cache narrowing, and cap to `opts.limit`
647
+ * (oldest-modified first). Returns an early envelope when the pool empties at
648
+ * any stage; otherwise returns the narrowed pool and the state the plan/apply
649
+ * passes consume. Behavior-identical to the former inlined narrowing block.
650
+ */
651
+ async function narrowConsolidationPool(opts, config, stashDir, startMs, warnings, sharedStateDb) {
783
652
  let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
784
653
  // Pre-flight: filter out stale DB entries whose files no longer exist on
785
654
  // disk. Without this, memories deleted by a prior run (but not yet
@@ -791,43 +660,177 @@ export async function akmConsolidate(opts = {}) {
791
660
  warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
792
661
  }
793
662
  memories = memories.filter((m) => fs.existsSync(m.filePath));
663
+ // (The former WS-3b Step 0a homeostatic demotion pass was removed — R4:
664
+ // it was default-off and self-undoing (the next salience recompute
665
+ // unconditionally overwrote the demoted values). Continuous decay now lives
666
+ // in computeSalience's recency term, whose floor decays on a long half-life.)
667
+ // ── WS-3b Step 0c: Filter hot-probation assets from LLM merge pool ─────────
668
+ // Hot-probation assets (system-generated, not yet graduated from intake pass)
669
+ // are processed by the dedup pre-pass but excluded from the LLM clustering.
670
+ // This prevents noisy extractions from polluting LLM context. The dedup pass
671
+ // below still runs against them so they're cleaned up deterministically.
672
+ // DEFAULT OFF — only active when `processes.extract.hotProbation.enabled === true`
673
+ // (the flag that causes extract to tag new extractions as hot-probation).
674
+ // Without that flag no assets will ever carry the hot-probation marker, so
675
+ // running the filter loop would be pure unnecessary I/O over the full corpus.
676
+ const hotProbationEnabled = getImproveProcessConfig(config, "extract", opts.improveProfile)?.hotProbation
677
+ ?.enabled === true;
678
+ let hotProbationCount = 0;
679
+ if (hotProbationEnabled) {
680
+ const hotProbationMemories = [];
681
+ const nonProbationMemories = [];
682
+ for (const m of memories) {
683
+ try {
684
+ const raw = fs.readFileSync(m.filePath, "utf8");
685
+ const parsed = parseFrontmatter(raw);
686
+ if (shouldSkipHotProbationInLlm(parsed.data)) {
687
+ hotProbationMemories.push(m);
688
+ hotProbationCount++;
689
+ }
690
+ else {
691
+ nonProbationMemories.push(m);
692
+ }
693
+ }
694
+ catch {
695
+ nonProbationMemories.push(m); // fail open
696
+ }
697
+ }
698
+ if (hotProbationCount > 0) {
699
+ warnings.push(`Hot-probation: ${hotProbationCount} hot-probation asset(s) routed to dedup-only pass (excluded from LLM merge pool).`);
700
+ memories = nonProbationMemories;
701
+ }
702
+ }
703
+ // ── Deterministic dedup pre-pass (#617) ─────────────────────────────────────
704
+ // Cheap, no-LLM fast path that collapses the obvious near-duplicates
705
+ // (`.derived` ↔ origin pairs + content twins) BEFORE the embedding-clustered
706
+ // LLM consolidation. DEFAULT OFF — when `dedup.enabled !== true` this is a
707
+ // no-op and the pass behaves byte-identically to today. Collapsed variants
708
+ // are pruned from the LLM pool so the model only ever sees genuinely
709
+ // distinct-but-related memories. Each dropped variant is archived (soft
710
+ // invalidation) before deletion, matching the LLM merge path.
711
+ // Dry-run never mutates the filesystem, so the dedup pre-pass is skipped
712
+ // entirely under `--dry-run` (the LLM plan preview below is unaffected).
713
+ let dedupCollapsed = 0;
714
+ if (opts.dedup?.enabled && !opts.dryRun) {
715
+ const dedupTimestamp = timestampForFilename();
716
+ const dedupResult = await runDeterministicDedup(stashDir, opts.dedup, config, (variantFilePath, variantName) => {
717
+ archiveMemory(variantFilePath, stashDir, `memory:${variantName}`, "collapsed by deterministic dedup pre-pass", -1, undefined, warnings);
718
+ backupFile(variantFilePath, getBackupDir(stashDir, dedupTimestamp), variantName);
719
+ }, opts.signal, sharedStateDb);
720
+ dedupCollapsed = dedupResult.collapsed;
721
+ warnings.push(...dedupResult.warnings);
722
+ if (dedupResult.consumedRefs.length > 0) {
723
+ const consumed = new Set(dedupResult.consumedRefs);
724
+ memories = memories.filter((m) => !consumed.has(`memory:${m.name}`));
725
+ warnings.push(`Deterministic dedup: collapsed ${dedupResult.collapsed} near-duplicate memor${dedupResult.collapsed === 1 ? "y" : "ies"} (no LLM) before chunking.`);
726
+ }
727
+ }
794
728
  if (memories.length === 0) {
795
729
  return {
796
- schemaVersion: 1,
797
- ok: true,
798
- shape: "consolidate-result",
799
- dryRun: opts.dryRun ?? false,
800
- previewOnly: false,
801
- target: opts.target ?? stashDir,
802
- processed: 0,
803
- merged: 0,
804
- deleted: 0,
805
- promoted: [],
806
- contradicted: 0,
807
- warnings,
808
- durationMs: Date.now() - startMs,
730
+ done: true,
731
+ result: makeConsolidateResult({
732
+ dryRun: opts.dryRun ?? false,
733
+ target: opts.target ?? stashDir,
734
+ // #617: the deterministic dedup pre-pass may have emptied the pool by
735
+ // collapsing every remaining memory into a canonical. Surface those
736
+ // collapses in `deleted` so the run reports the work it actually did.
737
+ deleted: dedupCollapsed,
738
+ warnings,
739
+ durationMs: Date.now() - startMs,
740
+ }),
809
741
  };
810
742
  }
811
743
  if (opts.incrementalSince) {
812
744
  memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings, opts.neighborsPerChanged);
813
745
  if (memories.length === 0) {
814
746
  return {
815
- schemaVersion: 1,
816
- ok: true,
817
- shape: "consolidate-result",
818
- dryRun: opts.dryRun ?? false,
819
- previewOnly: false,
820
- target: opts.target ?? stashDir,
821
- processed: 0,
822
- merged: 0,
823
- deleted: 0,
824
- promoted: [],
825
- contradicted: 0,
826
- warnings,
827
- durationMs: Date.now() - startMs,
747
+ done: true,
748
+ result: makeConsolidateResult({
749
+ dryRun: opts.dryRun ?? false,
750
+ target: opts.target ?? stashDir,
751
+ warnings,
752
+ durationMs: Date.now() - startMs,
753
+ }),
754
+ };
755
+ }
756
+ }
757
+ // WS-5 perf telemetry accumulators. These are collected throughout the run and
758
+ // merged into `perfTelemetry` on the final ConsolidateResult.
759
+ // `dedupPoolSize` = memories entering judgedCache narrowing (after dedup+incremental+limit).
760
+ // `judgedCacheSkipped` = memories skipped by the cache.
761
+ // `llmPoolSize` = memories actually sent to the LLM.
762
+ // `embedMs/cacheHits/cacheMisses` = accumulated from clusterMemoriesBySimilarity.
763
+ const perfMs = { dedupPoolSize: memories.length, judgedCacheSkipped: 0 };
764
+ // ── Judged-state cache narrowing (#581) ─────────────────────────────────────
765
+ // DEFAULT OFF. When enabled, skip every memory whose current content hash
766
+ // equals the hash recorded the last time the consolidate LLM judged it
767
+ // (judged-unchanged → no re-judge). This converts coverage from O(window) to
768
+ // O(changed/new) so one run can sweep the whole corpus while the LLM only
769
+ // sees genuinely new/changed memories. `currentHashByName` is populated for
770
+ // EVERY surviving memory (whether or not the cache is on) so the post-LLM
771
+ // recording step can upsert judged state without re-reading the files; when
772
+ // the cache is off it stays empty and the recording step is a no-op.
773
+ const judgedCacheEnabled = opts.judgedCache?.enabled !== false;
774
+ const currentHashByName = new Map();
775
+ if (judgedCacheEnabled) {
776
+ for (const m of memories) {
777
+ const h = computeMemoryContentHash(m.filePath);
778
+ if (h !== undefined)
779
+ currentHashByName.set(m.name, h);
780
+ }
781
+ let cachedMap = new Map();
782
+ {
783
+ // Use the shared state.db handle if available; open a local one otherwise.
784
+ const dbForJudged = sharedStateDb;
785
+ if (dbForJudged) {
786
+ try {
787
+ cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => `memory:${m.name}`));
788
+ }
789
+ catch {
790
+ cachedMap = new Map();
791
+ }
792
+ }
793
+ else {
794
+ try {
795
+ cachedMap = withStateDb((localDb) => getConsolidationJudgedMap(localDb, memories.map((m) => `memory:${m.name}`)));
796
+ }
797
+ catch {
798
+ // State DB unavailable → fail open: judge the full pool this run.
799
+ cachedMap = new Map();
800
+ }
801
+ }
802
+ }
803
+ const beforeCount = memories.length;
804
+ memories = memories.filter((m) => {
805
+ const cur = currentHashByName.get(m.name);
806
+ // No readable hash → keep (fail open; let the LLM judge it).
807
+ if (cur === undefined)
808
+ return true;
809
+ const cached = cachedMap.get(`memory:${m.name}`);
810
+ // Skip only when previously judged AND content is byte-identical since.
811
+ return !(cached !== undefined && cached.content_hash === cur);
812
+ });
813
+ const skipped = beforeCount - memories.length;
814
+ perfMs.judgedCacheSkipped = skipped; // WS-5 perf telemetry
815
+ if (skipped > 0) {
816
+ warnings.push(`Judged-state cache: skipped ${skipped} memor${skipped === 1 ? "y" : "ies"} judged-unchanged (no LLM); ${memories.length} remain for judging.`);
817
+ }
818
+ if (memories.length === 0) {
819
+ return {
820
+ done: true,
821
+ result: makeConsolidateResult({
822
+ dryRun: opts.dryRun ?? false,
823
+ target: opts.target ?? stashDir,
824
+ deleted: dedupCollapsed,
825
+ warnings,
826
+ durationMs: Date.now() - startMs,
827
+ }),
828
828
  };
829
829
  }
830
830
  }
831
+ if (opts.limit === undefined && memories.length > 150) {
832
+ warnings.push(`Consolidation: pool has ${memories.length} memories and no limit is set. Consider adding a limit to your consolidate config to prevent timeouts on slow LLM endpoints.`);
833
+ }
831
834
  if (opts.limit !== undefined && memories.length > opts.limit) {
832
835
  // Order oldest-modified-first before capping so the limit selects the
833
836
  // stalest memories rather than a fixed head of the (rowid-ordered) DB
@@ -849,13 +852,25 @@ export async function akmConsolidate(opts = {}) {
849
852
  warnings.push(`Consolidation: pool capped at ${opts.limit} of ${memories.length} memories (limit option, oldest-modified first).`);
850
853
  memories = memories.slice(0, opts.limit);
851
854
  }
855
+ return { done: false, memories, dedupCollapsed, perfMs, judgedCacheEnabled, currentHashByName };
856
+ }
857
+ /**
858
+ * Pass 2 — turn the narrowed pool into an executable plan. Sizes chunks to the
859
+ * model context window, clusters by embedding similarity, injects the
860
+ * anti-collapse random fraction, applies the cold-start budget cap, runs the
861
+ * per-chunk LLM calls (with retry + failure-rate abort), records judged-state
862
+ * cache outcomes, and reconciles the per-chunk op arrays via {@link mergePlans}.
863
+ * Populates `accounting` in place. Behavior-identical to the former inlined
864
+ * plan-generation block.
865
+ */
866
+ async function planConsolidation(opts, config, stashDir, startMs, memories, warnings, sharedStateDb, judgedCacheEnabled, currentHashByName, accounting) {
852
867
  // Consolidation always uses the HTTP LLM client directly — never the agent
853
868
  // CLI. The agent CLI is for interactive agent sessions (reflect, propose);
854
869
  // structured JSON generation works better and faster via HTTP.
855
870
  //
856
871
  // Honor `profiles.improve.default.processes.consolidate.profile` first; fall
857
872
  // back to the default LLM. See {@link resolveConsolidateLlmConfig}.
858
- const llmConfig = resolveConsolidateLlmConfig(config);
873
+ const llmConfig = resolveConsolidateLlmConfig(config, opts.improveProfile);
859
874
  const isHttpPath = !!llmConfig;
860
875
  // Chunk sizing: derive a safe chunk size from the configured model context
861
876
  // window so that the full prompt (system prompt + chunk user prompt) never
@@ -872,16 +887,64 @@ export async function akmConsolidate(opts = {}) {
872
887
  const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
873
888
  // -- Phase A: plan generation -----------------------------------------------
874
889
  const sourceName = opts.target ?? stashDir;
890
+ // WS-5: capture llmPoolSize = memories entering the LLM (after all filtering).
891
+ const llmPoolSize = memories.length;
875
892
  // C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
876
893
  // This ensures that semantically similar memories land in the same LLM
877
894
  // context window, allowing the model to detect and merge duplicates that
878
895
  // would otherwise be split across chunks and survive indefinitely.
879
896
  // mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
880
897
  // Fails open: if embeddings are unavailable or fail, original order is used.
881
- const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
898
+ const { ordered: clusteredMemories, embedTelemetry } = await clusterMemoriesBySimilarity(memories, config, sharedStateDb);
899
+ // WS-3b Anti-collapse step 8c: inject random (non-similar) clusters.
900
+ // A small fraction (default 5%) of the pool is shuffled into random positions
901
+ // so the pipeline isn't PURELY similarity-driven. This prevents rich-get-richer
902
+ // entrenchment where only the most-retrieved assets ever get consolidated.
903
+ // DEFAULT ON since R5 — opt out via antiCollapse.enabled: false.
904
+ let finalClusteredMemories = clusteredMemories;
905
+ {
906
+ const antiCollapseForCluster = getImproveProcessConfig(config, "consolidate", opts.improveProfile)?.antiCollapse ?? {};
907
+ if (antiCollapseForCluster.enabled !== false && clusteredMemories.length > 2) {
908
+ const fraction = antiCollapseForCluster.randomClusterFraction ?? 0.05;
909
+ const randomCount = Math.max(1, Math.floor(clusteredMemories.length * fraction));
910
+ // Pick `randomCount` positions to inject random (un-clustered) members.
911
+ // Use a seeded-ish shuffle: sort by hash of the name so it's deterministic
912
+ // per run but not strictly similarity-driven.
913
+ const shuffled = [...clusteredMemories].sort((a, b) => {
914
+ // Deterministic shuffle: compare sha256-ish (use name hash as proxy).
915
+ const ha = a.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
916
+ const hb = b.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
917
+ return ha - hb;
918
+ });
919
+ const randomSlice = shuffled.slice(0, randomCount);
920
+ const randomSet = new Set(randomSlice.map((m) => m.name));
921
+ // Insert random members at intervals through the clustered sequence.
922
+ const withRandom = [];
923
+ const interval = Math.max(2, Math.floor(clusteredMemories.length / randomCount));
924
+ let randomIdx = 0;
925
+ for (let i = 0; i < clusteredMemories.length; i++) {
926
+ const m = clusteredMemories[i];
927
+ if (m && !randomSet.has(m.name))
928
+ withRandom.push(m);
929
+ if (i > 0 && i % interval === 0 && randomIdx < randomSlice.length) {
930
+ const r = randomSlice[randomIdx++];
931
+ if (r)
932
+ withRandom.push(r);
933
+ }
934
+ }
935
+ // Append any remaining random members not yet inserted.
936
+ while (randomIdx < randomSlice.length) {
937
+ const r = randomSlice[randomIdx++];
938
+ if (r)
939
+ withRandom.push(r);
940
+ }
941
+ finalClusteredMemories = withRandom;
942
+ warnings.push(`Anti-collapse: injected ${randomCount} random (non-similarity-driven) cluster member(s) into consolidation pool (fraction=${fraction}).`);
943
+ }
944
+ }
882
945
  const chunks = [];
883
- for (let i = 0; i < clusteredMemories.length; i += chunkSize) {
884
- chunks.push(clusteredMemories.slice(i, i + chunkSize));
946
+ for (let i = 0; i < finalClusteredMemories.length; i += chunkSize) {
947
+ chunks.push(finalClusteredMemories.slice(i, i + chunkSize));
885
948
  }
886
949
  // 2026-05-27 prompt-context fix: precompute body-hashes of pending
887
950
  // consolidate proposals once, so the per-chunk prompt can annotate
@@ -890,71 +953,86 @@ export async function akmConsolidate(opts = {}) {
890
953
  // 4h on this user's stack. See
891
954
  // /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
892
955
  const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
956
+ // ── Cold-start budget estimation ─────────────────────────────────────────────
957
+ // Estimate wall-clock cost BEFORE issuing any LLM calls. When a signal is
958
+ // provided and the estimated cost exceeds ~60% of the remaining budget we
959
+ // auto-reduce the pool and log the reduction so the run never starts work
960
+ // it cannot finish (avoiding SIGTERM mid-LLM-call).
961
+ //
962
+ // Formula: chunks.length × p90_chunk_seconds. The p90 comes from
963
+ // `opts.p90ChunkSecondsDefault` (caller-supplied, typically from the profile
964
+ // config); absent = 30 s (conservative default matching a medium local LLM).
965
+ //
966
+ // "Remaining budget" is read from a custom property on the AbortSignal if
967
+ // the caller (improve.ts) has attached one. Without it no auto-reduction
968
+ // fires but the check is still cheap to run.
969
+ if (chunks.length > 10 && opts.signal) {
970
+ const p90Chunk = opts.p90ChunkSecondsDefault ?? 30;
971
+ const estimatedSeconds = chunks.length * p90Chunk;
972
+ // remainingBudgetMs is a non-standard extension set by improve.ts when it
973
+ // creates the budget AbortController. Undefined = no budget information.
974
+ const budgetMs = opts.signal.remainingBudgetMs;
975
+ if (budgetMs !== undefined && budgetMs > 0) {
976
+ const remainingSeconds = budgetMs / 1000;
977
+ if (estimatedSeconds > remainingSeconds * 0.6) {
978
+ const safeCaps = Math.max(1, Math.floor((remainingSeconds * 0.6) / p90Chunk));
979
+ const removedChunks = chunks.length - safeCaps;
980
+ if (removedChunks > 0) {
981
+ const msg = `[consolidate] cold-start budget: estimated ${estimatedSeconds.toFixed(0)}s > 60% of remaining ${remainingSeconds.toFixed(0)}s; ` +
982
+ `reducing pool from ${chunks.length} to ${safeCaps} chunks (${removedChunks} deferred to next run).`;
983
+ warn(msg);
984
+ warnings.push(msg);
985
+ chunks.splice(safeCaps);
986
+ }
987
+ }
988
+ }
989
+ }
893
990
  warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
894
991
  ` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
992
+ // Consolidate output merges memories (non-wiki) → stash authoring standards.
993
+ // Resolved ONCE per run and passed to each chunk prompt (facts not re-read
994
+ // per chunk).
995
+ const standardsContext = resolveStashStandards(stashDir);
895
996
  const chunkOpsArrays = [];
896
- // Structured skip-reason histogram (2026-05-26): every deterministic
897
- // post-LLM op rejection site below also calls `pushSkipReason` so the
898
- // health rollup can aggregate without regex-parsing English warning
899
- // strings. See `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
900
- const skipReasons = [];
901
- // Per-ref grouping of skipReasons entries. A ref occupies exactly one
902
- // accounting bucket and therefore exactly one skipReasons array entry;
903
- // subsequent skip ops for the same ref append to that entry's `skips[]`
904
- // rather than pushing a second array entry (that would inflate
905
- // Σ(skipReasons) and break the invariant by +1 per duplicate).
906
- const skipReasonByRef = new Map();
907
- const pushSkipReason = (op, ref, reason) => {
908
- // 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
909
- // to judgedNoAction in its own chunk (a different chunk proposed an op
910
- // for it that is now being rejected here), promote it from the
911
- // judgedNoAction bucket into the more specific skipReason bucket.
912
- // Preserves the invariant: processed == actioned + judgedNoAction +
913
- // Σ(skipReasons) + failedChunkMemories.
914
- if (judgedNoActionRefs.delete(ref))
915
- judgedNoAction--;
916
- const existing = skipReasonByRef.get(ref);
917
- if (existing) {
918
- // Already counted once for accounting. Append the extra skip to the
919
- // ref's grouped entry for observability without adding a new array
920
- // entry (which would break the accounting invariant).
921
- existing.skips.push({ op, reason });
922
- return;
923
- }
924
- const entry = { ref, skips: [{ op, reason }] };
925
- skipReasonByRef.set(ref, entry);
926
- skipReasons.push(entry);
927
- };
928
997
  // judgedNoAction tracks memories the LLM saw inside a chunk but proposed
929
998
  // no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
930
- let judgedNoAction = 0;
931
- // 2026-05-27 cross-chunk double-count fix: refs that contributed to
932
- // judgedNoAction in their own chunk. When a different chunk's op references
933
- // one of these as a secondary and that op later fails, the ref would land
934
- // in BOTH judgedNoAction and skipReasons (delta +1 per occurrence). Track
935
- // the set so the merge-failure path can decrement and re-bucket.
936
- const judgedNoActionRefs = new Set();
937
- // 2026-05-26 accounting-leak fix: memories that belong to a chunk whose
938
- // LLM call failed before any per-chunk noAction calculation runs. They
939
- // would otherwise vanish from the envelope's accounting (no judgedNoAction
940
- // bump, no skipReasons entry, no actioned counter).
941
- let failedChunkMemories = 0;
942
- // 2026-05-26 accounting-leak fix: per-secondary tally so successful merges
943
- // account for `1 + secondaries.length` memories instead of 1.
944
- let mergedSecondaries = 0;
999
+ // The structured skip-reason histogram (2026-05-26) plus the cross-chunk
1000
+ // double-count fixes now live on `accounting`; every deterministic post-LLM
1001
+ // op rejection site calls `accounting.pushSkipReason`. See
1002
+ // `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
1003
+ //
1004
+ // Judged-state cache (#581): coarse outcome per memory NAME the LLM actually
1005
+ // judged in a successfully-parsed chunk this run. "actioned" = an op targeted
1006
+ // it; "no_action" = the LLM saw it and proposed nothing. Populated only when
1007
+ // the cache is enabled (otherwise it stays empty and the post-loop recording
1008
+ // step is a no-op). Memories in failed/aborted chunks are NOT recorded, so a
1009
+ // transient LLM failure never poisons the cache into skipping them next run.
1010
+ const judgedOutcomeByName = new Map();
945
1011
  // C-6 / #392: Replace two-consecutive-failures abort with failure-rate threshold.
946
1012
  // Consecutive-count policies are brittle against transient LM Studio reloads:
947
1013
  // two transient failures abort the run even though the next chunk would succeed.
948
1014
  // Rate-based abort (≥50% failure over ≥4 chunks) is more robust.
949
1015
  // Tanenbaum, Distributed Systems §8 — rate-based policies with minimum sample sizes.
950
1016
  let totalChunksProcessed = 0;
951
- let totalChunksFailed = 0;
952
1017
  const ABORT_MIN_CHUNKS = 4;
953
1018
  const ABORT_FAILURE_RATE = 0.5;
954
1019
  for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
1020
+ // Budget-signal check: break cleanly before the next LLM call if the
1021
+ // caller's budget has been exhausted. Commits work done so far.
1022
+ if (opts.signal?.aborted) {
1023
+ const skipped = chunks.length - chunkIdx;
1024
+ const msg = `[consolidate] budget signal aborted before chunk ${chunkIdx + 1}/${chunks.length}; ${skipped} chunk(s) not processed (partial_timeout — work done so far committed).`;
1025
+ warn(msg);
1026
+ warnings.push(msg);
1027
+ // Account for memories in unprocessed chunks.
1028
+ for (let i = chunkIdx; i < chunks.length; i++) {
1029
+ accounting.failedChunkMemories += chunks[i].length;
1030
+ }
1031
+ break;
1032
+ }
955
1033
  // Abort if failure rate >= 50% over at least 4 processed chunks.
956
1034
  if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
957
- const failureRate = totalChunksFailed / totalChunksProcessed;
1035
+ const failureRate = accounting.totalChunksFailed / totalChunksProcessed;
958
1036
  if (failureRate >= ABORT_FAILURE_RATE) {
959
1037
  const skipped = chunks.length - chunkIdx;
960
1038
  const abortMsg = `Consolidation aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${totalChunksProcessed} chunks (>= ${ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable. ${skipped} chunk(s) skipped.`;
@@ -965,7 +1043,7 @@ export async function akmConsolidate(opts = {}) {
965
1043
  // rejected). Without this, the accounting invariant fails by
966
1044
  // `Σ(unattempted_chunk.length)` whenever the abort fires.
967
1045
  for (let i = chunkIdx; i < chunks.length; i++) {
968
- failedChunkMemories += chunks[i].length;
1046
+ accounting.failedChunkMemories += chunks[i].length;
969
1047
  }
970
1048
  break;
971
1049
  }
@@ -983,22 +1061,23 @@ export async function akmConsolidate(opts = {}) {
983
1061
  // LLM-failure-rate abort policy — no request was attempted.
984
1062
  if (chunk.length > 0 && chunk.every((m) => isHotCapturedMemory(m.filePath))) {
985
1063
  for (const m of chunk)
986
- judgedNoActionRefs.add(`memory:${m.name}`);
987
- judgedNoAction += chunk.length;
1064
+ accounting.judgedNoActionRefs.add(`memory:${m.name}`);
1065
+ accounting.judgedNoAction += chunk.length;
988
1066
  warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length}: all ${chunk.length} memories are captureMode: hot — skipping LLM (judged no-action).`);
989
1067
  continue;
990
1068
  }
991
1069
  warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
992
- const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
993
- let raw = await tryLlmFeature("memory_consolidation", config, async () => {
1070
+ const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes, standardsContext);
1071
+ // Single chunk LLM call, wrapped in the feature gate. Deduplicated across
1072
+ // the first attempt and the retry below (the two blocks were byte-identical
1073
+ // apart from their fallback error string). responseSchema lift (PR 1,
1074
+ // asset-writers-investigation §5): providers with `supportsJsonSchema: true`
1075
+ // enforce the shape upstream; others fall through to
1076
+ // `parseEmbeddedJsonResponse` on the response side.
1077
+ const callChunkLlm = (fallbackError) => tryLlmFeature("memory_consolidation", config, async () => {
994
1078
  if (!llmConfig)
995
1079
  return { ok: false, error: "No LLM configured for consolidation" };
996
1080
  try {
997
- // responseSchema lift (PR 1, asset-writers-investigation §5): pass
998
- // the consolidate plan schema so providers with
999
- // `supportsJsonSchema: true` enforce shape upstream. Providers that
1000
- // ignore the option fall through to the existing
1001
- // `parseEmbeddedJsonResponse` path on the response side.
1002
1081
  const content = await chatCompletion(llmConfig, [
1003
1082
  { role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
1004
1083
  { role: "user", content: userPrompt },
@@ -1008,36 +1087,24 @@ export async function akmConsolidate(opts = {}) {
1008
1087
  catch (e) {
1009
1088
  return { ok: false, error: String(e) };
1010
1089
  }
1011
- }, { ok: false, error: `chunk ${chunkIdx + 1} failed` });
1090
+ }, { ok: false, error: fallbackError });
1091
+ let raw = await callChunkLlm(`chunk ${chunkIdx + 1} failed`);
1012
1092
  if (!raw.ok) {
1013
1093
  // Single retry with 2s backoff before recording chunk as lost.
1014
1094
  // Recovers transient Shredder LM Studio timeouts without significantly
1015
1095
  // extending run time. Only marks failed if both attempts fail.
1016
1096
  await new Promise((r) => setTimeout(r, 2_000));
1017
- const retry = await tryLlmFeature("memory_consolidation", config, async () => {
1018
- if (!llmConfig)
1019
- return { ok: false, error: "No LLM configured for consolidation" };
1020
- try {
1021
- const content = await chatCompletion(llmConfig, [
1022
- { role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
1023
- { role: "user", content: userPrompt },
1024
- ], { responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA, enableThinking: false });
1025
- return { ok: true, content };
1026
- }
1027
- catch (e) {
1028
- return { ok: false, error: String(e) };
1029
- }
1030
- }, { ok: false, error: `chunk ${chunkIdx + 1} retry failed` });
1097
+ const retry = await callChunkLlm(`chunk ${chunkIdx + 1} retry failed`);
1031
1098
  if (!retry.ok) {
1032
1099
  warn(retry.error ?? `chunk ${chunkIdx + 1} failed after retry`);
1033
1100
  warnings.push(retry.error ?? `chunk ${chunkIdx + 1} failed after retry`);
1034
1101
  totalChunksProcessed++;
1035
- totalChunksFailed++;
1102
+ accounting.totalChunksFailed++;
1036
1103
  // Account for the chunk's memories under the failed-chunk bucket.
1037
1104
  // judgedNoAction does NOT run on this path (it's after the success
1038
1105
  // guards) so without this the accounting invariant breaks on every
1039
1106
  // chunk-level transport/parse failure.
1040
- failedChunkMemories += chunk.length;
1107
+ accounting.failedChunkMemories += chunk.length;
1041
1108
  continue;
1042
1109
  }
1043
1110
  raw = retry;
@@ -1054,8 +1121,8 @@ export async function akmConsolidate(opts = {}) {
1054
1121
  warn(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
1055
1122
  warnings.push(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
1056
1123
  totalChunksProcessed++;
1057
- totalChunksFailed++;
1058
- failedChunkMemories += chunk.length;
1124
+ accounting.totalChunksFailed++;
1125
+ accounting.failedChunkMemories += chunk.length;
1059
1126
  continue;
1060
1127
  }
1061
1128
  totalChunksProcessed++; // success
@@ -1095,41 +1162,185 @@ export async function akmConsolidate(opts = {}) {
1095
1162
  const memRef = `memory:${m.name}`;
1096
1163
  if (!targetRefs.has(memRef)) {
1097
1164
  chunkNoAction++;
1098
- judgedNoActionRefs.add(memRef);
1165
+ accounting.judgedNoActionRefs.add(memRef);
1166
+ // Judged-state cache (#581): the LLM saw this memory and proposed
1167
+ // nothing → record judged-unchanged so the next run can skip it.
1168
+ if (judgedCacheEnabled)
1169
+ judgedOutcomeByName.set(m.name, "no_action");
1170
+ }
1171
+ else if (judgedCacheEnabled) {
1172
+ // An op targeted this memory → it was judged + actioned.
1173
+ judgedOutcomeByName.set(m.name, "actioned");
1099
1174
  }
1100
1175
  }
1101
- judgedNoAction += chunkNoAction;
1176
+ accounting.judgedNoAction += chunkNoAction;
1102
1177
  chunkOpsArrays.push(ops);
1103
1178
  }
1179
+ // ── Judged-state cache recording (#581) ─────────────────────────────────────
1180
+ // Persist judged state for every memory the LLM actually judged this run so
1181
+ // the next run can skip the unchanged ones. Keyed by current content hash so
1182
+ // a later body edit (different hash) re-enters the LLM pool. DEFAULT OFF and
1183
+ // skipped under --dry-run (dry-run mutates nothing). Failed/aborted chunks
1184
+ // contributed no entries to `judgedOutcomeByName`, so a transient LLM outage
1185
+ // never caches a memory as judged.
1186
+ if (judgedCacheEnabled && !opts.dryRun && judgedOutcomeByName.size > 0) {
1187
+ // Use the shared state.db handle; open a local one as fallback.
1188
+ const doRecord = (db) => {
1189
+ const judgedAt = new Date(startMs).toISOString();
1190
+ for (const [name, outcome] of judgedOutcomeByName) {
1191
+ const hash = currentHashByName.get(name);
1192
+ if (hash === undefined)
1193
+ continue;
1194
+ upsertConsolidationJudged(db, {
1195
+ entryKey: `memory:${name}`,
1196
+ contentHash: hash,
1197
+ judgedAt,
1198
+ outcome,
1199
+ });
1200
+ }
1201
+ };
1202
+ if (sharedStateDb) {
1203
+ try {
1204
+ doRecord(sharedStateDb);
1205
+ }
1206
+ catch (e) {
1207
+ warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
1208
+ }
1209
+ }
1210
+ else {
1211
+ try {
1212
+ withStateDb((localDb) => doRecord(localDb));
1213
+ }
1214
+ catch (e) {
1215
+ warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
1216
+ }
1217
+ }
1218
+ }
1104
1219
  // Build the known-refs set from the already-filtered memory pool so
1105
1220
  // mergePlans() can reject LLM-hallucinated primary refs before execution.
1106
1221
  const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
1107
1222
  const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays, knownRefs);
1108
1223
  warnings.push(...mergeWarnings);
1224
+ return { allOps, totalChunks: chunks.length, llmPoolSize, embedTelemetry, isHttpPath, sourceName };
1225
+ }
1226
+ /**
1227
+ * Pass 3 — execute the reconciled plan against the filesystem: resolve the
1228
+ * write target, journal the batch, dispatch each op to its handler, then commit
1229
+ * the batch at the boundary and clean up the journal. Mutates `accounting` via
1230
+ * the op-handlers' `pushSkipReason`. Behavior-identical to the former inlined
1231
+ * write block. Never invoked on the dry-run or aborted-confirm paths.
1232
+ */
1233
+ async function applyConsolidationPlan(config, stashDir, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, activeProfile) {
1234
+ // -- Phase B + writes -------------------------------------------------------
1235
+ const target = resolveWriteTarget(config);
1236
+ const timestamp = timestampForFilename();
1237
+ const backupDir = getBackupDir(stashDir, timestamp);
1238
+ // Write journal before any mutations
1239
+ writeJournal(stashDir, allOps, timestamp);
1240
+ const counts = {
1241
+ merged: 0,
1242
+ deleted: 0,
1243
+ contradicted: 0, // C-3 / #382: count of contradiction edges written
1244
+ mergeFloorViolations: 0, // R5 §4.2: advisory merge-information-floor failures
1245
+ mergedSecondaries: 0,
1246
+ };
1247
+ const promoted = [];
1248
+ // Within-run dedup: track source refs for which a promote proposal was
1249
+ // already created this run. The LLM can return multiple promote ops for
1250
+ // different source memories that happen to have identical content (all are
1251
+ // duplicate memories), so we also need a content-hash guard below.
1252
+ const promotedSourceRefs = new Set();
1253
+ // Build a lookup map: ref → MemoryEntry
1254
+ const memoryByRef = new Map();
1255
+ for (const m of memories) {
1256
+ memoryByRef.set(`memory:${m.name}`, m);
1257
+ }
1258
+ const opCtx = {
1259
+ config,
1260
+ improveProfile: activeProfile,
1261
+ stashDir,
1262
+ sourceRun,
1263
+ target,
1264
+ backupDir,
1265
+ memoryByRef,
1266
+ promoted,
1267
+ promotedSourceRefs,
1268
+ warnings,
1269
+ counts,
1270
+ pushSkipReason: accounting.pushSkipReason,
1271
+ };
1272
+ // Thin dispatch over the op discriminator — each branch is now an isolated,
1273
+ // independently-testable handler that mutates `opCtx`.
1274
+ for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
1275
+ const op = allOps[opIndex];
1276
+ const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
1277
+ warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
1278
+ switch (op.op) {
1279
+ case "merge":
1280
+ await handleMergeOp(op, opIndex, opCtx);
1281
+ break;
1282
+ case "delete":
1283
+ await handleDeleteOp(op, opIndex, opCtx);
1284
+ break;
1285
+ case "promote":
1286
+ await handlePromoteOp(op, opCtx);
1287
+ break;
1288
+ case "contradict":
1289
+ await handleContradictOp(op, opCtx);
1290
+ break;
1291
+ }
1292
+ }
1293
+ const { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries } = counts;
1294
+ // 0.9.0 (issue #507): batch-at-boundary commit. The merge/delete loop above
1295
+ // wrote one merged primary and deleted N secondaries to the resolved target
1296
+ // with NO per-asset commit. If the target is a writable git source and any
1297
+ // asset was mutated, commit the whole batch ONCE here (stages .akm/ +
1298
+ // siblings together). No-op for filesystem/primary-stash targets.
1299
+ if (merged > 0 || deleted > 0) {
1300
+ commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
1301
+ }
1302
+ cleanupJournal(stashDir, timestamp);
1303
+ // [signoff 2026-06-15] TTL archive cleanup machinery RETIRED (WS-3a).
1304
+ // The elaborate archiveRetentionDays / archive-dir scan existed only to satisfy
1305
+ // the old irrecoverability constraint. Stashes are now git-backed, so git
1306
+ // history is the recovery path — no bespoke archive TTL needed. Any files in
1307
+ // .akm/archive/ will stay there harmlessly until the operator prunes them with
1308
+ // `git rm` or `find .akm/archive -mtime +90 -delete`. Changed N files this
1309
+ // run; recover any via `git show <sha>:<path>` or `git restore <path>`.
1310
+ if (merged > 0 || deleted > 0 || dedupCollapsed > 0) {
1311
+ const totalChanged = merged + deleted + dedupCollapsed;
1312
+ warnings.push(`Changed ${totalChanged} file(s) this run. Recover any via git if needed (git history is the backstop).`);
1313
+ }
1314
+ return { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries, promoted };
1315
+ }
1316
+ async function akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb) {
1317
+ // -- Pass 1: narrow the memory pool (may early-return an envelope) ----------
1318
+ const narrowed = await narrowConsolidationPool(opts, config, stashDir, startMs, warnings, sharedStateDb);
1319
+ if (narrowed.done)
1320
+ return narrowed.result;
1321
+ const { memories, dedupCollapsed, perfMs, judgedCacheEnabled, currentHashByName } = narrowed;
1322
+ // -- Pass 2: build the LLM plan (populates the shared accounting counters) ---
1323
+ const accounting = createConsolidateAccounting();
1324
+ const { allOps, totalChunks, llmPoolSize, embedTelemetry, isHttpPath, sourceName } = await planConsolidation(opts, config, stashDir, startMs, memories, warnings, sharedStateDb, judgedCacheEnabled, currentHashByName, accounting);
1109
1325
  // -- Dry-run: show AI plan without executing any writes --------------------
1110
1326
  if (opts.dryRun) {
1111
- return {
1112
- schemaVersion: 1,
1113
- ok: true,
1114
- shape: "consolidate-result",
1327
+ return makeConsolidateResult({
1115
1328
  dryRun: true,
1116
1329
  previewOnly: true,
1117
1330
  target: sourceName,
1118
1331
  processed: memories.length,
1119
- merged: 0,
1120
- deleted: 0,
1121
- promoted: [],
1122
- contradicted: 0,
1123
- failedChunks: totalChunksFailed,
1124
- totalChunks: chunks.length,
1125
- judgedNoAction,
1126
- skipReasons,
1127
- mergedSecondaries,
1128
- failedChunkMemories,
1332
+ failedChunks: accounting.totalChunksFailed,
1333
+ totalChunks,
1334
+ judgedNoAction: accounting.judgedNoAction,
1335
+ skipReasons: accounting.skipReasons,
1336
+ // No merge has executed on the preview path — the per-secondary tally is
1337
+ // provably still 0 here (it only increments in the op-execution loop).
1338
+ mergedSecondaries: 0,
1339
+ failedChunkMemories: accounting.failedChunkMemories,
1129
1340
  planned: allOps,
1130
1341
  warnings,
1131
1342
  durationMs: Date.now() - startMs,
1132
- };
1343
+ });
1133
1344
  }
1134
1345
  warn(`[consolidate] plan: ${allOps.length} operation(s)`);
1135
1346
  // -- HTTP path: warn about quality and confirm unless auto-accepted --------
@@ -1150,555 +1361,28 @@ export async function akmConsolidate(opts = {}) {
1150
1361
  const nonInteractive = process.stdin.isTTY === false || process.env.AKM_NON_INTERACTIVE === "1";
1151
1362
  const answer = nonInteractive ? false : await promptConfirm(`Apply ${n} operations? [y/N] `);
1152
1363
  if (!answer) {
1153
- return {
1154
- schemaVersion: 1,
1155
- ok: true,
1156
- shape: "consolidate-result",
1157
- dryRun: false,
1364
+ return makeConsolidateResult({
1158
1365
  previewOnly: true,
1159
1366
  target: sourceName,
1160
1367
  processed: memories.length,
1161
- merged: 0,
1162
- deleted: 0,
1163
- promoted: [],
1164
- contradicted: 0,
1165
- failedChunks: totalChunksFailed,
1166
- totalChunks: chunks.length,
1167
- judgedNoAction,
1168
- skipReasons,
1169
- mergedSecondaries,
1170
- failedChunkMemories,
1368
+ failedChunks: accounting.totalChunksFailed,
1369
+ totalChunks,
1370
+ judgedNoAction: accounting.judgedNoAction,
1371
+ skipReasons: accounting.skipReasons,
1372
+ // No merge executed on the abort path — mergedSecondaries is still 0.
1373
+ mergedSecondaries: 0,
1374
+ failedChunkMemories: accounting.failedChunkMemories,
1171
1375
  planned: allOps,
1172
1376
  warnings: [...warnings, nonInteractive ? "Non-interactive context: skipped apply." : "Aborted by user."],
1173
1377
  durationMs: Date.now() - startMs,
1174
- };
1175
- }
1176
- }
1177
- }
1178
- // -- Phase B + writes -------------------------------------------------------
1179
- const target = resolveWriteTarget(config);
1180
- const timestamp = timestampForFilename();
1181
- const backupDir = getBackupDir(stashDir, timestamp);
1182
- // Write journal before any mutations
1183
- writeJournal(stashDir, allOps, timestamp);
1184
- let merged = 0;
1185
- let deleted = 0;
1186
- const promoted = [];
1187
- let contradicted = 0; // C-3 / #382: count of contradiction edges written
1188
- // Within-run dedup: track source refs for which a promote proposal was
1189
- // already created this run. The LLM can return multiple promote ops for
1190
- // different source memories that happen to have identical content (all are
1191
- // duplicate memories), so we also need a content-hash guard below.
1192
- const promotedSourceRefs = new Set();
1193
- // Build a lookup map: ref → MemoryEntry
1194
- const memoryByRef = new Map();
1195
- for (const m of memories) {
1196
- memoryByRef.set(`memory:${m.name}`, m);
1197
- }
1198
- for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
1199
- const op = allOps[opIndex];
1200
- const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
1201
- warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
1202
- if (op.op === "merge") {
1203
- // Accounting helper: emit a per-participant skipReason for failed
1204
- // merges so primary + every loaded-memory secondary land in the
1205
- // structured skip histogram. Pre-2026-05-26 only the primary was
1206
- // counted (1 skipReason per failed merge), leaving N secondaries
1207
- // unaccounted for in the `processed == actioned + noAction + Σskips`
1208
- // invariant — the source of the 4–11 silent leaks per run.
1209
- const emitMergeFailureSkips = (reason) => {
1210
- if (memoryByRef.has(op.primary))
1211
- pushSkipReason("merge", op.primary, reason);
1212
- for (const secRef of op.secondaries) {
1213
- if (memoryByRef.has(secRef))
1214
- pushSkipReason("merge", secRef, reason);
1215
- }
1216
- };
1217
- const primaryEntry = memoryByRef.get(op.primary);
1218
- if (!primaryEntry) {
1219
- // This fires when a prior op in the same run consumed this ref as a
1220
- // secondary and Fix-A pruned it from memoryByRef. It should NOT fire
1221
- // for hallucinated primaries (those are dropped by mergePlans() before
1222
- // reaching here). If this counter is non-zero, suspect an intra-run
1223
- // cross-chunk race, not a filter regression.
1224
- warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
1225
- emitMergeFailureSkips("merge_primary_missing");
1226
- continue;
1227
- }
1228
- // Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
1229
- // before this run's own ops), the file may have been deleted by a
1230
- // concurrent process or an edge case the pre-flight filter missed.
1231
- if (!fs.existsSync(primaryEntry.filePath)) {
1232
- warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
1233
- emitMergeFailureSkips("merge_primary_file_gone");
1234
- continue;
1235
- }
1236
- // Phase B: generate merged content
1237
- const secondaryBodies = [];
1238
- for (const secRef of op.secondaries) {
1239
- const secEntry = memoryByRef.get(secRef);
1240
- if (!secEntry) {
1241
- warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
1242
- // No accounting impact: a missing secondary is a phantom ref and
1243
- // never contributed to any chunk's targetRefs reduction. We still
1244
- // continue the loop to gather the remaining valid secondaries.
1245
- continue;
1246
- }
1247
- secondaryBodies.push(secRef);
1248
- }
1249
- if (secondaryBodies.length === 0) {
1250
- warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
1251
- emitMergeFailureSkips("merge_no_valid_secondaries");
1252
- continue;
1253
- }
1254
- // Pre-flight hot guard — skip the LLM call entirely if any participant
1255
- // is hot or unparseable. Without this, mixed chunks still send hot merges
1256
- // to the planner which proposes them; generateMergedContent() is then
1257
- // called, produces output without `description`, and the skip is
1258
- // misattributed to merge_missing_description instead of the real cause.
1259
- const preflightParticipants = [op.primary, ...op.secondaries];
1260
- const preflightBlocked = preflightParticipants.flatMap((ref) => {
1261
- const e = memoryByRef.get(ref);
1262
- if (!e)
1263
- return [];
1264
- const verdict = consolidateGuardStatus(e.filePath);
1265
- if (verdict === "hot" || verdict === "unparseable")
1266
- return [{ ref, verdict }];
1267
- return [];
1268
- });
1269
- if (preflightBlocked.length > 0) {
1270
- const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1271
- warnings.push(`Merge: refused for ${op.primary} — ${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
1272
- emitMergeFailureSkips("merge_participant_blocked");
1273
- continue;
1274
- }
1275
- let primaryBody = "";
1276
- try {
1277
- primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
1278
- }
1279
- catch {
1280
- warnings.push(`Merge: could not read primary ${op.primary} — skipping.`);
1281
- emitMergeFailureSkips("merge_read_failed");
1282
- continue;
1283
- }
1284
- const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef);
1285
- if ("error" in mergeResult) {
1286
- warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
1287
- emitMergeFailureSkips(mergeResult.error);
1288
- continue;
1289
- }
1290
- const mergedContent = mergeResult.content;
1291
- // Validate frontmatter of merged content — must have a `---` block
1292
- // with at minimum a `description` field. We parse via the hand-rolled
1293
- // parser (cheap) AND require non-empty description. This guards against
1294
- // the historical defect where merged memories were written back with
1295
- // empty `description` and later polluted the promote path.
1296
- let parsedMerged;
1297
- try {
1298
- parsedMerged = parseFrontmatter(mergedContent);
1299
- }
1300
- catch {
1301
- warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
1302
- emitMergeFailureSkips("merge_invalid_frontmatter");
1303
- continue;
1304
- }
1305
- if (parsedMerged.frontmatter === null) {
1306
- warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
1307
- emitMergeFailureSkips("merge_invalid_frontmatter");
1308
- continue;
1309
- }
1310
- const mergedDesc = parsedMerged.data.description;
1311
- if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
1312
- warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
1313
- emitMergeFailureSkips("merge_missing_description");
1314
- continue;
1315
- }
1316
- const truncReason = detectTruncatedDescription(mergedDesc);
1317
- if (truncReason) {
1318
- warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
1319
- emitMergeFailureSkips("merge_truncated_description");
1320
- continue;
1321
- }
1322
- // captureMode:hot guard — refuse the merge if ANY participating memory
1323
- // (primary or secondary) was user-captured or has unparseable frontmatter
1324
- // (could have hidden a hot flag). Hot memories are user-explicit and
1325
- // must not be deleted/overwritten by the consolidate LLM. 14 user
1326
- // memories were silent-deleted by consolidate before this guard landed;
1327
- // recovery required copying from .akm/archive/ by hand.
1328
- const mergeParticipants = [op.primary, ...op.secondaries];
1329
- const blockedParticipants = mergeParticipants.flatMap((ref) => {
1330
- const e = memoryByRef.get(ref);
1331
- if (!e)
1332
- return [];
1333
- const verdict = consolidateGuardStatus(e.filePath);
1334
- if (verdict === "hot" || verdict === "unparseable")
1335
- return [{ ref, verdict }];
1336
- return [];
1337
- });
1338
- if (blockedParticipants.length > 0) {
1339
- const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1340
- warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
1341
- emitMergeFailureSkips("merge_participant_blocked");
1342
- continue;
1343
- }
1344
- // Backup secondaries before deleting
1345
- for (const secRef of op.secondaries) {
1346
- const secEntry = memoryByRef.get(secRef);
1347
- if (secEntry && fs.existsSync(secEntry.filePath)) {
1348
- backupFile(secEntry.filePath, backupDir, secEntry.name);
1349
- }
1350
- }
1351
- // Write merged primary
1352
- try {
1353
- const parsedPrimary = parseAssetRef(op.primary);
1354
- await writeAssetToSource(target.source, target.config, parsedPrimary, mergedContent);
1355
- }
1356
- catch (e) {
1357
- warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
1358
- emitMergeFailureSkips("merge_write_failed");
1359
- continue;
1360
- }
1361
- // Archive and delete secondaries (P1-B: soft-invalidation)
1362
- for (const secRef of op.secondaries) {
1363
- const secEntry = memoryByRef.get(secRef);
1364
- if (!secEntry)
1365
- continue;
1366
- if (fs.existsSync(secEntry.filePath)) {
1367
- archiveMemory(secEntry.filePath, stashDir, secRef, "merged into primary", opIndex, op.primary, warnings);
1368
- }
1369
- try {
1370
- const parsedSec = parseAssetRef(secRef);
1371
- await deleteAssetFromSource(target.source, target.config, parsedSec);
1372
- markJournalCompleted(stashDir, secRef);
1373
- }
1374
- catch (e) {
1375
- warnings.push(`Merge: delete failed for ${secRef}: ${String(e)}`);
1376
- }
1377
- }
1378
- markJournalCompleted(stashDir, op.primary);
1379
- merged++;
1380
- // 2026-05-26 accounting-leak fix: `merged` is op-level, but each
1381
- // successful merge actions `1 + secondaries.length` memories. Without
1382
- // this counter the accounting invariant breaks by `secondaries.length`
1383
- // per successful merge (chunk loop excluded all secondaries from
1384
- // judgedNoAction via targetRefs, but only the primary is credited to
1385
- // `merged`). Count only loaded-memory secondaries; phantom secondary
1386
- // refs never affected any chunk's targetRefs in the first place.
1387
- for (const secRef of op.secondaries) {
1388
- if (memoryByRef.has(secRef))
1389
- mergedSecondaries++;
1390
- }
1391
- // Prune consumed refs from memoryByRef so later ops in this run cannot
1392
- // reference an absorbed secondary as a merge primary and proceed with a
1393
- // stale entry. Primary is rewritten (not deleted), so we only remove
1394
- // secondaries; the primary ref remains valid under its new content.
1395
- for (const secRef of op.secondaries) {
1396
- memoryByRef.delete(secRef);
1397
- }
1398
- }
1399
- else if (op.op === "delete") {
1400
- const entry = memoryByRef.get(op.ref);
1401
- if (!entry) {
1402
- warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
1403
- // Phantom ref: not in the batch so not in processed. Pushing to
1404
- // skipReasons would inflate Σ(skipReasons) without a matching processed
1405
- // entry, breaking the accounting invariant. Visibility is preserved via
1406
- // the warnings array above.
1407
- continue;
1408
- }
1409
- // captureMode:hot guard — refuse to delete user-captured memories OR
1410
- // memories whose frontmatter is unparseable (could have hidden the hot
1411
- // flag). The consolidate LLM was deleting hot-captured user memos as
1412
- // "redundant" — 14 such deletes were silently archived between
1413
- // 2026-05-19 and 2026-05-20 before this guard. Hot memories are
1414
- // user-explicit and may only be deleted by the user.
1415
- const guard = consolidateGuardStatus(entry.filePath);
1416
- if (guard === "hot" || guard === "unparseable") {
1417
- warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
1418
- pushSkipReason("delete", op.ref, "captureMode_hot_refused");
1419
- continue;
1420
- }
1421
- if (fs.existsSync(entry.filePath)) {
1422
- backupFile(entry.filePath, backupDir, entry.name);
1423
- // P1-B: soft-invalidation archive before hard delete
1424
- archiveMemory(entry.filePath, stashDir, op.ref, op.reason, opIndex, undefined, warnings);
1425
- }
1426
- try {
1427
- const parsedRef = parseAssetRef(op.ref);
1428
- await deleteAssetFromSource(target.source, target.config, parsedRef);
1429
- markJournalCompleted(stashDir, op.ref);
1430
- deleted++;
1431
- // Prune from memoryByRef so later ops in this run cannot reference a
1432
- // deleted memory as a merge primary or secondary.
1433
- memoryByRef.delete(op.ref);
1434
- }
1435
- catch (e) {
1436
- // Distinguish "file already absent" from genuine failures. A prior run
1437
- // may have deleted the file but the DB was not yet re-indexed, so the
1438
- // ref still appeared in memoryByRef. The delete goal is already met.
1439
- const msg = e instanceof Error ? e.message : String(e);
1440
- if (msg.includes("not found in source")) {
1441
- warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
1442
- pushSkipReason("delete", op.ref, "delete_already_gone");
1443
- }
1444
- else {
1445
- warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
1446
- pushSkipReason("delete", op.ref, "delete_failed");
1447
- }
1448
- }
1449
- }
1450
- else if (op.op === "promote") {
1451
- const entry = memoryByRef.get(op.ref);
1452
- if (!entry) {
1453
- warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
1454
- // Phantom ref: not in processed, so no skipReason (same rationale as
1455
- // delete_ref_missing above).
1456
- continue;
1457
- }
1458
- // Within-run source-ref dedup: skip if this source memory was already
1459
- // promoted earlier in this run (safety belt — mergePlans already
1460
- // deduplicates promote ops by source ref via Map, but this guard also
1461
- // catches any future code paths that bypass mergePlans).
1462
- if (promotedSourceRefs.has(op.ref)) {
1463
- warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
1464
- pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
1465
- continue;
1466
- }
1467
- let knowledgeRef = op.knowledgeRef;
1468
- try {
1469
- parseAssetRef(knowledgeRef);
1470
- }
1471
- catch {
1472
- const slug = op.knowledgeRef
1473
- .replace(/^knowledge:/, "")
1474
- .replace(/[^a-z0-9-]/gi, "-")
1475
- .toLowerCase();
1476
- knowledgeRef = `knowledge:${slug}`;
1477
- warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
1478
- }
1479
- // Idempotency: check pending proposals by target ref
1480
- const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
1481
- if (existingProposals.some((p) => p.status === "pending")) {
1482
- warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
1483
- pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
1484
- continue;
1485
- }
1486
- // Idempotency: check if knowledge asset already exists
1487
- const parsedKnowledgeRef = parseAssetRef(knowledgeRef);
1488
- const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
1489
- if (fs.existsSync(destPath)) {
1490
- warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
1491
- pushSkipReason("promote", op.ref, "promote_already_exists");
1492
- continue;
1493
- }
1494
- let memoryContent = "";
1495
- try {
1496
- memoryContent = fs.readFileSync(entry.filePath, "utf8");
1497
- }
1498
- catch (e) {
1499
- warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
1500
- pushSkipReason("promote", op.ref, "promote_read_failed");
1501
- continue;
1502
- }
1503
- // Defensive sanitization: legacy memory files written by older
1504
- // consolidate runs may still carry outer code fences or broken YAML.
1505
- // Strip them here so we never propose a polluted asset.
1506
- const promoteSanitized = sanitizeMergedContent(memoryContent);
1507
- if (!promoteSanitized.ok) {
1508
- warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
1509
- pushSkipReason("promote", op.ref, "promote_sanitization_failed");
1510
- continue;
1511
- }
1512
- memoryContent = promoteSanitized.result.content;
1513
- // SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
1514
- // frontmatter carries `status: superseded`. Predicate at module top
1515
- // (`hasSupersededStatus`) so tests can exercise it directly.
1516
- if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
1517
- warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
1518
- pushSkipReason("promote", op.ref, "promote_superseded");
1519
- continue;
1520
- }
1521
- // Parse the source memory up-front so the body/frontmatter checks below
1522
- // share the same parsed view.
1523
- const parsedMemory = parseFrontmatter(memoryContent);
1524
- // Reject sources whose body is too small to make useful knowledge.
1525
- // Observed failure: memory files whose body is literally a tags string
1526
- // ("discord,notification,send-notification") get promoted to knowledge
1527
- // proposals that no reviewer would accept. Threshold is conservative —
1528
- // 100 chars catches single-line tag dumps without rejecting genuinely
1529
- // terse but valid notes.
1530
- const PROMOTE_BODY_MIN_CHARS = 100;
1531
- const sourceBody = parsedMemory.content.trim();
1532
- if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
1533
- warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
1534
- pushSkipReason("promote", op.ref, "promote_source_too_small");
1535
- continue;
1536
- }
1537
- // Cross-run + within-run content dedup: if an identical body already
1538
- // exists in ANY pending consolidate proposal (regardless of target ref),
1539
- // skip. This prevents duplicate proposals when:
1540
- // (a) Multiple source memories have identical bodies but differ only
1541
- // in noise frontmatter (`inferenceProcessed: true` twin alongside
1542
- // the original; differing `updated:` timestamps; etc.) — the body
1543
- // is the load-bearing content, so dedup must hash on body only.
1544
- // (b) A prior run created a proposal for the same body under a
1545
- // different knowledgeRef slug.
1546
- const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
1547
- const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1548
- const contentDupProposal = allPendingConsolidateProposals.find((p) => {
1549
- const otherBody = parseFrontmatter(p.payload.content).content.trim();
1550
- return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
1551
- });
1552
- if (contentDupProposal) {
1553
- warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
1554
- pushSkipReason("promote", op.ref, "dedup_pending_proposal");
1555
- continue;
1556
- }
1557
- try {
1558
- // Use LLM-provided description; fall back to memory's own description
1559
- // (post-sanitization frontmatter is authoritative).
1560
- const description = (typeof op.description === "string" && op.description.trim()
1561
- ? op.description.trim()
1562
- : parsedMemory.data?.description?.trim()) ?? "";
1563
- // Validate the resolved frontmatter before emitting a proposal.
1564
- // Required field: non-empty description. Reject obvious truncation
1565
- // markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
1566
- // so the queue never sees half-formed metadata that the reviewer
1567
- // would only reject.
1568
- const fmCheck = validateProposalFrontmatter({ description });
1569
- if (!fmCheck.ok) {
1570
- warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
1571
- pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
1572
- continue;
1573
- }
1574
- // Merge `description` INTO the body's YAML frontmatter so it lands in
1575
- // the on-disk asset when the proposal is accepted. The descriptionQuality
1576
- // validator parses `payload.content` body (not the envelope
1577
- // `payload.frontmatter`), and a memory's native frontmatter has
1578
- // `captureMode`/`beliefState`/etc. but never `description` — without
1579
- // this merge, 60+ pending proposals were blocked at accept-time with
1580
- // MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
1581
- // (The body-frontmatter assumption baked into the 2026-05-20 comment
1582
- // below was wrong: body fm and envelope fm only converge when the
1583
- // writer explicitly merges them, which it now does.)
1584
- const mergedBodyFm = {
1585
- ...(parsedMemory.data ?? {}),
1586
- description,
1587
- };
1588
- const serializedMergedFm = serializeFrontmatter(mergedBodyFm);
1589
- const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
1590
- // Pre-emit dedup against pending consolidate proposals from the
1591
- // same improve run (slug-variant match). The cross-run content-hash
1592
- // dedup inside `mergePlans` handles duplicates against existing
1593
- // stash assets — see commit history for the deletion of the
1594
- // unbounded embedding + cross-type slug branches.
1595
- const dedup = await checkPreEmitDedup({
1596
- candidateRef: knowledgeRef,
1597
- candidateText: `${description}. ${memoryContent}`,
1598
- stashDir,
1599
- config,
1600
1378
  });
1601
- if (dedup.duplicate) {
1602
- warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
1603
- pushSkipReason("promote", op.ref, "promote_dedup_window");
1604
- continue;
1605
- }
1606
- const proposalResult = createProposal(stashDir, {
1607
- ref: knowledgeRef,
1608
- source: "consolidate",
1609
- sourceRun,
1610
- payload: {
1611
- content: proposalContent,
1612
- frontmatter: { description },
1613
- },
1614
- ...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
1615
- });
1616
- if (isProposalSkipped(proposalResult)) {
1617
- warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
1618
- pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
1619
- }
1620
- else {
1621
- promoted.push(proposalResult.id);
1622
- promotedSourceRefs.add(op.ref);
1623
- markJournalCompleted(stashDir, op.ref);
1624
- }
1625
- }
1626
- catch (e) {
1627
- warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
1628
- pushSkipReason("promote", op.ref, "promote_create_failed");
1629
- }
1630
- }
1631
- else if (op.op === "contradict") {
1632
- // C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
1633
- // (the SCC resolver in memory-improve.ts) has edges to work on.
1634
- // Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
1635
- const entry = memoryByRef.get(op.ref);
1636
- const contradictorEntry = memoryByRef.get(op.contradictedByRef);
1637
- if (!entry) {
1638
- warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
1639
- // Phantom ref: not in processed, so no skipReason (same rationale as
1640
- // delete_ref_missing).
1641
- continue;
1642
- }
1643
- if (!contradictorEntry) {
1644
- warnings.push(`Contradict: ${op.contradictedByRef} not found — skipping.`);
1645
- // op.ref IS in the batch (entry found above) so the skipReason is
1646
- // correctly charged against a real processed memory.
1647
- pushSkipReason("contradict", op.ref, "contradict_target_missing");
1648
- continue;
1649
- }
1650
- try {
1651
- // Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
1652
- writeContradictEdge(entry.filePath, op.contradictedByRef);
1653
- contradicted++;
1654
- markJournalCompleted(stashDir, op.ref);
1655
- }
1656
- catch (e) {
1657
- warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
1658
- pushSkipReason("contradict", op.ref, "contradict_write_failed");
1659
- }
1660
- }
1661
- }
1662
- // 0.9.0 (issue #507): batch-at-boundary commit. The merge/delete loop above
1663
- // wrote one merged primary and deleted N secondaries to the resolved target
1664
- // with NO per-asset commit. If the target is a writable git source and any
1665
- // asset was mutated, commit the whole batch ONCE here (stages .akm/ +
1666
- // siblings together). No-op for filesystem/primary-stash targets.
1667
- if (merged > 0 || deleted > 0) {
1668
- commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
1669
- }
1670
- cleanupJournal(stashDir, timestamp);
1671
- // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
1672
- // C-5 / #391: emit an `archive_cleanup` event before each deletion so the
1673
- // audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf) —
1674
- // any event that is recorded must be queryable; silent deletes are an anti-pattern.
1675
- const archiveDir = path.join(stashDir, ".akm", "archive");
1676
- if (fs.existsSync(archiveDir)) {
1677
- const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
1678
- const cutoff = Date.now() - retentionMs;
1679
- for (const fname of fs.readdirSync(archiveDir)) {
1680
- const fp = path.join(archiveDir, fname);
1681
- try {
1682
- const stat = fs.statSync(fp);
1683
- if (stat.mtimeMs < cutoff) {
1684
- // Emit event before deletion so the record survives the purge.
1685
- appendEvent({
1686
- eventType: "archive_cleanup",
1687
- metadata: {
1688
- file: fname,
1689
- filePath: fp,
1690
- ageMs: Date.now() - stat.mtimeMs,
1691
- retentionMs,
1692
- },
1693
- });
1694
- fs.unlinkSync(fp);
1695
- }
1696
- }
1697
- catch {
1698
- /* ignore race conditions */
1699
1379
  }
1700
1380
  }
1701
1381
  }
1382
+ // -- Pass 3: execute the plan against the filesystem ------------------------
1383
+ const { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries, promoted } = await applyConsolidationPlan(config, stashDir, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, opts.improveProfile);
1384
+ const runDurationMs = Date.now() - startMs;
1385
+ const budgetFraction = opts.runBudgetMs !== undefined && opts.runBudgetMs > 0 ? runDurationMs / opts.runBudgetMs : undefined;
1702
1386
  return {
1703
1387
  schemaVersion: 1,
1704
1388
  ok: true,
@@ -1708,245 +1392,577 @@ export async function akmConsolidate(opts = {}) {
1708
1392
  target: sourceName,
1709
1393
  processed: memories.length,
1710
1394
  merged,
1711
- deleted,
1395
+ // #617: fold the deterministic dedup pre-pass collapses into the reported
1396
+ // deleted count. Each collapse removed exactly one variant file with NO
1397
+ // LLM call before the LLM pass ran on the pruned pool.
1398
+ deleted: deleted + dedupCollapsed,
1712
1399
  promoted,
1713
1400
  contradicted,
1714
- failedChunks: totalChunksFailed,
1715
- totalChunks: chunks.length,
1716
- judgedNoAction,
1717
- skipReasons,
1401
+ mergeFloorViolations,
1402
+ failedChunks: accounting.totalChunksFailed,
1403
+ totalChunks,
1404
+ judgedNoAction: accounting.judgedNoAction,
1405
+ skipReasons: accounting.skipReasons,
1718
1406
  mergedSecondaries,
1719
- failedChunkMemories,
1407
+ failedChunkMemories: accounting.failedChunkMemories,
1720
1408
  warnings,
1721
- durationMs: Date.now() - startMs,
1409
+ durationMs: runDurationMs,
1410
+ perfTelemetry: {
1411
+ dedupPoolSize: perfMs.dedupPoolSize,
1412
+ llmPoolSize,
1413
+ judgedCacheSkipped: perfMs.judgedCacheSkipped,
1414
+ embedMs: embedTelemetry.embedMs,
1415
+ embedCacheHits: embedTelemetry.cacheHits,
1416
+ embedCacheMisses: embedTelemetry.cacheMisses,
1417
+ ...(budgetFraction !== undefined ? { estimatedBudgetFractionUsed: budgetFraction } : {}),
1418
+ },
1722
1419
  };
1723
1420
  }
1724
- // ── Helpers ─────────────────────────────────────────────────────────────────
1725
- // ── LLM-output sanitization ─────────────────────────────────────────────────
1726
- //
1727
- // Three classes of LLM defect have been observed across hundreds of
1728
- // consolidate proposals (see audit notes in this branch):
1729
- //
1730
- // 1. Code-fence leakage: the entire merged asset is wrapped in
1731
- // ```markdown ``` (or ```yaml ```) despite the prompt forbidding
1732
- // fences. The post-processor used to pass this through verbatim, so the
1733
- // first character of the asset content became a backtick rather than
1734
- // `---`, defeating the frontmatter parser.
1735
- // 2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
1736
- // with unbalanced quotes that break the YAML reader. The post-processor
1737
- // historically passed the LLM's raw scalar straight into a manually
1738
- // assembled `description: <raw>` line.
1739
- // 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
1740
- // runs out mid-sentence, leaving things like
1741
- // `description: "Tables in narrow column containers need max-width:100% +"`
1742
- // with no closing context.
1743
- //
1744
- // `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
1745
- // all three at the point where LLM output is consumed.
1746
- /**
1747
- * Attempt to recover a frontmatter block that is missing its closing `---`.
1748
- *
1749
- * Scans lines after the opening `---` for the first blank line or the first
1750
- * line that cannot be a YAML scalar (i.e. not a key-value, indented
1751
- * continuation, comment, or list item). Injects `---` before that line so
1752
- * the normal parser can proceed.
1753
- *
1754
- * Returns the patched string on success, or `null` if the structure is too
1755
- * ambiguous to recover safely (e.g. no opening `---`, or no body content
1756
- * found after the frontmatter key-value lines).
1757
- */
1758
- function recoverMalformedFrontmatter(raw) {
1759
- if (!raw.startsWith("---"))
1760
- return null;
1761
- const lines = raw.split(/\r?\n/);
1762
- // Skip the opening `---` line (index 0).
1763
- let insertAt = -1;
1764
- for (let i = 1; i < lines.length; i++) {
1765
- const line = lines[i];
1766
- // A blank line marks the end of the frontmatter block in many YAML variants.
1767
- if (line.trim() === "") {
1768
- insertAt = i;
1769
- break;
1421
+ /** Execute one `merge` op (behavior-identical to the former inlined branch). */
1422
+ export async function handleMergeOp(op, opIndex, ctx) {
1423
+ const { config, stashDir, target, backupDir, memoryByRef, warnings, pushSkipReason, counts } = ctx;
1424
+ // Accounting helper: emit a per-participant skipReason for failed
1425
+ // merges so primary + every loaded-memory secondary land in the
1426
+ // structured skip histogram. Pre-2026-05-26 only the primary was
1427
+ // counted (1 skipReason per failed merge), leaving N secondaries
1428
+ // unaccounted for in the `processed == actioned + noAction + Σskips`
1429
+ // invariant the source of the 4–11 silent leaks per run.
1430
+ const emitMergeFailureSkips = (reason) => {
1431
+ if (memoryByRef.has(op.primary))
1432
+ pushSkipReason("merge", op.primary, reason);
1433
+ for (const secRef of op.secondaries) {
1434
+ if (memoryByRef.has(secRef))
1435
+ pushSkipReason("merge", secRef, reason);
1770
1436
  }
1771
- // A line that is clearly body content: doesn't look like a YAML key, an
1772
- // indented continuation, a comment, or a sequence item.
1773
- const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
1774
- /^\s+\S/.test(line) || // indented continuation / nested
1775
- /^\s*#/.test(line) || // YAML comment
1776
- /^\s*-\s/.test(line); // sequence item
1777
- if (!isYaml) {
1778
- insertAt = i;
1779
- break;
1437
+ };
1438
+ const primaryEntry = memoryByRef.get(op.primary);
1439
+ if (!primaryEntry) {
1440
+ // This fires when a prior op in the same run consumed this ref as a
1441
+ // secondary and Fix-A pruned it from memoryByRef. It should NOT fire
1442
+ // for hallucinated primaries (those are dropped by mergePlans() before
1443
+ // reaching here). If this counter is non-zero, suspect an intra-run
1444
+ // cross-chunk race, not a filter regression.
1445
+ warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
1446
+ emitMergeFailureSkips("merge_primary_missing");
1447
+ return;
1448
+ }
1449
+ // Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
1450
+ // before this run's own ops), the file may have been deleted by a
1451
+ // concurrent process or an edge case the pre-flight filter missed.
1452
+ if (!fs.existsSync(primaryEntry.filePath)) {
1453
+ warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
1454
+ emitMergeFailureSkips("merge_primary_file_gone");
1455
+ return;
1456
+ }
1457
+ // Phase B: generate merged content
1458
+ const secondaryBodies = [];
1459
+ for (const secRef of op.secondaries) {
1460
+ const secEntry = memoryByRef.get(secRef);
1461
+ if (!secEntry) {
1462
+ warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
1463
+ // No accounting impact: a missing secondary is a phantom ref and
1464
+ // never contributed to any chunk's targetRefs reduction. We still
1465
+ // continue the loop to gather the remaining valid secondaries.
1466
+ continue;
1780
1467
  }
1468
+ secondaryBodies.push(secRef);
1781
1469
  }
1782
- if (insertAt < 0)
1783
- return null;
1784
- const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
1785
- return result;
1786
- }
1787
- /**
1788
- * Outer-fence stripper specific to consolidate. Unlike the shared
1789
- * `stripMarkdownFences` helper (which only handles markdown fences), this
1790
- * variant additionally recognises `yaml` and bare-language fences and refuses
1791
- * to strip an unbalanced fence i.e. a leading ``` with no trailing ``` is
1792
- * treated as a malformed response, not partially sanitized.
1793
- *
1794
- * Returns `null` when only one half of a fence pair is present (caller
1795
- * should reject the response entirely).
1796
- */
1797
- export function stripOuterCodeFence(raw) {
1798
- const trimmed = raw.trim();
1799
- const leading = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n/i);
1800
- const trailing = trimmed.match(/\r?\n```\s*$/);
1801
- if (!leading && !trailing)
1802
- return { content: trimmed, stripped: false };
1803
- if (!leading || !trailing)
1804
- return null; // unbalancedrefuse
1805
- const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
1806
- return { content: inner, stripped: true };
1807
- }
1808
- export function sanitizeMergedContent(raw) {
1809
- // Step 1: Strip outer code fence.
1810
- // Recovery path: if only the leading fence is present, strip it and continue
1811
- // provided the inner content starts with `---`. Trailing-only fences are NOT
1812
- // recovered — a trailing ``` is more likely a body code block than a forgotten
1813
- // wrapper, so recovering would silently corrupt the body.
1814
- let body;
1815
- {
1816
- const fenceResult = stripOuterCodeFence(raw);
1817
- if (fenceResult) {
1818
- body = fenceResult.content;
1470
+ if (secondaryBodies.length === 0) {
1471
+ warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
1472
+ emitMergeFailureSkips("merge_no_valid_secondaries");
1473
+ return;
1474
+ }
1475
+ // Pre-flight hot guard — skip the LLM call entirely if any participant
1476
+ // is hot or unparseable. Without this, mixed chunks still send hot merges
1477
+ // to the planner which proposes them; generateMergedContent() is then
1478
+ // called, produces output without `description`, and the skip is
1479
+ // misattributed to merge_missing_description instead of the real cause.
1480
+ const preflightParticipants = [op.primary, ...op.secondaries];
1481
+ const preflightBlocked = preflightParticipants.flatMap((ref) => {
1482
+ const e = memoryByRef.get(ref);
1483
+ if (!e)
1484
+ return [];
1485
+ const verdict = consolidateGuardStatus(e.filePath);
1486
+ if (verdict === "hot" || verdict === "unparseable")
1487
+ return [{ ref, verdict }];
1488
+ return [];
1489
+ });
1490
+ if (preflightBlocked.length > 0) {
1491
+ const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1492
+ warnings.push(`Merge: refused for ${op.primary}${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
1493
+ emitMergeFailureSkips("merge_participant_blocked");
1494
+ return;
1495
+ }
1496
+ let primaryBody = "";
1497
+ try {
1498
+ primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
1499
+ }
1500
+ catch {
1501
+ warnings.push(`Merge: could not read primary ${op.primary} skipping.`);
1502
+ emitMergeFailureSkips("merge_read_failed");
1503
+ return;
1504
+ }
1505
+ const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef, ctx.improveProfile);
1506
+ if ("error" in mergeResult) {
1507
+ warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
1508
+ emitMergeFailureSkips(mergeResult.error);
1509
+ return;
1510
+ }
1511
+ let mergedContent = mergeResult.content;
1512
+ // Validate frontmatter of merged content — must have a `---` block
1513
+ // with at minimum a `description` field. We parse via the hand-rolled
1514
+ // parser (cheap) AND require non-empty description. This guards against
1515
+ // the historical defect where merged memories were written back with
1516
+ // empty `description` and later polluted the promote path.
1517
+ let parsedMerged;
1518
+ try {
1519
+ parsedMerged = parseFrontmatter(mergedContent);
1520
+ }
1521
+ catch {
1522
+ warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
1523
+ emitMergeFailureSkips("merge_invalid_frontmatter");
1524
+ return;
1525
+ }
1526
+ if (parsedMerged.frontmatter === null) {
1527
+ warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
1528
+ emitMergeFailureSkips("merge_invalid_frontmatter");
1529
+ return;
1530
+ }
1531
+ const mergedDesc = parsedMerged.data.description;
1532
+ if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
1533
+ warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
1534
+ emitMergeFailureSkips("merge_missing_description");
1535
+ return;
1536
+ }
1537
+ const truncReason = detectTruncatedDescription(mergedDesc);
1538
+ if (truncReason) {
1539
+ warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
1540
+ emitMergeFailureSkips("merge_truncated_description");
1541
+ return;
1542
+ }
1543
+ // captureMode:hot guard — refuse the merge if ANY participating memory
1544
+ // (primary or secondary) was user-captured or has unparseable frontmatter
1545
+ // (could have hidden a hot flag). Hot memories are user-explicit and
1546
+ // must not be deleted/overwritten by the consolidate LLM. 14 user
1547
+ // memories were silent-deleted by consolidate before this guard landed;
1548
+ // recovery required copying from .akm/archive/ by hand.
1549
+ const mergeParticipants = [op.primary, ...op.secondaries];
1550
+ const blockedParticipants = mergeParticipants.flatMap((ref) => {
1551
+ const e = memoryByRef.get(ref);
1552
+ if (!e)
1553
+ return [];
1554
+ const verdict = consolidateGuardStatus(e.filePath);
1555
+ if (verdict === "hot" || verdict === "unparseable")
1556
+ return [{ ref, verdict }];
1557
+ return [];
1558
+ });
1559
+ if (blockedParticipants.length > 0) {
1560
+ const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1561
+ warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
1562
+ emitMergeFailureSkips("merge_participant_blocked");
1563
+ return;
1564
+ }
1565
+ // WS-3b: Anti-collapse generation guard (step 8a).
1566
+ // DEFAULT ON since R5 (opt out via antiCollapse.enabled: false). Refuses
1567
+ // to merge two assets both above generation N (default 2) — prevents the
1568
+ // pipeline from building ever-deeper LLM-merged trees that lose the
1569
+ // source fidelity of the original episodes.
1570
+ const antiCollapseConfig = getImproveProcessConfig(config, "consolidate", ctx.improveProfile)?.antiCollapse ?? {};
1571
+ if (antiCollapseConfig.enabled !== false) {
1572
+ const allParticipants = [op.primary, ...op.secondaries];
1573
+ // One read per participant: generation counter, stripped body (for the
1574
+ // information floor), and existing source_refs (for the provenance union).
1575
+ const participantInfo = allParticipants.map((ref) => {
1576
+ const e = memoryByRef.get(ref);
1577
+ if (!e)
1578
+ return { ref, generation: 0, body: "", sourceRefs: [] };
1579
+ try {
1580
+ const raw = fs.readFileSync(e.filePath, "utf8");
1581
+ const parsed = parseFrontmatter(raw);
1582
+ const fm = parsed.data;
1583
+ const sourceRefs = Array.isArray(fm.source_refs) ? fm.source_refs.map(String) : [];
1584
+ return { ref, generation: readAssetGeneration(fm), body: stripFrontmatterBody(raw), sourceRefs };
1585
+ }
1586
+ catch {
1587
+ return { ref, generation: 0, body: "", sourceRefs: [] };
1588
+ }
1589
+ });
1590
+ const sourceGenerations = participantInfo.map((p) => p.generation);
1591
+ const generationCheck = checkGenerationGuard(sourceGenerations, antiCollapseConfig);
1592
+ if (generationCheck.refused) {
1593
+ warnings.push(`Merge: ${generationCheck.reason}`);
1594
+ emitMergeFailureSkips("merge_generation_guard");
1595
+ return;
1819
1596
  }
1820
- else {
1821
- const trimmed = raw.trim();
1822
- const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
1823
- const inner = leadingMatch ? leadingMatch[1].trim() : null;
1824
- if (!inner?.startsWith("---")) {
1825
- return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
1597
+ // WS-3b: Lexical diversity check (step 8b).
1598
+ // Low n-gram diversity ⇒ likely correlated-extraction artifact; raise merge threshold.
1599
+ if (antiCollapseConfig.lexicalDiversityCheck !== false) {
1600
+ const bodies = participantInfo.map((p) => p.body).filter((b) => b.length > 0);
1601
+ const diversityCheck = checkLexicalDiversity(bodies, antiCollapseConfig);
1602
+ if (diversityCheck.lowDiversity) {
1603
+ // Low-diversity cluster: just warn (don't refuse merge since the dedup
1604
+ // path handles exact twins). The warning surfaces in health telemetry.
1605
+ warnings.push(`Merge: cluster around ${op.primary} has low lexical diversity (${diversityCheck.diversity?.toFixed(2) ?? "?"} < 0.30) — likely correlated extraction; merge proceeds but review is recommended.`);
1826
1606
  }
1827
- body = inner;
1607
+ }
1608
+ // Inject generation counter into merged content frontmatter (step 8a).
1609
+ // merged.generation = max(sourceGenerations) + 1. source_refs is the
1610
+ // UNION of participants + everything they already cited (R5 §4.2 —
1611
+ // the old set-if-absent behavior dropped second-generation provenance).
1612
+ const provenanceUnion = [...new Set([...allParticipants, ...participantInfo.flatMap((p) => p.sourceRefs)])];
1613
+ mergedContent = injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceUnion);
1614
+ // R5 §4.2: merge-information floor — ADVISORY in v1. A merge that
1615
+ // shrinks provenance or genericizes below the retention floor is
1616
+ // counted + warned, never refused (promotion path: design doc §7).
1617
+ try {
1618
+ const mergedParsed = parseFrontmatter(mergedContent);
1619
+ const mergedFm = mergedParsed.data;
1620
+ const mergedSourceRefs = Array.isArray(mergedFm.source_refs) ? mergedFm.source_refs.map(String) : [];
1621
+ const floorCheck = checkMergeInformationFloor(mergedParsed.content, mergedSourceRefs, participantInfo, antiCollapseConfig);
1622
+ if (!floorCheck.passed) {
1623
+ counts.mergeFloorViolations++;
1624
+ warnings.push(`Merge: information floor advisory for ${op.primary}: ${floorCheck.reason ?? "unspecified"} — merge proceeds (v1 observe-only).`);
1625
+ }
1626
+ }
1627
+ catch {
1628
+ // Floor measurement is best-effort; never blocks the merge path.
1828
1629
  }
1829
1630
  }
1830
- // Strip <think> blocks (some local models still emit them despite system prompts).
1831
- body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
1832
- // Step 2: Verify frontmatter sentinel.
1833
- // Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
1834
- // merged content:") before the `---`. Accept if `---` appears within 300 chars.
1835
- // Beyond that it's more likely a body section divider, not a frontmatter start.
1836
- if (!body.startsWith("---")) {
1837
- const nlIdx = body.indexOf("\n---");
1838
- if (nlIdx >= 0 && nlIdx < 300) {
1839
- body = body.slice(nlIdx + 1);
1631
+ // Backup secondaries before deleting
1632
+ for (const secRef of op.secondaries) {
1633
+ const secEntry = memoryByRef.get(secRef);
1634
+ if (secEntry && fs.existsSync(secEntry.filePath)) {
1635
+ backupFile(secEntry.filePath, backupDir, secEntry.name);
1840
1636
  }
1841
- else {
1842
- return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
1843
- }
1844
- }
1845
- // Extract frontmatter block.
1846
- // Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
1847
- // by scanning lines after the opening `---` for the first blank line or the
1848
- // first line that isn't a YAML key-value pair, then inject `---` there.
1849
- let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1850
- if (!match) {
1851
- const recovered = recoverMalformedFrontmatter(body);
1852
- if (recovered) {
1853
- match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1854
- }
1855
- if (!match) {
1856
- return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
1857
- }
1858
- }
1859
- // Re-parse via the yaml library so any quote-escaping mistakes either get
1860
- // normalised or surface as a parse error we can reject.
1861
- // Recovery: if the strict yaml library fails, fall back to the lenient
1862
- // hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
1863
- // quirks (unescaped special chars, bare scalars, etc.). If it recovers
1864
- // at least one key, proceed — serializeFrontmatter below will re-serialize
1865
- // cleanly. Only reject if both parsers fail to extract any data.
1866
- let parsedFm;
1637
+ }
1638
+ // Write merged primary
1867
1639
  try {
1868
- parsedFm = yamlParse(match[1]);
1640
+ const parsedPrimary = parseAssetRef(op.primary);
1641
+ await writeAssetToSource(target.source, target.config, parsedPrimary, mergedContent);
1869
1642
  }
1870
1643
  catch (e) {
1871
- const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
1872
- if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
1873
- parsedFm = fallback.data;
1644
+ warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
1645
+ emitMergeFailureSkips("merge_write_failed");
1646
+ return;
1647
+ }
1648
+ // Archive and delete secondaries (P1-B: soft-invalidation)
1649
+ for (const secRef of op.secondaries) {
1650
+ const secEntry = memoryByRef.get(secRef);
1651
+ if (!secEntry)
1652
+ continue;
1653
+ if (fs.existsSync(secEntry.filePath)) {
1654
+ archiveMemory(secEntry.filePath, stashDir, secRef, "merged into primary", opIndex, op.primary, warnings);
1874
1655
  }
1875
- else {
1876
- return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
1656
+ try {
1657
+ const parsedSec = parseAssetRef(secRef);
1658
+ await deleteAssetFromSource(target.source, target.config, parsedSec);
1659
+ markJournalCompleted(stashDir, secRef);
1877
1660
  }
1661
+ catch (e) {
1662
+ warnings.push(`Merge: delete failed for ${secRef}: ${String(e)}`);
1663
+ }
1664
+ }
1665
+ markJournalCompleted(stashDir, op.primary);
1666
+ counts.merged++;
1667
+ // 2026-05-26 accounting-leak fix: `merged` is op-level, but each
1668
+ // successful merge actions `1 + secondaries.length` memories. Without
1669
+ // this counter the accounting invariant breaks by `secondaries.length`
1670
+ // per successful merge (chunk loop excluded all secondaries from
1671
+ // judgedNoAction via targetRefs, but only the primary is credited to
1672
+ // `merged`). Count only loaded-memory secondaries; phantom secondary
1673
+ // refs never affected any chunk's targetRefs in the first place.
1674
+ for (const secRef of op.secondaries) {
1675
+ if (memoryByRef.has(secRef))
1676
+ counts.mergedSecondaries++;
1677
+ }
1678
+ // Prune consumed refs from memoryByRef so later ops in this run cannot
1679
+ // reference an absorbed secondary as a merge primary and proceed with a
1680
+ // stale entry. Primary is rewritten (not deleted), so we only remove
1681
+ // secondaries; the primary ref remains valid under its new content.
1682
+ for (const secRef of op.secondaries) {
1683
+ memoryByRef.delete(secRef);
1878
1684
  }
1879
- if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
1880
- return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
1685
+ }
1686
+ /** Execute one `delete` op (behavior-identical to the former inlined branch). */
1687
+ export async function handleDeleteOp(op, opIndex, ctx) {
1688
+ const { stashDir, target, backupDir, memoryByRef, warnings, pushSkipReason, counts } = ctx;
1689
+ const entry = memoryByRef.get(op.ref);
1690
+ if (!entry) {
1691
+ warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
1692
+ // Phantom ref: not in the batch so not in processed. Pushing to
1693
+ // skipReasons would inflate Σ(skipReasons) without a matching processed
1694
+ // entry, breaking the accounting invariant. Visibility is preserved via
1695
+ // the warnings array above.
1696
+ return;
1697
+ }
1698
+ // captureMode:hot guard — refuse to delete user-captured memories OR
1699
+ // memories whose frontmatter is unparseable (could have hidden the hot
1700
+ // flag). The consolidate LLM was deleting hot-captured user memos as
1701
+ // "redundant" — 14 such deletes were silently archived between
1702
+ // 2026-05-19 and 2026-05-20 before this guard. Hot memories are
1703
+ // user-explicit and may only be deleted by the user.
1704
+ const guard = consolidateGuardStatus(entry.filePath);
1705
+ if (guard === "hot" || guard === "unparseable") {
1706
+ warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
1707
+ pushSkipReason("delete", op.ref, "captureMode_hot_refused");
1708
+ return;
1709
+ }
1710
+ if (fs.existsSync(entry.filePath)) {
1711
+ backupFile(entry.filePath, backupDir, entry.name);
1712
+ // P1-B: soft-invalidation archive before hard delete
1713
+ archiveMemory(entry.filePath, stashDir, op.ref, op.reason, opIndex, undefined, warnings);
1881
1714
  }
1882
- const fm = parsedFm;
1883
- // Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
1884
- // `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
1885
- // these, but small models still do. Replace any such leak with today's ISO
1886
- // date OR drop the field if we can't safely normalise it.
1887
- normalizeUpdatedField(fm);
1888
- // Re-serialise via yaml.stringify to fix any quoting quirks.
1889
- let serialized;
1890
1715
  try {
1891
- serialized = serializeFrontmatter(fm);
1716
+ const parsedRef = parseAssetRef(op.ref);
1717
+ await deleteAssetFromSource(target.source, target.config, parsedRef);
1718
+ markJournalCompleted(stashDir, op.ref);
1719
+ counts.deleted++;
1720
+ // Prune from memoryByRef so later ops in this run cannot reference a
1721
+ // deleted memory as a merge primary or secondary.
1722
+ memoryByRef.delete(op.ref);
1892
1723
  }
1893
1724
  catch (e) {
1894
- return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
1725
+ // Distinguish "file already absent" from genuine failures. A prior run
1726
+ // may have deleted the file but the DB was not yet re-indexed, so the
1727
+ // ref still appeared in memoryByRef. The delete goal is already met.
1728
+ const msg = e instanceof Error ? e.message : String(e);
1729
+ if (msg.includes("not found in source")) {
1730
+ warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
1731
+ pushSkipReason("delete", op.ref, "delete_already_gone");
1732
+ }
1733
+ else {
1734
+ warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
1735
+ pushSkipReason("delete", op.ref, "delete_failed");
1736
+ }
1895
1737
  }
1896
- const cleaned = assembleAssetFromString(serialized, match[2]);
1897
- return { ok: true, result: { content: cleaned, frontmatter: fm } };
1898
1738
  }
1899
- /**
1900
- * Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
1901
- * LLM. The consolidate prompt forbids these, but small models still produce
1902
- * literal `today` / `{today: null}` / `now` values.
1903
- *
1904
- * Rules:
1905
- * - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
1906
- * - A Date object (some YAML parsers materialise dates) is converted to its
1907
- * ISO yyyy-mm-dd form.
1908
- * - A placeholder string ("today", "now", "{today}", "${today}", template
1909
- * variables) is replaced with today's ISO date.
1910
- * - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
1911
- * - `null`, empty string, missing → left alone (no field added; reviewers
1912
- * should not silently gain metadata they didn't write).
1913
- *
1914
- * Exported for unit testing.
1915
- */
1916
- export function normalizeUpdatedField(fm) {
1917
- if (!("updated" in fm))
1739
+ /** Execute one `promote` op (behavior-identical to the former inlined branch). */
1740
+ export async function handlePromoteOp(op, ctx) {
1741
+ const { config, stashDir, sourceRun, target, memoryByRef, warnings, pushSkipReason, promoted, promotedSourceRefs } = ctx;
1742
+ const entry = memoryByRef.get(op.ref);
1743
+ if (!entry) {
1744
+ warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
1745
+ // Phantom ref: not in processed, so no skipReason (same rationale as
1746
+ // delete_ref_missing above).
1918
1747
  return;
1919
- const v = fm.updated;
1920
- if (v === null || v === undefined || v === "")
1748
+ }
1749
+ // Within-run source-ref dedup: skip if this source memory was already
1750
+ // promoted earlier in this run (safety belt — mergePlans already
1751
+ // deduplicates promote ops by source ref via Map, but this guard also
1752
+ // catches any future code paths that bypass mergePlans).
1753
+ if (promotedSourceRefs.has(op.ref)) {
1754
+ warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
1755
+ pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
1756
+ return;
1757
+ }
1758
+ let knowledgeRef = op.knowledgeRef;
1759
+ try {
1760
+ parseAssetRef(knowledgeRef);
1761
+ }
1762
+ catch {
1763
+ const slug = op.knowledgeRef
1764
+ .replace(/^knowledge:/, "")
1765
+ .replace(/[^a-z0-9-]/gi, "-")
1766
+ .toLowerCase();
1767
+ knowledgeRef = `knowledge:${slug}`;
1768
+ warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
1769
+ }
1770
+ // Idempotency: check pending proposals by target ref
1771
+ const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
1772
+ if (existingProposals.some((p) => p.status === "pending")) {
1773
+ warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
1774
+ pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
1921
1775
  return;
1922
- const todayIso = new Date().toISOString().slice(0, 10);
1923
- if (v instanceof Date) {
1924
- fm.updated = v.toISOString().slice(0, 10);
1776
+ }
1777
+ // Idempotency: check if knowledge asset already exists
1778
+ const parsedKnowledgeRef = parseAssetRef(knowledgeRef);
1779
+ const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
1780
+ if (fs.existsSync(destPath)) {
1781
+ warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
1782
+ pushSkipReason("promote", op.ref, "promote_already_exists");
1783
+ return;
1784
+ }
1785
+ let memoryContent = "";
1786
+ try {
1787
+ memoryContent = fs.readFileSync(entry.filePath, "utf8");
1788
+ }
1789
+ catch (e) {
1790
+ warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
1791
+ pushSkipReason("promote", op.ref, "promote_read_failed");
1792
+ return;
1793
+ }
1794
+ // Defensive sanitization: legacy memory files written by older
1795
+ // consolidate runs may still carry outer code fences or broken YAML.
1796
+ // Strip them here so we never propose a polluted asset.
1797
+ const promoteSanitized = sanitizeMergedContent(memoryContent);
1798
+ if (!promoteSanitized.ok) {
1799
+ warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
1800
+ pushSkipReason("promote", op.ref, "promote_sanitization_failed");
1801
+ return;
1802
+ }
1803
+ memoryContent = promoteSanitized.result.content;
1804
+ // SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
1805
+ // frontmatter carries `status: superseded`. Predicate at module top
1806
+ // (`hasSupersededStatus`) so tests can exercise it directly.
1807
+ if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
1808
+ warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
1809
+ pushSkipReason("promote", op.ref, "promote_superseded");
1810
+ return;
1811
+ }
1812
+ // Parse the source memory up-front so the body/frontmatter checks below
1813
+ // share the same parsed view.
1814
+ const parsedMemory = parseFrontmatter(memoryContent);
1815
+ // Reject sources whose body is too small to make useful knowledge.
1816
+ // Observed failure: memory files whose body is literally a tags string
1817
+ // ("discord,notification,send-notification") get promoted to knowledge
1818
+ // proposals that no reviewer would accept. Threshold is conservative —
1819
+ // 100 chars catches single-line tag dumps without rejecting genuinely
1820
+ // terse but valid notes.
1821
+ const PROMOTE_BODY_MIN_CHARS = 100;
1822
+ const sourceBody = parsedMemory.content.trim();
1823
+ if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
1824
+ warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
1825
+ pushSkipReason("promote", op.ref, "promote_source_too_small");
1826
+ return;
1827
+ }
1828
+ // Cross-run + within-run content dedup: if an identical body already
1829
+ // exists in ANY pending consolidate proposal (regardless of target ref),
1830
+ // skip. This prevents duplicate proposals when:
1831
+ // (a) Multiple source memories have identical bodies but differ only
1832
+ // in noise frontmatter (`inferenceProcessed: true` twin alongside
1833
+ // the original; differing `updated:` timestamps; etc.) — the body
1834
+ // is the load-bearing content, so dedup must hash on body only.
1835
+ // (b) A prior run created a proposal for the same body under a
1836
+ // different knowledgeRef slug.
1837
+ // Use cacheHash (case-preserving stripped body) to match the canonical
1838
+ // hash domain used by the body-embedding cache and pending-proposal set.
1839
+ const bodyHash = cacheHash(sourceBody);
1840
+ const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1841
+ const contentDupProposal = allPendingConsolidateProposals.find((p) => {
1842
+ return cacheHash(p.payload.content) === bodyHash;
1843
+ });
1844
+ if (contentDupProposal) {
1845
+ warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
1846
+ pushSkipReason("promote", op.ref, "dedup_pending_proposal");
1925
1847
  return;
1926
1848
  }
1927
- if (typeof v === "string") {
1928
- const trimmed = v.trim().toLowerCase();
1929
- if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
1930
- return; // already a real date
1931
- if (trimmed === "today" ||
1932
- trimmed === "now" ||
1933
- trimmed === "{today}" ||
1934
- // biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
1935
- trimmed === "${today}" ||
1936
- trimmed === "{{today}}" ||
1937
- /^\{?\s*today\s*\}?$/.test(trimmed)) {
1938
- fm.updated = todayIso;
1849
+ try {
1850
+ // Use LLM-provided description; fall back to memory's own description
1851
+ // (post-sanitization frontmatter is authoritative).
1852
+ const description = (typeof op.description === "string" && op.description.trim()
1853
+ ? op.description.trim()
1854
+ : parsedMemory.data?.description?.trim()) ?? "";
1855
+ // Validate the resolved frontmatter before emitting a proposal.
1856
+ // Required field: non-empty description. Reject obvious truncation
1857
+ // markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
1858
+ // so the queue never sees half-formed metadata that the reviewer
1859
+ // would only reject.
1860
+ const fmCheck = validateProposalFrontmatter({ description });
1861
+ if (!fmCheck.ok) {
1862
+ warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
1863
+ pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
1864
+ return;
1865
+ }
1866
+ // Merge `description` INTO the body's YAML frontmatter so it lands in
1867
+ // the on-disk asset when the proposal is accepted. The descriptionQuality
1868
+ // validator parses `payload.content` body (not the envelope
1869
+ // `payload.frontmatter`), and a memory's native frontmatter has
1870
+ // `captureMode`/`beliefState`/etc. but never `description` — without
1871
+ // this merge, 60+ pending proposals were blocked at accept-time with
1872
+ // MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
1873
+ // (The body-frontmatter assumption baked into the 2026-05-20 comment
1874
+ // below was wrong: body fm and envelope fm only converge when the
1875
+ // writer explicitly merges them, which it now does.)
1876
+ const mergedBodyFm = {
1877
+ ...(parsedMemory.data ?? {}),
1878
+ description,
1879
+ };
1880
+ const serializedMergedFm = serializeFrontmatter(mergedBodyFm);
1881
+ const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
1882
+ // Pre-emit dedup against pending consolidate proposals from the
1883
+ // same improve run (slug-variant match). The cross-run content-hash
1884
+ // dedup inside `mergePlans` handles duplicates against existing
1885
+ // stash assets — see commit history for the deletion of the
1886
+ // unbounded embedding + cross-type slug branches.
1887
+ const dedup = await checkPreEmitDedup({
1888
+ candidateRef: knowledgeRef,
1889
+ candidateText: `${description}. ${memoryContent}`,
1890
+ stashDir,
1891
+ config,
1892
+ });
1893
+ if (dedup.duplicate) {
1894
+ warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
1895
+ pushSkipReason("promote", op.ref, "promote_dedup_window");
1939
1896
  return;
1940
1897
  }
1941
- // Unknown string format — leave alone so it's visible in the diff.
1898
+ const proposalResult = createProposal(stashDir, {
1899
+ ref: knowledgeRef,
1900
+ source: "consolidate",
1901
+ sourceRun,
1902
+ payload: {
1903
+ content: proposalContent,
1904
+ frontmatter: { description },
1905
+ },
1906
+ ...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
1907
+ });
1908
+ if (isProposalSkipped(proposalResult)) {
1909
+ warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
1910
+ pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
1911
+ }
1912
+ else {
1913
+ promoted.push(proposalResult.id);
1914
+ promotedSourceRefs.add(op.ref);
1915
+ markJournalCompleted(stashDir, op.ref);
1916
+ }
1917
+ }
1918
+ catch (e) {
1919
+ warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
1920
+ pushSkipReason("promote", op.ref, "promote_create_failed");
1921
+ }
1922
+ }
1923
+ /** Execute one `contradict` op (behavior-identical to the former inlined branch). */
1924
+ export async function handleContradictOp(op, ctx) {
1925
+ const { stashDir, memoryByRef, warnings, pushSkipReason, counts } = ctx;
1926
+ // Confidence gate: surface-level topic overlap causes false positives
1927
+ // (investigation 2026-06-18). Require ≥0.92 confidence before writing
1928
+ // contradiction edges. Missing confidence field defaults to 1.0 for
1929
+ // backward compatibility with responses that predate this field.
1930
+ const opConfidence = typeof op.confidence === "number" ? op.confidence : 1.0;
1931
+ if (opConfidence < 0.92) {
1932
+ warnings.push(`Contradict: confidence ${opConfidence.toFixed(2)} below 0.92 threshold for ${op.ref} <-> ${op.contradictedByRef} — skipping.`);
1933
+ pushSkipReason("contradict", op.ref, "contradict_low_confidence");
1934
+ return;
1935
+ }
1936
+ // C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
1937
+ // (the SCC resolver in memory-improve.ts) has edges to work on.
1938
+ // Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
1939
+ const entry = memoryByRef.get(op.ref);
1940
+ const contradictorEntry = memoryByRef.get(op.contradictedByRef);
1941
+ if (!entry) {
1942
+ warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
1943
+ // Phantom ref: not in processed, so no skipReason (same rationale as
1944
+ // delete_ref_missing).
1942
1945
  return;
1943
1946
  }
1944
- if (typeof v === "object") {
1945
- // Maps like `{today: null}`, `{now: null}`clearly a template leak.
1946
- fm.updated = todayIso;
1947
+ if (!contradictorEntry) {
1948
+ warnings.push(`Contradict: ${op.contradictedByRef} not foundskipping.`);
1949
+ // op.ref IS in the batch (entry found above) so the skipReason is
1950
+ // correctly charged against a real processed memory.
1951
+ pushSkipReason("contradict", op.ref, "contradict_target_missing");
1947
1952
  return;
1948
1953
  }
1954
+ try {
1955
+ // Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
1956
+ writeContradictEdge(entry.filePath, op.contradictedByRef);
1957
+ counts.contradicted++;
1958
+ markJournalCompleted(stashDir, op.ref);
1959
+ }
1960
+ catch (e) {
1961
+ warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
1962
+ pushSkipReason("contradict", op.ref, "contradict_write_failed");
1963
+ }
1949
1964
  }
1965
+ // ── Helpers ─────────────────────────────────────────────────────────────────
1950
1966
  /**
1951
1967
  * Normalise a knowledge slug for variant-aware deduplication. Collapses:
1952
1968
  * - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
@@ -2019,11 +2035,13 @@ async function checkPreEmitDedup(opts) {
2019
2035
  * doesn't match the pattern (assumed to already be an ISO timestamp).
2020
2036
  */
2021
2037
  function parseSinceToIso(since) {
2022
- const m = since.match(/^(\d+)(m|h|d)$/);
2023
- if (!m)
2038
+ // Canonical CLI unit grammar: `m` = minutes, `M` = months (see core/time.ts
2039
+ // DURATION_UNITS). Non-matching input is returned unchanged (assumed to
2040
+ // already be an ISO timestamp).
2041
+ const ms = parseDuration(since, DURATION_UNITS);
2042
+ if (ms === null)
2024
2043
  return since;
2025
- const multiplier = { m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2]];
2026
- return new Date(Date.now() - parseInt(m[1], 10) * multiplier).toISOString();
2044
+ return new Date(Date.now() - ms).toISOString();
2027
2045
  }
2028
2046
  export function narrowToIncrementalCandidates(memories, since, warnings, neighborsPerChanged = 5) {
2029
2047
  const sinceIso = parseSinceToIso(since);
@@ -2129,7 +2147,7 @@ function loadMemoriesForSource(source, stashDir, warnings) {
2129
2147
  }
2130
2148
  return memories;
2131
2149
  }
2132
- async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef) {
2150
+ async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef, activeProfile) {
2133
2151
  // Only handle single-secondary merges per design (one call per merge op)
2134
2152
  const secRef = secondaryRefs[0];
2135
2153
  const secEntry = memoryByRef.get(secRef);
@@ -2173,7 +2191,7 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
2173
2191
  .join("\n");
2174
2192
  // Use the same per-process profile resolution as the chunk-plan call above
2175
2193
  // so the merge generation step doesn't silently revert to the default LLM.
2176
- const llmConfig = resolveConsolidateLlmConfig(config);
2194
+ const llmConfig = resolveConsolidateLlmConfig(config, activeProfile);
2177
2195
  const result = await tryLlmFeature("memory_consolidation", config, async () => {
2178
2196
  if (!llmConfig)
2179
2197
  return { ok: false, error: "No LLM configured for consolidation" };
@@ -2251,8 +2269,8 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
2251
2269
  }
2252
2270
  normalizeUpdatedField(repairedFmData);
2253
2271
  const repairedYaml = serializeFrontmatter(repairedFmData);
2254
- const bodyPart = mergedFm.content ?? "";
2255
- return { content: `---\n${repairedYaml}\n---\n${bodyPart}` };
2272
+ const bodyPart = typeof mergedFm.content === "string" ? mergedFm.content : "";
2273
+ return { content: assembleAssetFromString(repairedYaml, bodyPart) };
2256
2274
  }
2257
2275
  }
2258
2276
  catch {