akm-cli 0.9.0-beta.9 → 0.9.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (381) hide show
  1. package/CHANGELOG.md +715 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/help/help-improve.md +9 -6
  6. package/dist/assets/hints/cli-hints-full.md +6 -5
  7. package/dist/assets/profiles/default.json +9 -4
  8. package/dist/assets/profiles/frequent.json +1 -1
  9. package/dist/assets/profiles/memory-focus.json +1 -1
  10. package/dist/assets/profiles/proactive-maintenance.json +25 -0
  11. package/dist/assets/profiles/quick.json +1 -1
  12. package/dist/assets/profiles/recombine-only.json +21 -0
  13. package/dist/assets/profiles/reflect-distill.json +30 -0
  14. package/dist/assets/profiles/synthesize.json +15 -0
  15. package/dist/assets/profiles/thorough.json +1 -1
  16. package/dist/assets/prompts/consolidate-system.md +23 -0
  17. package/dist/assets/prompts/contradiction-judge.md +33 -0
  18. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  19. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  20. package/dist/assets/prompts/extract-session.md +11 -3
  21. package/dist/assets/prompts/graph-extract-system.md +1 -0
  22. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  23. package/dist/assets/prompts/memory-infer-system.md +1 -0
  24. package/dist/assets/prompts/memory-infer-user.md +5 -0
  25. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  26. package/dist/assets/prompts/procedural-system.md +44 -0
  27. package/dist/assets/prompts/recombine-system.md +40 -0
  28. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  29. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  30. package/dist/assets/prompts/workflow-unit-preamble.md +26 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  34. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  35. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  36. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  37. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  38. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  39. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  40. package/dist/assets/templates/html/health.html +281 -111
  41. package/dist/assets/wiki/ingest-workflow-template.md +45 -16
  42. package/dist/assets/wiki/schema-template.md +4 -4
  43. package/dist/cli/clack.js +56 -0
  44. package/dist/cli/config-migrate.js +7 -1
  45. package/dist/cli/confirm.js +1 -1
  46. package/dist/cli/parse-args.js +46 -1
  47. package/dist/cli/shared.js +28 -0
  48. package/dist/cli.js +25 -21
  49. package/dist/commands/agent/agent-dispatch.js +3 -2
  50. package/dist/commands/agent/agent-support.js +0 -7
  51. package/dist/commands/agent/contribute-cli.js +26 -7
  52. package/dist/commands/config-cli.js +26 -13
  53. package/dist/commands/env/child-env.js +47 -0
  54. package/dist/commands/env/env-binding.js +95 -0
  55. package/dist/commands/env/env-cli.js +228 -292
  56. package/dist/commands/env/env.js +14 -67
  57. package/dist/commands/env/secret-cli.js +140 -138
  58. package/dist/commands/feedback-cli.js +156 -155
  59. package/dist/commands/graph/graph-cli.js +5 -13
  60. package/dist/commands/graph/graph.js +3 -3
  61. package/dist/commands/health/advisories.js +151 -0
  62. package/dist/commands/health/checks.js +103 -16
  63. package/dist/commands/health/html-report.js +447 -81
  64. package/dist/commands/health/improve-metrics.js +771 -0
  65. package/dist/commands/health/llm-usage.js +65 -0
  66. package/dist/commands/health/md-report.js +103 -0
  67. package/dist/commands/health/metrics.js +278 -0
  68. package/dist/commands/health/stash-exposure.js +46 -0
  69. package/dist/commands/health/surfaces.js +216 -0
  70. package/dist/commands/health/task-runs.js +135 -0
  71. package/dist/commands/health/types.js +26 -0
  72. package/dist/commands/health/windows.js +195 -0
  73. package/dist/commands/health.js +91 -1091
  74. package/dist/commands/improve/anti-collapse.js +170 -0
  75. package/dist/commands/improve/calibration.js +161 -0
  76. package/dist/commands/improve/collapse-detector.js +421 -0
  77. package/dist/commands/improve/consolidate/chunking.js +141 -0
  78. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  79. package/dist/commands/improve/consolidate/merge.js +145 -0
  80. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  81. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  82. package/dist/commands/improve/consolidate.js +1295 -1277
  83. package/dist/commands/improve/dedup.js +482 -0
  84. package/dist/commands/improve/distill/content-repair.js +202 -0
  85. package/dist/commands/improve/distill/promote-memory.js +229 -0
  86. package/dist/commands/improve/distill/quality-gate.js +236 -0
  87. package/dist/commands/improve/distill-guards.js +127 -0
  88. package/dist/commands/improve/distill-promotion-policy.js +826 -167
  89. package/dist/commands/improve/distill.js +228 -605
  90. package/dist/commands/improve/eligibility.js +434 -0
  91. package/dist/commands/improve/encoding-salience.js +205 -0
  92. package/dist/commands/improve/extract-cli.js +179 -59
  93. package/dist/commands/improve/extract-prompt.js +54 -3
  94. package/dist/commands/improve/extract-watch.js +140 -0
  95. package/dist/commands/improve/extract.js +409 -43
  96. package/dist/commands/improve/feedback-valence.js +54 -0
  97. package/dist/commands/improve/hot-probation.js +45 -0
  98. package/dist/commands/improve/improve-auto-accept.js +157 -10
  99. package/dist/commands/improve/improve-cli.js +115 -73
  100. package/dist/commands/improve/improve-profiles.js +28 -8
  101. package/dist/commands/improve/improve-result-file.js +15 -25
  102. package/dist/commands/improve/improve-session.js +58 -0
  103. package/dist/commands/improve/improve.js +485 -2764
  104. package/dist/commands/improve/locks.js +154 -0
  105. package/dist/commands/improve/loop-stages.js +1100 -0
  106. package/dist/commands/improve/memory/memory-belief.js +14 -15
  107. package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
  108. package/dist/commands/improve/memory/memory-improve.js +27 -27
  109. package/dist/commands/improve/outcome-loop.js +270 -0
  110. package/dist/commands/improve/preparation.js +2002 -0
  111. package/dist/commands/improve/proactive-maintenance.js +37 -35
  112. package/dist/commands/improve/procedural.js +398 -0
  113. package/dist/commands/improve/recombine.js +818 -0
  114. package/dist/commands/improve/reflect-noise.js +0 -0
  115. package/dist/commands/improve/reflect.js +206 -45
  116. package/dist/commands/improve/salience.js +455 -0
  117. package/dist/commands/improve/schema-similarity-gate.js +168 -0
  118. package/dist/commands/improve/shared.js +51 -0
  119. package/dist/commands/improve/triage.js +93 -0
  120. package/dist/commands/lint/agent-linter.js +19 -24
  121. package/dist/commands/lint/base-linter.js +173 -60
  122. package/dist/commands/lint/command-linter.js +19 -24
  123. package/dist/commands/lint/env-key-rules.js +38 -1
  124. package/dist/commands/lint/fact-linter.js +39 -0
  125. package/dist/commands/lint/index.js +31 -13
  126. package/dist/commands/lint/memory-linter.js +1 -1
  127. package/dist/commands/lint/registry.js +7 -2
  128. package/dist/commands/lint/task-linter.js +3 -3
  129. package/dist/commands/lint/workflow-linter.js +26 -1
  130. package/dist/commands/observability-cli.js +4 -4
  131. package/dist/commands/proposal/drain-policies.js +13 -4
  132. package/dist/commands/proposal/drain.js +45 -51
  133. package/dist/commands/proposal/legacy-import.js +115 -0
  134. package/dist/commands/proposal/proposal-cli.js +24 -34
  135. package/dist/commands/proposal/proposal.js +2 -1
  136. package/dist/commands/proposal/propose.js +8 -3
  137. package/dist/commands/proposal/repository.js +829 -0
  138. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  139. package/dist/commands/proposal/validators/proposals.js +93 -895
  140. package/dist/commands/read/curate.js +410 -111
  141. package/dist/commands/read/knowledge.js +10 -3
  142. package/dist/commands/read/remember-cli.js +133 -138
  143. package/dist/commands/read/search-cli.js +15 -8
  144. package/dist/commands/read/search.js +22 -11
  145. package/dist/commands/read/show.js +106 -14
  146. package/dist/commands/registry-cli.js +76 -87
  147. package/dist/commands/remember.js +11 -12
  148. package/dist/commands/sources/add-cli.js +91 -95
  149. package/dist/commands/sources/history.js +1 -1
  150. package/dist/commands/sources/init.js +66 -18
  151. package/dist/commands/sources/installed-stashes.js +11 -3
  152. package/dist/commands/sources/migration-help.js +7 -4
  153. package/dist/commands/sources/schema-repair.js +44 -46
  154. package/dist/commands/sources/self-update.js +2 -2
  155. package/dist/commands/sources/source-add.js +7 -3
  156. package/dist/commands/sources/sources-cli.js +3 -3
  157. package/dist/commands/sources/stash-cli.js +19 -39
  158. package/dist/commands/sources/stash-skeleton.js +57 -8
  159. package/dist/commands/tasks/default-tasks.js +15 -2
  160. package/dist/commands/tasks/tasks-cli.js +20 -29
  161. package/dist/commands/tasks/tasks.js +39 -11
  162. package/dist/commands/wiki-cli.js +23 -38
  163. package/dist/commands/workflow-cli.js +291 -13
  164. package/dist/core/asset/asset-registry.js +3 -1
  165. package/dist/core/asset/asset-spec.js +79 -5
  166. package/dist/core/asset/frontmatter.js +188 -167
  167. package/dist/core/asset/markdown.js +8 -0
  168. package/dist/core/authoring-rules.js +92 -0
  169. package/dist/core/common.js +4 -23
  170. package/dist/core/concurrent.js +10 -1
  171. package/dist/core/config/config-io.js +10 -1
  172. package/dist/core/config/config-migration.js +18 -40
  173. package/dist/core/config/config-schema.js +403 -62
  174. package/dist/core/config/config-types.js +3 -3
  175. package/dist/core/config/config.js +67 -22
  176. package/dist/core/deep-merge.js +38 -0
  177. package/dist/core/errors.js +1 -0
  178. package/dist/core/eval/rank-metrics.js +113 -0
  179. package/dist/core/events.js +4 -7
  180. package/dist/core/improve-types.js +47 -8
  181. package/dist/core/json-schema.js +142 -0
  182. package/dist/core/logs-db.js +14 -75
  183. package/dist/core/parse.js +36 -16
  184. package/dist/core/paths.js +18 -18
  185. package/dist/core/standards/resolve-standards-context.js +87 -0
  186. package/dist/core/standards/resolve-stash-standards.js +99 -0
  187. package/dist/core/standards/resolve-type-conventions.js +66 -0
  188. package/dist/core/state/migrations.js +770 -0
  189. package/dist/core/state-db.js +132 -1126
  190. package/dist/core/structured.js +69 -0
  191. package/dist/core/time.js +53 -0
  192. package/dist/core/warn.js +21 -0
  193. package/dist/core/write-source.js +37 -0
  194. package/dist/indexer/db/db.js +261 -770
  195. package/dist/indexer/db/entry-mapper.js +41 -0
  196. package/dist/indexer/db/graph-db.js +129 -86
  197. package/dist/indexer/db/llm-cache.js +2 -2
  198. package/dist/indexer/db/schema.js +516 -0
  199. package/dist/indexer/ensure-index.js +36 -92
  200. package/dist/indexer/feedback/utility-policy.js +75 -0
  201. package/dist/indexer/graph/graph-boost.js +51 -41
  202. package/dist/indexer/graph/graph-extraction.js +207 -4
  203. package/dist/indexer/index-writer-lock.js +18 -11
  204. package/dist/indexer/index-written-assets.js +105 -0
  205. package/dist/indexer/indexer.js +182 -204
  206. package/dist/indexer/passes/dir-staleness.js +114 -0
  207. package/dist/indexer/passes/memory-inference.js +13 -5
  208. package/dist/indexer/passes/metadata.js +20 -0
  209. package/dist/indexer/read-preflight.js +23 -0
  210. package/dist/indexer/search/db-search.js +89 -13
  211. package/dist/indexer/search/fts-query.js +51 -0
  212. package/dist/indexer/search/ranking-contributors.js +95 -9
  213. package/dist/indexer/search/ranking.js +79 -3
  214. package/dist/indexer/search/search-fields.js +6 -0
  215. package/dist/indexer/search/search-source.js +32 -21
  216. package/dist/indexer/search/semantic-status.js +4 -0
  217. package/dist/indexer/walk/matchers.js +48 -0
  218. package/dist/indexer/walk/walker.js +21 -13
  219. package/dist/integrations/agent/builders.js +41 -13
  220. package/dist/integrations/agent/config.js +20 -59
  221. package/dist/integrations/agent/detect.js +9 -0
  222. package/dist/integrations/agent/index.js +3 -19
  223. package/dist/integrations/agent/model-aliases.js +16 -2
  224. package/dist/integrations/agent/profiles.js +79 -6
  225. package/dist/integrations/agent/prompts.js +75 -9
  226. package/dist/integrations/agent/runner-dispatch.js +83 -0
  227. package/dist/integrations/agent/runner.js +13 -9
  228. package/dist/integrations/agent/spawn.js +206 -81
  229. package/dist/integrations/harnesses/aider/agent-builder.js +113 -0
  230. package/dist/integrations/harnesses/aider/index.js +58 -0
  231. package/dist/integrations/harnesses/aider/result-extractor.js +53 -0
  232. package/dist/integrations/harnesses/amazonq/agent-builder.js +153 -0
  233. package/dist/integrations/harnesses/amazonq/index.js +59 -0
  234. package/dist/integrations/harnesses/amazonq/result-extractor.js +48 -0
  235. package/dist/integrations/harnesses/claude/agent-builder.js +46 -7
  236. package/dist/integrations/harnesses/claude/index.js +27 -23
  237. package/dist/integrations/harnesses/claude/result-extractor.js +52 -0
  238. package/dist/integrations/harnesses/claude/session-log.js +10 -0
  239. package/dist/integrations/harnesses/codex/agent-builder.js +137 -0
  240. package/dist/integrations/harnesses/codex/index.js +63 -0
  241. package/dist/integrations/harnesses/codex/result-extractor.js +73 -0
  242. package/dist/integrations/harnesses/copilot/agent-builder.js +122 -0
  243. package/dist/integrations/harnesses/copilot/index.js +60 -0
  244. package/dist/integrations/harnesses/copilot/result-extractor.js +151 -0
  245. package/dist/integrations/harnesses/gemini/agent-builder.js +121 -0
  246. package/dist/integrations/harnesses/gemini/index.js +60 -0
  247. package/dist/integrations/harnesses/gemini/result-extractor.js +121 -0
  248. package/dist/integrations/harnesses/index.js +28 -7
  249. package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
  250. package/dist/integrations/harnesses/opencode/index.js +17 -16
  251. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  252. package/dist/integrations/harnesses/opencode-sdk/harness.js +65 -0
  253. package/dist/integrations/harnesses/opencode-sdk/index.js +10 -34
  254. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +642 -71
  255. package/dist/integrations/harnesses/openhands/agent-builder.js +126 -0
  256. package/dist/integrations/harnesses/openhands/index.js +58 -0
  257. package/dist/integrations/harnesses/openhands/result-extractor.js +103 -0
  258. package/dist/integrations/harnesses/pi/agent-builder.js +104 -0
  259. package/dist/integrations/harnesses/pi/index.js +58 -0
  260. package/dist/integrations/harnesses/pi/result-extractor.js +135 -0
  261. package/dist/integrations/harnesses/types.js +8 -0
  262. package/dist/integrations/session-logs/index.js +40 -11
  263. package/dist/llm/call-ai.js +2 -2
  264. package/dist/llm/client.js +34 -11
  265. package/dist/llm/embedder.js +67 -4
  266. package/dist/llm/embedders/cache.js +3 -1
  267. package/dist/llm/embedders/deterministic.js +66 -0
  268. package/dist/llm/embedders/local.js +73 -3
  269. package/dist/llm/feature-gate.js +16 -15
  270. package/dist/llm/graph-extract.js +67 -44
  271. package/dist/llm/memory-infer-impl.js +138 -0
  272. package/dist/llm/memory-infer.js +1 -127
  273. package/dist/llm/metadata-enhance.js +44 -31
  274. package/dist/llm/structured-call.js +49 -0
  275. package/dist/migrate-storage-node.mjs +8 -0
  276. package/dist/output/context.js +5 -5
  277. package/dist/output/renderers.js +87 -15
  278. package/dist/output/shapes/curate.js +14 -2
  279. package/dist/output/shapes/helpers.js +0 -3
  280. package/dist/output/shapes/passthrough.js +6 -1
  281. package/dist/output/text/helpers.js +241 -2
  282. package/dist/output/text/workflow.js +4 -1
  283. package/dist/registry/providers/skills-sh.js +21 -147
  284. package/dist/registry/providers/static-index.js +15 -157
  285. package/dist/registry/resolve.js +27 -9
  286. package/dist/runtime.js +25 -1
  287. package/dist/schemas/akm-config.json +14225 -0
  288. package/dist/schemas/akm-workflow.json +328 -0
  289. package/dist/scripts/migrate-storage.js +2743 -8390
  290. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +1652 -607
  291. package/dist/setup/detect.js +9 -0
  292. package/dist/setup/legacy-config.js +106 -0
  293. package/dist/setup/prompt.js +57 -0
  294. package/dist/setup/providers.js +14 -0
  295. package/dist/setup/registry-stash-loader.js +12 -0
  296. package/dist/setup/semantic-assets.js +124 -0
  297. package/dist/setup/setup.js +52 -1614
  298. package/dist/setup/steps/connection.js +734 -0
  299. package/dist/setup/steps/output.js +31 -0
  300. package/dist/setup/steps/platforms.js +124 -0
  301. package/dist/setup/steps/semantic.js +27 -0
  302. package/dist/setup/steps/sources.js +222 -0
  303. package/dist/setup/steps/stashdir.js +42 -0
  304. package/dist/setup/steps/tasks.js +152 -0
  305. package/dist/sources/include.js +6 -2
  306. package/dist/sources/providers/filesystem.js +0 -1
  307. package/dist/sources/providers/git-install.js +210 -0
  308. package/dist/sources/providers/git-provider.js +234 -0
  309. package/dist/sources/providers/git-stash.js +248 -0
  310. package/dist/sources/providers/git.js +10 -661
  311. package/dist/sources/providers/npm.js +2 -6
  312. package/dist/sources/providers/provider-utils.js +13 -7
  313. package/dist/sources/providers/sync-from-ref.js +9 -1
  314. package/dist/sources/providers/website.js +9 -5
  315. package/dist/sources/website-ingest.js +187 -29
  316. package/dist/sources/wiki-fetchers/registry.js +53 -0
  317. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  318. package/dist/storage/database.js +45 -10
  319. package/dist/storage/managed-db.js +82 -0
  320. package/dist/storage/repositories/canaries-repository.js +107 -0
  321. package/dist/storage/repositories/consolidation-repository.js +38 -0
  322. package/dist/storage/repositories/embeddings-repository.js +72 -0
  323. package/dist/storage/repositories/events-repository.js +187 -0
  324. package/dist/storage/repositories/extract-sessions-repository.js +96 -0
  325. package/dist/storage/repositories/improve-runs-repository.js +146 -0
  326. package/dist/storage/repositories/index-db.js +14 -8
  327. package/dist/storage/repositories/proposals-repository.js +220 -0
  328. package/dist/storage/repositories/recombine-repository.js +213 -0
  329. package/dist/storage/repositories/registry-cache.js +93 -0
  330. package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
  331. package/dist/storage/repositories/task-history-repository.js +93 -0
  332. package/dist/storage/repositories/workflow-runs-repository.js +189 -1
  333. package/dist/storage/sqlite-pragmas.js +146 -0
  334. package/dist/tasks/backends/cron.js +1 -1
  335. package/dist/tasks/backends/index.js +9 -0
  336. package/dist/tasks/backends/launchd.js +1 -1
  337. package/dist/tasks/backends/schtasks.js +1 -1
  338. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  339. package/dist/tasks/runner.js +15 -13
  340. package/dist/text-import-hook.mjs +1 -1
  341. package/dist/wiki/wiki.js +52 -11
  342. package/dist/workflows/authoring/authoring.js +123 -10
  343. package/dist/workflows/authoring/workflow-program-template.yaml +31 -0
  344. package/dist/workflows/cli.js +5 -0
  345. package/dist/workflows/db.js +138 -4
  346. package/dist/workflows/exec/brief.js +484 -0
  347. package/dist/workflows/exec/native-executor.js +975 -0
  348. package/dist/workflows/exec/param-secrets.js +115 -0
  349. package/dist/workflows/exec/report.js +1295 -0
  350. package/dist/workflows/exec/run-workflow.js +596 -0
  351. package/dist/workflows/exec/scheduler.js +100 -0
  352. package/dist/workflows/exec/step-work.js +1156 -0
  353. package/dist/workflows/exec/unit-writer.js +23 -0
  354. package/dist/workflows/exec/watch.js +116 -0
  355. package/dist/workflows/exec/worktree.js +171 -0
  356. package/dist/workflows/ir/compile.js +388 -0
  357. package/dist/workflows/ir/params.js +54 -0
  358. package/dist/workflows/ir/plan-hash.js +33 -0
  359. package/dist/workflows/ir/schema.js +4 -0
  360. package/dist/workflows/parser.js +3 -1
  361. package/dist/workflows/program/expressions.js +369 -0
  362. package/dist/workflows/program/parser.js +760 -0
  363. package/dist/workflows/program/project.js +105 -0
  364. package/dist/workflows/program/schema.js +54 -0
  365. package/dist/workflows/renderer.js +82 -5
  366. package/dist/workflows/runtime/agent-identity.js +59 -14
  367. package/dist/workflows/runtime/runs.js +248 -153
  368. package/dist/workflows/runtime/unit-checkin.js +45 -0
  369. package/dist/workflows/runtime/workflow-asset-loader.js +188 -0
  370. package/dist/workflows/validate-summary.js +26 -10
  371. package/dist/workflows/validator.js +1 -1
  372. package/docs/README.md +69 -18
  373. package/docs/data-and-telemetry.md +7 -5
  374. package/docs/migration/release-notes/0.7.0.md +1 -1
  375. package/docs/migration/release-notes/0.9.0-beta.60.md +19 -0
  376. package/docs/migration/release-notes/0.9.0.md +39 -0
  377. package/package.json +10 -10
  378. package/dist/assets/tasks/core/update-stashes.yml +0 -4
  379. package/dist/commands/db-cli.js +0 -23
  380. package/dist/indexer/db/db-backup.js +0 -376
  381. package/dist/indexer/passes/staleness-detect.js +0 -488
@@ -4,188 +4,46 @@
4
4
  import fs from "node:fs";
5
5
  import path from "node:path";
6
6
  import { assertNever } from "../../core/assert.js";
7
- import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
8
- import { parseFrontmatter } from "../../core/asset/frontmatter.js";
9
- import { daysToMs, isAssetType } from "../../core/common.js";
10
- import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
11
- import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
7
+ import { daysToMs } from "../../core/common.js";
8
+ import { loadConfig } from "../../core/config/config.js";
9
+ import { rethrowIfTestIsolationError } from "../../core/errors.js";
12
10
  import { appendEvent, readEvents } from "../../core/events.js";
13
- import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
14
- import { classifyImproveAction } from "../../core/improve-types.js";
15
- import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
11
+ import { classifyImproveAction, foldDistillSkipped } from "../../core/improve-types.js";
16
12
  import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
17
- import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
13
+ import { openStateDatabase } from "../../core/state-db.js";
18
14
  import { info, warn } from "../../core/warn.js";
19
- import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
15
+ import { closeDatabase, getEntryCount, openExistingDatabase } from "../../indexer/db/db.js";
20
16
  import { ensureIndex } from "../../indexer/ensure-index.js";
21
- import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
22
- import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
23
17
  import { akmIndex } from "../../indexer/indexer.js";
24
- import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
25
- import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
26
- import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
27
- import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
28
- import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
29
- import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
30
- import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
31
- import { isLlmFeatureEnabled, isProcessEnabled } from "../../llm/feature-gate.js";
18
+ import { resolveSourceEntries } from "../../indexer/search/search-source.js";
19
+ import { resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
32
20
  import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
33
21
  import { withLlmStage } from "../../llm/usage-telemetry.js";
34
22
  import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
35
- import { akmLint } from "../lint/index.js";
36
23
  import { drainProposals } from "../proposal/drain.js";
37
24
  import { resolveDrainPolicy } from "../proposal/drain-policies.js";
38
- import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
39
- import { runSchemaRepairPass } from "../sources/schema-repair.js";
40
- import { checkDeadUrls } from "../url-checker.js";
41
- import { akmConsolidate } from "./consolidate.js";
42
- import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
43
- import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
44
- import { countEvalCases, writeEvalCase } from "./eval-cases.js";
45
- import { akmExtract, countNewExtractCandidates } from "./extract.js";
46
- import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
47
- import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
25
+ import { akmDistill } from "./distill.js";
26
+ // Eligibility / candidate-selection predicates live in ./eligibility.
27
+ import { buildLatestProposalTsMap, collectEligibleRefs, memoryCleanupParentRef, resolveImproveScope, shouldAnalyzeMemoryCleanup, } from "./eligibility.js";
28
+ import { countEvalCases } from "./eval-cases.js";
29
+ import { resolveImproveProfile, resolveProcessEnabled } from "./improve-profiles.js";
30
+ // #607 per-process lock primitives live in ./locks. Imported for internal use;
31
+ // resetHeldProcessLocks is re-exported (the test seam imports it from here).
32
+ import { PROCESS_LOCK_DEFS, processLockPath, releaseAllProcessLocks, releaseHeldLocksIfOwned, releaseProcessLock, tryAcquireProcessLock, withOptionalProcessLock, } from "./locks.js";
33
+ // The cycle loop / post-loop / maintenance stages live in ./loop-stages.
34
+ import { runImproveLoopStage, runImprovePostLoopStage } from "./loop-stages.js";
48
35
  import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
49
- import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
50
- import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
36
+ import { analyzeMemoryCleanup } from "./memory/memory-improve.js";
37
+ // The pre-loop preparation pipeline lives in ./preparation.
38
+ import { runImprovePreparationStage } from "./preparation.js";
39
+ import { DEFAULT_DUE_DAYS, filterProactiveDue } from "./proactive-maintenance.js";
51
40
  import { akmReflect } from "./reflect.js";
52
- // #607 Lock Decomposition: fine-grained per-process locks replace the single
53
- // `improve.lock`. Three independent locks allow concurrent improve runs when
54
- // they touch different subsystems (e.g. quick-shredder consolidate can run
55
- // alongside daily reflect+distill).
56
- //
57
- // consolidate.lock — protects consolidate + memoryInference (both write index.db)
58
- // reflect-distill.lock — protects reflect + distill (both write state.db proposals)
59
- // triage.lock — protects triage (writes proposal promotions)
60
- //
61
- // Stale timeouts are per-lock, tuned to the expected runtime of the protected
62
- // processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
63
- // triage is fast (30min).
64
- const PROCESS_LOCK_DEFS = {
65
- consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
66
- reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
67
- triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
68
- };
69
- const heldProcessLocks = new Set();
70
- export function resetHeldProcessLocks() {
71
- heldProcessLocks.clear();
72
- }
73
- function processLockPath(lockBaseDir, lockName) {
74
- return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
75
- }
76
- function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
77
- fs.mkdirSync(path.dirname(lockPath), { recursive: true });
78
- const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
79
- if (tryAcquireLockSync(lockPath, lockPayload())) {
80
- heldProcessLocks.add(lockPath);
81
- return "acquired";
82
- }
83
- const probe = probeLock(lockPath, { staleAfterMs });
84
- const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
85
- const lock = rawContent
86
- ? (() => {
87
- try {
88
- return JSON.parse(rawContent);
89
- }
90
- catch {
91
- return null;
92
- }
93
- })()
94
- : null;
95
- if (probe.state === "stale") {
96
- try {
97
- appendEvent({
98
- eventType: "improve_lock_recovered",
99
- metadata: {
100
- lockName: lockLabel,
101
- stalePid: lock?.pid ?? null,
102
- lockedAt: lock?.startedAt ?? null,
103
- recoveredAt: new Date().toISOString(),
104
- lockAgeMs: probe.ageMs ?? null,
105
- reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
106
- },
107
- });
108
- }
109
- catch {
110
- /* event emission is best-effort; never block lock recovery */
111
- }
112
- releaseLock(lockPath);
113
- if (tryAcquireLockSync(lockPath, lockPayload())) {
114
- heldProcessLocks.add(lockPath);
115
- return "acquired";
116
- }
117
- if (skipIfLocked) {
118
- warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
119
- return "skipped";
120
- }
121
- throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
122
- }
123
- if (skipIfLocked) {
124
- warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
125
- return "skipped";
126
- }
127
- throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
128
- }
129
- function releaseProcessLock(lockPath) {
130
- try {
131
- fs.unlinkSync(lockPath);
132
- }
133
- catch {
134
- // ignore
135
- }
136
- heldProcessLocks.delete(lockPath);
137
- }
138
- function releaseAllProcessLocks() {
139
- for (const p of heldProcessLocks) {
140
- try {
141
- fs.unlinkSync(p);
142
- }
143
- catch {
144
- // ignore
145
- }
146
- }
147
- heldProcessLocks.clear();
148
- }
149
- function resolveImproveScope(scope) {
150
- const trimmed = scope?.trim();
151
- if (!trimmed)
152
- return { mode: "all" };
153
- try {
154
- parseAssetRef(trimmed);
155
- return { mode: "ref", value: trimmed };
156
- }
157
- catch {
158
- if (!isAssetType(trimmed)) {
159
- throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
160
- `If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
161
- }
162
- return { mode: "type", value: trimmed };
163
- }
164
- }
165
- /**
166
- * Render the end-of-run stash-sync commit message, expanding `{token}`
167
- * placeholders against this run's results. Unknown tokens are passed through
168
- * verbatim so adding new tokens later never breaks an existing template, and so
169
- * a literal brace in a message is harmless.
170
- *
171
- * Supported tokens (the "free" set — derived from data already on the result):
172
- * {timestamp} `YYYY-MM-DD HH:MM:SS` (UTC)
173
- * {date} `YYYY-MM-DD` (UTC)
174
- * {time} `HH:MM:SS` (UTC)
175
- * {scope} scope value (e.g. a ref/type) or the scope mode (`all`)
176
- * {refs} number of planned refs this run processed
177
- * {accepted} number of proposals auto-accepted by the confidence gate
178
- * {triage_promoted} proposals promoted by the triage pre-pass (0 if triage did not run)
179
- * {triage_rejected} proposals rejected by the triage pre-pass (0 if triage did not run)
180
- * {runId} this run's id (empty string when absent)
181
- *
182
- * The result is still passed through `sanitizeCommitMessage` downstream in
183
- * `saveGitStash`, so token values never widen the commit-message attack surface
184
- * (newlines/control chars are collapsed there).
185
- *
186
- * `nowMs` is injected (not read from `Date.now()`) so the function is pure and
187
- * deterministically testable.
188
- */
41
+ import { errMessage } from "./shared.js";
42
+ export { resetHeldProcessLocks } from "./locks.js";
43
+ // Re-exported from ./loop-stages for test importers (improve-db-locking).
44
+ export { runImproveMaintenancePasses } from "./loop-stages.js";
45
+ // Re-exported from ./preparation so existing importers (tests, callers) resolve.
46
+ export { maybeAutoTuneThreshold } from "./preparation.js";
189
47
  export function renderSyncCommitMessage(template, result, nowMs) {
190
48
  const iso = new Date(nowMs).toISOString();
191
49
  const tokens = {
@@ -201,350 +59,6 @@ export function renderSyncCommitMessage(template, result, nowMs) {
201
59
  };
202
60
  return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
203
61
  }
204
- /**
205
- * Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
206
- * merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
207
- * Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
208
- */
209
- function dedupeRefs(refs) {
210
- const seen = new Set();
211
- const out = [];
212
- for (const r of refs) {
213
- if (seen.has(r.ref))
214
- continue;
215
- seen.add(r.ref);
216
- out.push(r);
217
- }
218
- return out;
219
- }
220
- async function collectEligibleRefs(scope, stashDir, improveProfile) {
221
- if (scope.mode === "ref" && scope.value) {
222
- const parsed = parseAssetRef(scope.value);
223
- const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
224
- const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
225
- if (!filePath) {
226
- return {
227
- plannedRefs: [],
228
- memorySummary: { eligible: 0, derived: 0 },
229
- profileFilteredRefs: [],
230
- };
231
- }
232
- return {
233
- plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
234
- memorySummary: {
235
- eligible: parsed.type === "memory" ? 1 : 0,
236
- derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
237
- },
238
- profileFilteredRefs: [],
239
- };
240
- }
241
- let sources;
242
- try {
243
- sources = resolveSourceEntries(stashDir);
244
- }
245
- catch {
246
- return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
247
- }
248
- if (sources.length === 0) {
249
- return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
250
- }
251
- // Only operate on writable sources — never mutate read-only registry caches
252
- // or remote stashes that the user did not mark writable.
253
- let writableDirs;
254
- try {
255
- writableDirs = getWritableStashDirs(stashDir);
256
- }
257
- catch {
258
- writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
259
- }
260
- const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
261
- let db;
262
- try {
263
- db = openExistingDatabase();
264
- const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
265
- // First apply the existing stashDir-scope filter (no-op when stashDir is unset).
266
- if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
267
- return false;
268
- // Then restrict to writable sources only.
269
- return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
270
- });
271
- const planned = new Map();
272
- const profileFiltered = new Map();
273
- let memoryEligible = 0;
274
- let memoryDerived = 0;
275
- for (const indexed of entries) {
276
- const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
277
- const isDerived = indexed.entry.name.endsWith(".derived");
278
- // `.derived` memories are LLM-inferred and intentionally skip reflect
279
- // (see the synthetic `derived-memory-reflect-skipped` branch in the
280
- // improve loop). Enqueueing them here just produced one synthetic skip
281
- // per derived memory per hour with no real work — pure churn observed
282
- // 2026-05-21: 11 derived refs re-planned every hour during idle periods.
283
- // The cleanup phase (analyzeMemoryCleanup) inspects derived memories
284
- // independently of `plannedRefs`, so dropping them here loses nothing.
285
- if (!isDerived && !planned.has(ref) && !profileFiltered.has(ref)) {
286
- // 2026-05-27: extend the .derived precedent to profile-incompatible
287
- // refs. If every per-ref pass (reflect + distill) on the active
288
- // profile would refuse this ref, drop it from `plannedRefs`. The
289
- // caller emits `improve_skipped { reason: profile_filtered_all_passes }`
290
- // once `eventsCtx` is available so the audit trail is preserved in a
291
- // single event per ref instead of 2× synthetic actions per run.
292
- // Background: see /tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md
293
- if (improveProfile && isProfileFilteredForAllPasses(ref, improveProfile)) {
294
- profileFiltered.set(ref, {
295
- ref,
296
- reason: "profile_filtered_all_passes",
297
- filePath: indexed.filePath,
298
- });
299
- }
300
- else {
301
- planned.set(ref, {
302
- ref,
303
- reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
304
- filePath: indexed.filePath,
305
- });
306
- }
307
- }
308
- if (indexed.entry.type === "memory") {
309
- memoryEligible += 1;
310
- if (isDerived)
311
- memoryDerived += 1;
312
- }
313
- }
314
- return {
315
- plannedRefs: [...planned.values()],
316
- memorySummary: { eligible: memoryEligible, derived: memoryDerived },
317
- profileFilteredRefs: [...profileFiltered.values()],
318
- };
319
- }
320
- catch (error) {
321
- // The bun-test isolation guard must never be downgraded to "empty plan".
322
- rethrowIfTestIsolationError(error);
323
- if (error instanceof NotFoundError || error instanceof Error) {
324
- return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
325
- }
326
- throw error;
327
- }
328
- finally {
329
- if (db)
330
- closeDatabase(db);
331
- }
332
- }
333
- function isEntryInScope(entryStashDir, filePath, stashDir) {
334
- if (!stashDir)
335
- return true;
336
- const resolvedEntryStashDir = path.resolve(entryStashDir);
337
- const resolvedFilePath = path.resolve(filePath);
338
- const resolvedScopeStashDir = path.resolve(stashDir);
339
- return (resolvedEntryStashDir === resolvedScopeStashDir ||
340
- resolvedEntryStashDir.startsWith(`${resolvedScopeStashDir}${path.sep}`) ||
341
- resolvedFilePath.startsWith(`${resolvedScopeStashDir}${path.sep}`));
342
- }
343
- /**
344
- * Return true when the indexed entry belongs to one of the writable source
345
- * directories. Entries from read-only registry caches or remote stashes that
346
- * the user has not marked writable must never enter the improve/distill loop.
347
- */
348
- function isEntryInWritableSource(entryStashDir, filePath, writableDirSet) {
349
- const resolvedEntryStashDir = path.resolve(entryStashDir);
350
- const resolvedFilePath = path.resolve(filePath);
351
- for (const writableDir of writableDirSet) {
352
- if (resolvedEntryStashDir === writableDir ||
353
- resolvedEntryStashDir.startsWith(`${writableDir}${path.sep}`) ||
354
- resolvedFilePath.startsWith(`${writableDir}${path.sep}`)) {
355
- return true;
356
- }
357
- }
358
- return false;
359
- }
360
- function memoryCleanupParentRef(scope, stashDir) {
361
- if (scope.mode !== "ref" || !scope.value)
362
- return undefined;
363
- const parsed = parseAssetRef(scope.value);
364
- if (parsed.type !== "memory")
365
- return undefined;
366
- if (!parsed.name.endsWith(".derived"))
367
- return scope.value;
368
- const sources = resolveSourceEntries(stashDir);
369
- for (const source of sources) {
370
- const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
371
- if (!fs.existsSync(candidate))
372
- continue;
373
- const raw = fs.readFileSync(candidate, "utf8");
374
- const fm = parseFrontmatter(raw).data;
375
- const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
376
- if (sourceRef) {
377
- try {
378
- const parent = parseAssetRef(sourceRef.trim());
379
- if (parent.type === "memory")
380
- return makeAssetRef(parent.type, parent.name);
381
- }
382
- catch { }
383
- }
384
- }
385
- return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
386
- }
387
- function isLessonCandidate(ref) {
388
- // Only lesson assets need lesson-schema validation (description + when_to_use).
389
- // Memories have their own distill path via shouldDistillMemoryRef.
390
- // All other types go through reflect, not distill.
391
- return parseAssetRef(ref).type === "lesson";
392
- }
393
- /**
394
- * Planner-side check: should this ref enter the distill queue?
395
- *
396
- * Distill produces lessons from non-lesson sources. Two cases are eligible:
397
- *
398
- * 1. Memory refs that pass {@link shouldDistillMemoryRef} (the existing
399
- * memory→lesson/knowledge promotion path).
400
- *
401
- * Refs whose `type` is in {@link DISTILL_REFUSED_INPUT_TYPES} (currently
402
- * `lesson:*`) are explicitly excluded — distill refuses them at runtime and
403
- * queuing them just produces a no-op `skipped` outcome per ref per hour. That
404
- * planner waste was the bug fixed in commit
405
- * fix(improve): drop distill-refused types from planner.
406
- *
407
- * Note: prior to this fix the gate used `isLessonCandidate(ref)` directly,
408
- * which was true *only* for `lesson:*` refs — exactly the set distill refuses.
409
- * The result: every hourly run re-queued the same lesson refs, the same skip
410
- * message returned, and no work was ever done. See
411
- * `tests/commands/improve-distill-planner-skip-lessons.test.ts`.
412
- */
413
- function isDistillCandidateRef(ref, stashDir) {
414
- const parsed = parseAssetRef(ref);
415
- if (isDistillRefusedInputType(parsed.type))
416
- return false;
417
- return shouldDistillMemoryRef(ref, stashDir);
418
- }
419
- function shouldDistillMemoryRef(ref, stashDir) {
420
- const parsed = parseAssetRef(ref);
421
- if (parsed.type !== "memory")
422
- return false;
423
- const sources = resolveSourceEntries(stashDir);
424
- for (const source of sources) {
425
- const candidate = `${source.path}/memories/${parsed.name}.md`;
426
- if (!fs.existsSync(candidate))
427
- continue;
428
- const raw = fs.readFileSync(candidate, "utf8");
429
- const fm = parseFrontmatter(raw).data;
430
- const quality = typeof fm.quality === "string" ? fm.quality : undefined;
431
- if (quality === "proposed")
432
- return false;
433
- return !parsed.name.endsWith(".derived");
434
- }
435
- return !parsed.name.endsWith(".derived");
436
- }
437
- // ── Signal-delta eligibility helpers (0.8.0) ────────────────────────────────
438
- //
439
- // The 0.8.0 redesign replaced flat time-based cooldowns for reflect/distill
440
- // with a *signal-delta* gate: a ref is re-eligible iff new feedback has
441
- // landed since the last proposal was generated for it. These helpers build
442
- // the two timestamp maps the gate needs in bulk, so the planner avoids
443
- // N+1 queries across the full postCleanupRefs set.
444
- /**
445
- * Latest feedback event timestamp per ref in the active window. Reads all
446
- * `feedback` events newer than `sinceIso` in one query and indexes by ref,
447
- * keeping the maximum `ts` per ref.
448
- *
449
- * Only events with a meaningful payload count as "signal" — `metadata.signal`
450
- * (positive/negative) OR `metadata.note` (a free-form annotation). Empty
451
- * metadata events are ignored so a stray `akm feedback <ref>` invocation
452
- * without a flag doesn't trigger downstream re-processing.
453
- */
454
- function buildLatestFeedbackTsMap(refs, sinceIso) {
455
- const out = new Map();
456
- if (refs.length === 0)
457
- return out;
458
- const refSet = new Set(refs);
459
- const { events } = readEvents({ type: "feedback", since: sinceIso });
460
- for (const e of events) {
461
- const ref = e.ref;
462
- if (!ref || !refSet.has(ref))
463
- continue;
464
- const meta = e.metadata;
465
- const hasSignal = meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
466
- if (!hasSignal)
467
- continue;
468
- const ts = e.ts ?? "";
469
- if (ts > (out.get(ref) ?? ""))
470
- out.set(ref, ts);
471
- }
472
- return out;
473
- }
474
- /**
475
- * Latest proposal timestamp per input-ref, filtered by source ('reflect' or
476
- * 'distill'). Reads the corresponding `*_invoked` events from state.db —
477
- * these events are emitted at proposal creation time and carry the *input*
478
- * asset ref (memory:foo, skill:bar, etc.) directly. We use them rather than
479
- * `listProposals` because distill proposals are keyed by the derived
480
- * lesson/knowledge ref, not the source memory — joining back through the
481
- * payload would be fragile.
482
- */
483
- function buildLatestProposalTsMap(refs, source) {
484
- const out = new Map();
485
- if (refs.length === 0)
486
- return out;
487
- const refSet = new Set(refs);
488
- const eventType = source === "reflect" ? "reflect_invoked" : "distill_invoked";
489
- const { events } = readEvents({ type: eventType });
490
- for (const e of events) {
491
- const ref = e.ref;
492
- if (!ref || !refSet.has(ref))
493
- continue;
494
- // For distill_invoked we only count attempts that produced (or attempted
495
- // to produce) a real proposal — config_disabled / parse-error outcomes
496
- // should not move the signal-delta cursor forward.
497
- if (eventType === "distill_invoked") {
498
- const outcome = e.metadata?.outcome;
499
- if (outcome !== "queued" && outcome !== "skipped" && outcome !== "validation_failed")
500
- continue;
501
- }
502
- const ts = e.ts ?? "";
503
- if (ts > (out.get(ref) ?? ""))
504
- out.set(ref, ts);
505
- }
506
- return out;
507
- }
508
- /**
509
- * Signal-delta eligibility predicate.
510
- *
511
- * True iff `latestFeedback[ref]` is defined AND either no prior proposal
512
- * exists for this (ref, source) OR `latestFeedback[ref] > lastProposal[ref]`.
513
- *
514
- * Refs with no feedback signal at all are ineligible by definition — the
515
- * high-retrieval fallback path (see `noFeedbackCandidates` later in the
516
- * planner) handles never-touched-but-frequently-read assets separately.
517
- */
518
- function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
519
- const fb = latestFeedback.get(ref);
520
- if (!fb)
521
- return false;
522
- const lp = lastProposal.get(ref);
523
- if (!lp)
524
- return true;
525
- return fb > lp;
526
- }
527
- /**
528
- * H7 (#566): cooperative budget watchdog with a captured, RAII-cleared hard-kill.
529
- *
530
- * When the wall-clock budget expires, `onExhausted` (normally an
531
- * `AbortController.abort`) signals cooperative cancellation so the run can drain
532
- * its in-flight log/`state.db` flush and unwind naturally. A second hard-kill
533
- * timer is then armed as a watchdog: it only `exit(0)`s if the drain itself
534
- * overruns `hardKillGraceMs`, preventing the process from outliving the task
535
- * timeout window (lock-cascade fix).
536
- *
537
- * Both timers are captured; the returned dispose() clears whichever is still
538
- * pending. Callers invoke it from a `finally`, so a *clean* drain reaches the
539
- * `finally` and cancels the pending hard-kill before it can fire — the previous
540
- * detached `setTimeout(() => process.exit(0), 5000)` always fired, truncating a
541
- * clean flush. The hard-kill timer is `unref()`-ed so it never keeps the event
542
- * loop alive on its own: once the run drains it exits with its own code, not the
543
- * forced 0.
544
- *
545
- * Dependencies are injectable purely so the concurrency-sensitive timing
546
- * contract can be exercised deterministically in unit tests.
547
- */
548
62
  export function armBudgetWatchdog(budgetMs, controller, deps) {
549
63
  const setTimeoutFn = deps?.setTimeoutFn ?? setTimeout;
550
64
  const clearTimeoutFn = deps?.clearTimeoutFn ?? clearTimeout;
@@ -576,6 +90,11 @@ export async function akmImprove(options = {}) {
576
90
  const ensureIndexFn = options.ensureIndexFn ?? ensureIndex;
577
91
  const reindexFn = options.reindexFn ?? akmIndex;
578
92
  const drainProposalsFn = options.drainProposalsFn ?? drainProposals;
93
+ // #616 multi-cycle test seams. Default to the real module-local fns.
94
+ const collectEligibleRefsImpl = options.collectEligibleRefsFn ?? collectEligibleRefs;
95
+ const runImprovePreparationStageImpl = options.runImprovePreparationStageFn ?? runImprovePreparationStage;
96
+ const runImproveLoopStageImpl = options.runImproveLoopStageFn ?? runImproveLoopStage;
97
+ const runImprovePostLoopStageImpl = options.runImprovePostLoopStageFn ?? runImprovePostLoopStage;
579
98
  // Resolve the improve profile for this run. Profile drives type filtering,
580
99
  // process gating, and default autoAccept/limit values.
581
100
  const _earlyConfig = options.config ?? loadConfig();
@@ -590,6 +109,9 @@ export async function akmImprove(options = {}) {
590
109
  // CLI --limit takes precedence over both.
591
110
  limit: options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit,
592
111
  };
112
+ // #616 — bounded multi-cycle phasing. CLI/programmatic override wins over
113
+ // profile.maxCycles; default 1 => single pass (byte-identical to pre-#616).
114
+ const maxCycles = Math.max(1, Math.trunc(options.maxCycles ?? improveProfile.maxCycles ?? 1));
593
115
  let primaryStashDir;
594
116
  try {
595
117
  primaryStashDir = resolveSourceEntries(options.stashDir)[0]?.path;
@@ -606,6 +128,23 @@ export async function akmImprove(options = {}) {
606
128
  // timeout root cause). Because beforeEach runs synchronously, env is still the
607
129
  // calling test's own at this point; we capture it before yielding the loop.
608
130
  const resolvedStateDbPath = getStateDbPathInDataDir();
131
+ // #612 / WS-4 — bounded, OPT-IN per-phase auto-accept threshold auto-tune.
132
+ // DEFAULT OFF: `autoTune: false` (or absent) is a complete no-op.
133
+ //
134
+ // WS-4 change: thresholds are now PER PHASE. The old single global mutation
135
+ // of `options.autoAccept` is retired — it caused every phase to share one
136
+ // calibration signal, so a reflect-dominated run could tighten the consolidate
137
+ // gate (or vice-versa). Instead:
138
+ // - Each `makeGateConfig` call reads the phase's stored threshold from
139
+ // state.db (Migration 012) and uses it as `phaseThreshold`, overriding
140
+ // the `globalThreshold` (= options.autoAccept) for that phase.
141
+ // - Per-phase `maybeAutoTuneThreshold` calls fire AFTER each phase's gate
142
+ // has run and persist the new threshold to state.db for the NEXT run.
143
+ // - `options.autoAccept` stays unchanged (it is the operator-supplied
144
+ // baseline, not a mutable run-time state).
145
+ //
146
+ // The global tune call is intentionally removed here. See per-phase calls
147
+ // below (near each makeGateConfig / runAutoAcceptGate block).
609
148
  // #607 Lock decomposition: three per-process locks replace the single
610
149
  // `improve.lock`. Each process acquires only the lock(s) it needs, so
611
150
  // quick-shredder consolidate can run alongside daily reflect+distill.
@@ -617,70 +156,23 @@ export async function akmImprove(options = {}) {
617
156
  // Lock base directory — same `.akm/` under the primary stash dir.
618
157
  const lockBaseDir = primaryStashDir ? path.join(primaryStashDir, ".akm") : path.join(options.stashDir ?? ".", ".akm");
619
158
  const preEnsureCleanupWarnings = [];
620
- let plannedRefs;
621
- let memorySummary;
622
- let profileFilteredRefs;
159
+ // #616: assigned by runIndexAndCollect() (closure) so TS cannot prove definite
160
+ // assignment — seed with empty values; the first runIndexAndCollect() call
161
+ // (cycle 1, in the first try) always overwrites them before any read.
162
+ let plannedRefs = [];
163
+ let memorySummary = { eligible: 0, derived: 0 };
164
+ let profileFilteredRefs = [];
623
165
  let memoryCleanupPlan;
624
166
  let guidance;
625
167
  let triageDrain;
626
- try {
627
- // #607: Per-process lock acquisition. Each process acquires only the lock(s)
628
- // it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
629
- // locks (decision: dry-run never mutates the queue).
630
- if (!options.dryRun) {
631
- // Backstop release on process.exit() (signal handler / budget watchdog),
632
- // which skips the finally below. Removed in that finally on the normal path.
633
- const releaseAllOnExit = () => {
634
- for (const p of heldProcessLocks) {
635
- releaseLockIfOwned(p, process.pid);
636
- }
637
- };
638
- process.on("exit", releaseAllOnExit);
639
- // #607 triage pre-pass: acquire triage.lock, drain the standing pending
640
- // backlog BEFORE ensureIndex so improve generates fresh proposals against
641
- // a cleared queue (no `duplicate_pending` collisions) and ensureIndex
642
- // absorbs triage's promotions for free. Release immediately after —
643
- // triage.lock is not needed again until the next improve run.
644
- if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
645
- if (scope.mode === "ref") {
646
- warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
647
- }
648
- else {
649
- const triageLPath = processLockPath(lockBaseDir, "triage");
650
- const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
651
- if (triageResult === "skipped") {
652
- triageDrain = undefined;
653
- }
654
- else {
655
- try {
656
- const triageConfig = improveProfile.processes?.triage;
657
- const policy = resolveDrainPolicy(triageConfig?.policy);
658
- const applyMode = triageConfig?.applyMode ?? "queue";
659
- const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
660
- const judgment = triageConfig?.judgment
661
- ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
662
- : null;
663
- triageDrain = await drainProposalsFn({
664
- stashDir: primaryStashDir,
665
- policy,
666
- applyMode,
667
- maxAccepts,
668
- dryRun: false,
669
- excludeIds: new Set(),
670
- ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
671
- judgment,
672
- });
673
- }
674
- catch (err) {
675
- warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
676
- }
677
- finally {
678
- releaseProcessLock(triageLPath);
679
- }
680
- }
681
- }
682
- }
683
- }
168
+ // #616 — ensureIndex + collectEligibleRefs + memory-cleanup recompute, lifted
169
+ // into a helper so the SAME sequence runs once for cycle 1 (below, in the
170
+ // first try) and is re-run at the top of each subsequent multi-cycle cycle.
171
+ // Re-running ensureIndex between cycles makes cycle N's gate-promoted
172
+ // proposals visible to cycle N+1's collectEligibleRefs. Mutates the
173
+ // outer-scope plannedRefs/memorySummary/profileFilteredRefs/memoryCleanupPlan/
174
+ // guidance so for maxCycles:1 the body is byte-identical to pre-#616.
175
+ const runIndexAndCollect = async () => {
684
176
  // #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs. The eligible-ref
685
177
  // query reads the `entries` table; if a DB version upgrade just dropped that
686
178
  // table (or the index is otherwise empty), the prior run order silently
@@ -711,7 +203,7 @@ export async function akmImprove(options = {}) {
711
203
  await ensureIndexFn(primaryStashDir, { mode: "blocking" });
712
204
  }
713
205
  catch (err) {
714
- preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
206
+ preEnsureCleanupWarnings.push(`ensureIndex failed: ${errMessage(err)}`);
715
207
  }
716
208
  // #339 loud-fail: if the index was empty pre-ensureIndex but is now
717
209
  // populated, a version-upgrade-triggered rebuild just happened. Surface
@@ -738,7 +230,7 @@ export async function akmImprove(options = {}) {
738
230
  }
739
231
  }
740
232
  }
741
- ({ plannedRefs, memorySummary, profileFilteredRefs } = await collectEligibleRefs(scope, options.stashDir, improveProfile));
233
+ ({ plannedRefs, memorySummary, profileFilteredRefs } = await collectEligibleRefsImpl(scope, options.stashDir, improveProfile));
742
234
  const cleanupParentRef = memoryCleanupParentRef(scope, options.stashDir);
743
235
  // M-1 (#367): Run contradiction-detection BEFORE analyzeMemoryCleanup so
744
236
  // the SCC resolver in resolveFamilyContradictions has edges to work on.
@@ -750,7 +242,7 @@ export async function akmImprove(options = {}) {
750
242
  }
751
243
  catch (err) {
752
244
  // Non-fatal: contradiction detection is a best-effort pass.
753
- warn(`[improve] contradiction detection failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
245
+ warn(`[improve] contradiction detection failed (non-fatal): ${errMessage(err)}`);
754
246
  }
755
247
  }
756
248
  memoryCleanupPlan = shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)
@@ -760,6 +252,70 @@ export async function akmImprove(options = {}) {
760
252
  memorySummary.eligible > 0
761
253
  ? "Improve folds memory cleanup into the same proposal queue: speculative promotions still go through reflect/distill proposals, while high-confidence redundant derived memories are moved into a recoverable cleanup archive instead of being left active in the stash."
762
254
  : undefined;
255
+ };
256
+ // Holds our own process.on("exit") backstop so the finally can remove EXACTLY
257
+ // that handler (not every exit listener in the process). Declared in the scope
258
+ // shared by the try and its finally; assigned when the backstop is registered.
259
+ let exitBackstop;
260
+ try {
261
+ // #607: Per-process lock acquisition. Each process acquires only the lock(s)
262
+ // it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
263
+ // locks (decision: dry-run never mutates the queue).
264
+ if (!options.dryRun) {
265
+ // Backstop release on process.exit() (signal handler / budget watchdog),
266
+ // which skips the finally below. Removed in that finally on the normal path.
267
+ const releaseAllOnExit = () => releaseHeldLocksIfOwned(process.pid);
268
+ exitBackstop = releaseAllOnExit;
269
+ process.on("exit", releaseAllOnExit);
270
+ // #607 triage pre-pass: acquire triage.lock, drain the standing pending
271
+ // backlog BEFORE ensureIndex so improve generates fresh proposals against
272
+ // a cleared queue (no `duplicate_pending` collisions) and ensureIndex
273
+ // absorbs triage's promotions for free. Release immediately after —
274
+ // triage.lock is not needed again until the next improve run.
275
+ if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
276
+ if (scope.mode === "ref") {
277
+ warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
278
+ }
279
+ else {
280
+ const triageLPath = processLockPath(lockBaseDir, "triage");
281
+ const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
282
+ if (triageResult === "skipped") {
283
+ triageDrain = undefined;
284
+ }
285
+ else {
286
+ try {
287
+ const triageConfig = improveProfile.processes?.triage;
288
+ const policy = resolveDrainPolicy(triageConfig?.policy);
289
+ const applyMode = triageConfig?.applyMode ?? "queue";
290
+ const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
291
+ const judgment = triageConfig?.judgment
292
+ ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
293
+ : null;
294
+ triageDrain = await drainProposalsFn({
295
+ stashDir: primaryStashDir,
296
+ policy,
297
+ applyMode,
298
+ maxAccepts,
299
+ dryRun: false,
300
+ excludeIds: new Set(),
301
+ ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
302
+ judgment,
303
+ });
304
+ }
305
+ catch (err) {
306
+ warn(`[improve] triage pre-pass failed (non-fatal): ${errMessage(err)}`);
307
+ }
308
+ finally {
309
+ releaseProcessLock(triageLPath);
310
+ }
311
+ }
312
+ }
313
+ }
314
+ }
315
+ // #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs (now inside the
316
+ // helper). Cycle 1 runs it here; subsequent multi-cycle cycles re-run it via
317
+ // the same helper at the top of each cycle below.
318
+ await runIndexAndCollect();
763
319
  if (options.dryRun) {
764
320
  const result = {
765
321
  schemaVersion: 1,
@@ -792,6 +348,16 @@ export async function akmImprove(options = {}) {
792
348
  // run past the declared budget.
793
349
  // References: Anthropic *Building Effective Agents* (2024); CoALA §5 (arXiv:2309.02427).
794
350
  const budgetAbortController = new AbortController();
351
+ // Attach a live `remainingBudgetMs` getter to the signal so sub-callers
352
+ // (e.g. consolidate.ts cold-start budget estimation) can read the remaining
353
+ // wall-clock budget without needing an extra plumbing parameter. The property
354
+ // is computed at access time via a getter so it always reflects the actual
355
+ // elapsed time rather than a stale snapshot taken at arm time.
356
+ Object.defineProperty(budgetAbortController.signal, "remainingBudgetMs", {
357
+ get: () => Math.max(0, budgetMs - (Date.now() - startMs)),
358
+ enumerable: false,
359
+ configurable: true,
360
+ });
795
361
  // Declared in the outer scope so the `finally` can clear the timer even if a
796
362
  // throw occurs before/after it is armed. Defaults to a no-op until armed.
797
363
  let clearBudgetTimer = () => { };
@@ -806,6 +372,63 @@ export async function akmImprove(options = {}) {
806
372
  // #576: clears the per-run LLM usage sink. Defaults to a no-op until the sink
807
373
  // is installed inside the try; the `finally` always calls it.
808
374
  let disposeLlmUsageSink = () => { };
375
+ // ── Crash-safe / incremental stash sync (#662) ──────────────────────────────
376
+ // The primary stash writes as a filesystem source DURING the run
377
+ // (write-source.ts case-3); those writes become a git commit only when this
378
+ // closure runs. Historically the only call site was a single BATCH commit at
379
+ // the very end of the happy path, so a run interrupted AFTER writing but
380
+ // BEFORE finishing — a mid-cycle crash, a budget abort, or an external
381
+ // SIGTERM/`process.exit` — left every write uncommitted until some LATER run
382
+ // happened to finish cleanly and swept the whole backlog up. We now call this
383
+ // from THREE places: between cycles (bank each completed cycle), at end-of-run
384
+ // (the converged commit), and from the catch path (commit what was written
385
+ // before the crash). That shrinks the worst-case loss from "the entire run" to
386
+ // "the in-flight cycle".
387
+ //
388
+ // Declared in the OUTER scope (not inside the try) so the catch block can reach
389
+ // it. Idempotent + NON-FATAL: `saveGitStash` short-circuits a clean working
390
+ // tree ("nothing to commit") and a thrown sync error is swallowed here, so a
391
+ // repeat call after a no-op cycle is cheap and a failed push never fails the
392
+ // run. Gated identically to the original end-of-run block (git-backed primary
393
+ // stash, sync not disabled). `eventsCtx` is captured by reference, so calls
394
+ // after the db-backed context is installed inside the try use the live handle.
395
+ const effectiveSync = { ...improveProfile.sync, ...options.sync };
396
+ const commitStashBatch = (messageContext) => {
397
+ if (!primaryStashDir || effectiveSync.enabled === false || !isGitBackedStash(primaryStashDir)) {
398
+ return undefined;
399
+ }
400
+ const saveGitStashFn = options.saveGitStashFn ?? saveGitStash;
401
+ const writableOverride = resolveWritableOverride(_earlyConfig);
402
+ const push = effectiveSync.push !== false;
403
+ const message = renderSyncCommitMessage(effectiveSync.message ?? "akm improve auto-sync", messageContext, Date.now());
404
+ try {
405
+ const syncResult = saveGitStashFn(undefined, message, writableOverride, { push, repoDir: primaryStashDir });
406
+ appendEvent({
407
+ eventType: "stash_synced",
408
+ metadata: {
409
+ committed: syncResult.committed,
410
+ pushed: syncResult.pushed,
411
+ skipped: syncResult.skipped,
412
+ reason: syncResult.reason ?? null,
413
+ },
414
+ }, eventsCtx);
415
+ return {
416
+ committed: syncResult.committed,
417
+ pushed: syncResult.pushed,
418
+ skipped: syncResult.skipped,
419
+ ...(syncResult.reason !== undefined ? { reason: syncResult.reason } : {}),
420
+ };
421
+ }
422
+ catch (syncErr) {
423
+ const reason = errMessage(syncErr);
424
+ warn(`improve: stash sync failed (non-fatal): ${reason}`);
425
+ appendEvent({
426
+ eventType: "stash_synced",
427
+ metadata: { committed: false, pushed: false, skipped: true, reason },
428
+ }, eventsCtx);
429
+ return { committed: false, pushed: false, skipped: true, reason };
430
+ }
431
+ };
809
432
  try {
810
433
  // H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
811
434
  // budget timer and the hard-kill timer it schedules on exhaustion, returning
@@ -847,88 +470,244 @@ export async function akmImprove(options = {}) {
847
470
  },
848
471
  }, eventsCtx);
849
472
  }
850
- // #607: acquire consolidate.lock for the preparation stage (consolidate,
851
- // ensureIndex, extract all write index.db). Released immediately after.
852
- const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
853
- const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
854
- const preparation = await runImprovePreparationStage({
855
- scope,
856
- options,
857
- plannedRefs,
858
- memoryCleanupPlan,
859
- primaryStashDir,
860
- memorySummary,
861
- reindexFn,
862
- startMs,
863
- budgetMs,
864
- eventsCtx,
865
- initialCleanupWarnings: preEnsureCleanupWarnings,
866
- improveProfile,
867
- });
868
- if (consolidatePrepAcquired)
869
- releaseProcessLock(consolidateLPath);
870
- // D6: pre-load all proposal_rejected events from the last 30 days once,
871
- // so the per-asset loop can use a Map lookup instead of N DB round trips.
872
- const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
873
- const rejectedProposalSince = new Date(Date.now() - REJECTED_PROPOSAL_WINDOW_MS).toISOString();
874
- const allRejectedProposalEvents = readEvents({ type: "proposal_rejected", since: rejectedProposalSince }).events;
875
- const rejectedProposalsByRef = new Map();
876
- for (const e of allRejectedProposalEvents) {
877
- if (e.ref && (!rejectedProposalsByRef.has(e.ref) || e.ts > (rejectedProposalsByRef.get(e.ref)?.ts ?? ""))) {
878
- rejectedProposalsByRef.set(e.ref, e);
473
+ // #616 bounded multi-cycle phasing. The prep->loop->post-loop sequence is
474
+ // wrapped in an N-cycle loop. Each cycle re-runs ensureIndex +
475
+ // collectEligibleRefs (via runIndexAndCollect) so gate-accepted output of
476
+ // cycle N becomes selectable input to cycle N+1. The per-stage process locks
477
+ // (consolidate / reflect-distill) are acquired+released INSIDE each cycle,
478
+ // exactly as the single-pass path did. For maxCycles:1 the loop runs once and
479
+ // every accumulator below collapses to the single-cycle value (sum-of-one,
480
+ // concat-of-one, last==only) => BYTE-IDENTICAL to pre-#616.
481
+ //
482
+ // Accumulators (see CONSTRAINTS / aggregation plan in #616): SUM the count
483
+ // fields and durations; CONCAT the array fields; LAST-WINS for point-in-time
484
+ // objects (the final cycle's value reflects the converged state).
485
+ let cyclesRun = 0;
486
+ // Last-wins point-in-time values (assigned every cycle; the final cycle wins).
487
+ let preparation;
488
+ let memoryRefsForInference = new Set();
489
+ let consolidation;
490
+ let memoryInference;
491
+ let graphExtraction;
492
+ let recombination;
493
+ let proceduralCompilation;
494
+ let cycleMetrics;
495
+ // Summed counters/durations.
496
+ let prepGateCount = 0;
497
+ let prepGateFailedCount = 0;
498
+ let reflectsWithErrorContext = 0;
499
+ let loopGateCount = 0;
500
+ let loopGateFailedCount = 0;
501
+ let postLoopGateCount = 0;
502
+ let postLoopGateFailedCount = 0;
503
+ let memoryInferenceDurationMs = 0;
504
+ let graphExtractionDurationMs = 0;
505
+ let orphansPurged;
506
+ let proposalsExpired;
507
+ // Concatenated arrays.
508
+ const allWarnings = [];
509
+ let deadUrls;
510
+ const finalActions = [];
511
+ for (let cycleIndex = 0; cycleIndex < maxCycles; cycleIndex++) {
512
+ // #616 budget gate: never start a NEW cycle once the run's wall-clock
513
+ // budget is exhausted (or the run was aborted). Cycle 0 ALWAYS runs so
514
+ // maxCycles:1 is byte-identical regardless of budget.
515
+ if (cycleIndex > 0) {
516
+ const remaining = budgetAbortController.signal.remainingBudgetMs;
517
+ if (budgetAbortController.signal.aborted || (remaining !== undefined && remaining <= 0)) {
518
+ break;
519
+ }
520
+ }
521
+ // #662 incremental sync: bank the PREVIOUS cycle's writes before starting a
522
+ // new one, so a crash/abort/timeout mid-run loses at most the in-flight
523
+ // cycle rather than the whole run. Guarded on `cycleIndex > 0`, so the
524
+ // common maxCycles:1 path never calls this — its single end-of-run commit
525
+ // below stays the only sync and the serialized envelope is byte-identical
526
+ // to pre-#662. `saveGitStash` no-ops a clean tree, so a cycle that wrote
527
+ // nothing costs only a `git status`.
528
+ if (cycleIndex > 0) {
529
+ commitStashBatch({ scope, plannedRefs, runId: options.runId });
530
+ }
531
+ // Re-run ensureIndex + collectEligibleRefs + memory-cleanup recompute for
532
+ // cycles 2+ (cycle 1 already ran them in the first try above). This makes
533
+ // cycle N's gate-promoted proposals visible to this cycle's ref selection.
534
+ if (cycleIndex > 0) {
535
+ await runIndexAndCollect();
536
+ // Re-emit the profile-filtered audit summary for this cycle's selection.
537
+ if (profileFilteredRefs.length > 0) {
538
+ appendEvent({
539
+ eventType: "improve_skipped",
540
+ ref: undefined,
541
+ metadata: { reason: "profile_filtered_all_passes", count: profileFilteredRefs.length },
542
+ }, eventsCtx);
543
+ }
544
+ }
545
+ // #607: acquire consolidate.lock for the preparation stage (consolidate,
546
+ // ensureIndex, extract all write index.db). Released immediately after.
547
+ const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
548
+ preparation = await withOptionalProcessLock({
549
+ lockPath: consolidateLPath,
550
+ staleAfterMs: PROCESS_LOCK_DEFS.consolidate.staleAfterMs,
551
+ skipIfLocked: options.skipIfLocked,
552
+ label: "consolidate",
553
+ }, () => runImprovePreparationStageImpl({
554
+ scope,
555
+ options,
556
+ plannedRefs,
557
+ memoryCleanupPlan,
558
+ primaryStashDir,
559
+ memorySummary,
560
+ reindexFn,
561
+ startMs,
562
+ budgetMs,
563
+ eventsCtx,
564
+ initialCleanupWarnings: preEnsureCleanupWarnings,
565
+ improveProfile,
566
+ budgetSignal: budgetAbortController.signal,
567
+ }));
568
+ prepGateCount += preparation.gateAutoAcceptedCount;
569
+ prepGateFailedCount += preparation.gateAutoAcceptFailedCount;
570
+ // D6: pre-load all proposal_rejected events from the last 30 days once,
571
+ // so the per-asset loop can use a Map lookup instead of N DB round trips.
572
+ const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
573
+ const rejectedProposalSince = new Date(Date.now() - REJECTED_PROPOSAL_WINDOW_MS).toISOString();
574
+ const allRejectedProposalEvents = readEvents({ type: "proposal_rejected", since: rejectedProposalSince }).events;
575
+ const rejectedProposalsByRef = new Map();
576
+ for (const e of allRejectedProposalEvents) {
577
+ if (e.ref && (!rejectedProposalsByRef.has(e.ref) || e.ts > (rejectedProposalsByRef.get(e.ref)?.ts ?? ""))) {
578
+ rejectedProposalsByRef.set(e.ref, e);
579
+ }
580
+ }
581
+ // #607: acquire reflect-distill.lock for the loop stage (reflect + distill
582
+ // both write proposals to state.db). Released immediately after.
583
+ const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
584
+ const loopResult = await withOptionalProcessLock({
585
+ lockPath: reflectDistillLPath,
586
+ staleAfterMs: PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs,
587
+ skipIfLocked: options.skipIfLocked,
588
+ label: "reflect-distill",
589
+ }, () => {
590
+ // Post-lock cooldown re-filter for proactive refs (#SELECT-TIME-LEAK).
591
+ // Planning built `lastReflectProposalTs` BEFORE acquiring this lock, so a
592
+ // concurrent run's `reflect_invoked` writes are invisible to it. Now that
593
+ // we hold the lock, re-read fresh timestamp maps for the proactive subset
594
+ // and drop any ref whose cooldown has been consumed by the concurrent run.
595
+ const proactiveLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource === "proactive");
596
+ let postLockLoopRefs = preparation.loopRefs;
597
+ if (proactiveLoopRefs.length > 0) {
598
+ const proactiveRefStrs = proactiveLoopRefs.map((r) => r.ref);
599
+ const freshReflectTs = buildLatestProposalTsMap(proactiveRefStrs, "reflect");
600
+ const freshDistillTs = buildLatestProposalTsMap(proactiveRefStrs, "distill");
601
+ const pmDueDays = improveProfile.processes?.proactiveMaintenance?.dueDays ?? DEFAULT_DUE_DAYS;
602
+ const stillDue = new Set(filterProactiveDue(proactiveLoopRefs, freshReflectTs, freshDistillTs, pmDueDays, Date.now()).map((r) => r.ref));
603
+ const dropped = proactiveLoopRefs.filter((r) => !stillDue.has(r.ref));
604
+ if (dropped.length > 0) {
605
+ info(`[improve] post-lock cooldown re-filter: dropped ${dropped.length} proactive ref(s) claimed by concurrent run (${dropped.map((r) => r.ref).join(", ")})`);
606
+ postLockLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource !== "proactive" || stillDue.has(r.ref));
607
+ }
608
+ }
609
+ return runImproveLoopStageImpl({
610
+ scope,
611
+ options,
612
+ primaryStashDir,
613
+ reflectFn,
614
+ distillFn,
615
+ loopRefs: postLockLoopRefs,
616
+ actions: preparation.actions,
617
+ signalBearingSet: preparation.signalBearingSet,
618
+ distillCooledRefs: preparation.distillCooledRefs,
619
+ distillOnlyRefs: preparation.distillOnlyRefs,
620
+ recentErrors: preparation.recentErrors,
621
+ rejectedProposalsByRef,
622
+ utilityMap: preparation.utilityMap,
623
+ startMs,
624
+ budgetMs,
625
+ eventsCtx,
626
+ improveProfile,
627
+ budgetSignal: budgetAbortController.signal,
628
+ });
629
+ });
630
+ const loopGateCountThisCycle = loopResult.gateAutoAcceptedCount;
631
+ reflectsWithErrorContext += loopResult.reflectsWithErrorContext;
632
+ loopGateCount += loopResult.gateAutoAcceptedCount;
633
+ loopGateFailedCount += loopResult.gateAutoAcceptFailedCount;
634
+ memoryRefsForInference = loopResult.memoryRefsForInference;
635
+ // #551: consolidation now runs in the preparation stage (before extract);
636
+ // its result and run-flag are read from `preparation`, not the post-loop.
637
+ consolidation = preparation.consolidation;
638
+ // #607: acquire consolidate.lock for the post-loop stage (memoryInference +
639
+ // graphExtraction both write index.db). Released immediately after.
640
+ const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
641
+ const postLoopResult = await withOptionalProcessLock({
642
+ lockPath: consolidatePostLPath,
643
+ staleAfterMs: PROCESS_LOCK_DEFS.consolidate.staleAfterMs,
644
+ skipIfLocked: options.skipIfLocked,
645
+ label: "consolidate",
646
+ }, () => runImprovePostLoopStageImpl({
647
+ scope,
648
+ options,
649
+ primaryStashDir,
650
+ actionableRefs: preparation.actionableRefs,
651
+ appliedCleanup: preparation.appliedCleanup,
652
+ cleanupWarnings: preparation.cleanupWarnings,
653
+ memoryRefsForInference,
654
+ reindexFn,
655
+ eventsCtx,
656
+ budgetSignal: budgetAbortController.signal,
657
+ improveProfile,
658
+ consolidationRan: preparation.consolidationRan,
659
+ // R5: floor violations from this run's consolidate pass + the
660
+ // auto-accepted volume so far (prep + loop gates) for churn detection.
661
+ consolidationMergeFloorViolations: preparation.consolidation.mergeFloorViolations ?? 0,
662
+ acceptedActions: preparation.gateAutoAcceptedCount + loopGateCountThisCycle,
663
+ }));
664
+ const postLoopGateCountThisCycle = postLoopResult.gateAutoAcceptedCount;
665
+ // Last-wins point-in-time objects.
666
+ memoryInference = postLoopResult.memoryInference;
667
+ graphExtraction = postLoopResult.graphExtraction;
668
+ recombination = postLoopResult.recombination;
669
+ proceduralCompilation = postLoopResult.proceduralCompilation;
670
+ // Keep the last QUALIFYING cycle's snapshot — a later non-qualifying
671
+ // cycle in a maxCycles>1 run must not clobber it with undefined.
672
+ if (postLoopResult.cycleMetrics)
673
+ cycleMetrics = postLoopResult.cycleMetrics;
674
+ // Summed counters/durations.
675
+ postLoopGateCount += postLoopResult.gateAutoAcceptedCount;
676
+ postLoopGateFailedCount += postLoopResult.gateAutoAcceptFailedCount;
677
+ memoryInferenceDurationMs += postLoopResult.memoryInferenceDurationMs;
678
+ graphExtractionDurationMs += postLoopResult.graphExtractionDurationMs;
679
+ if (postLoopResult.orphansPurged !== undefined) {
680
+ orphansPurged = (orphansPurged ?? 0) + postLoopResult.orphansPurged;
681
+ }
682
+ if (postLoopResult.proposalsExpired !== undefined) {
683
+ proposalsExpired = (proposalsExpired ?? 0) + postLoopResult.proposalsExpired;
684
+ }
685
+ // Concatenated arrays.
686
+ allWarnings.push(...postLoopResult.allWarnings);
687
+ if (postLoopResult.deadUrls !== undefined) {
688
+ deadUrls = [...(deadUrls ?? []), ...postLoopResult.deadUrls];
689
+ }
690
+ const maintenanceActions = postLoopResult.maintenanceActions;
691
+ if (maintenanceActions && maintenanceActions.length > 0) {
692
+ finalActions.push(...preparation.actions, ...maintenanceActions);
879
693
  }
694
+ else {
695
+ finalActions.push(...preparation.actions);
696
+ }
697
+ cyclesRun++;
698
+ // #616 fixed-point stop: a cycle that produced ZERO gate-accepted proposals
699
+ // (summed across prep + loop + post-loop) would feed cycle N+1 an identical
700
+ // ref set, so end the loop here rather than spin a pointless next cycle.
701
+ const gateAcceptedThisCycle = preparation.gateAutoAcceptedCount + loopGateCountThisCycle + postLoopGateCountThisCycle;
702
+ if (gateAcceptedThisCycle === 0)
703
+ break;
880
704
  }
881
- // #607: acquire reflect-distill.lock for the loop stage (reflect + distill
882
- // both write proposals to state.db). Released immediately after.
883
- const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
884
- const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
885
- const { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount: loopGateCount, gateAutoAcceptFailedCount: loopGateFailedCount, } = await runImproveLoopStage({
886
- scope,
887
- options,
888
- primaryStashDir,
889
- reflectFn,
890
- distillFn,
891
- loopRefs: preparation.loopRefs,
892
- actions: preparation.actions,
893
- signalBearingSet: preparation.signalBearingSet,
894
- distillCooledRefs: preparation.distillCooledRefs,
895
- distillOnlyRefs: preparation.distillOnlyRefs,
896
- recentErrors: preparation.recentErrors,
897
- rejectedProposalsByRef,
898
- utilityMap: preparation.utilityMap,
899
- startMs,
900
- budgetMs,
901
- eventsCtx,
902
- improveProfile,
903
- });
904
- if (reflectDistillAcquired)
905
- releaseProcessLock(reflectDistillLPath);
906
- // #551: consolidation now runs in the preparation stage (before extract);
907
- // its result and run-flag are read from `preparation`, not the post-loop.
908
- const consolidation = preparation.consolidation;
909
- // #607: acquire consolidate.lock for the post-loop stage (memoryInference +
910
- // graphExtraction both write index.db). Released immediately after.
911
- const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
912
- const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
913
- const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
914
- scope,
915
- options,
916
- primaryStashDir,
917
- actionableRefs: preparation.actionableRefs,
918
- appliedCleanup: preparation.appliedCleanup,
919
- cleanupWarnings: preparation.cleanupWarnings,
920
- memoryRefsForInference,
921
- reindexFn,
922
- eventsCtx,
923
- budgetSignal: budgetAbortController.signal,
924
- improveProfile,
925
- consolidationRan: preparation.consolidationRan,
926
- });
927
- if (consolidatePostAcquired)
928
- releaseProcessLock(consolidatePostLPath);
929
- const finalActions = maintenanceActions && maintenanceActions.length > 0
930
- ? [...preparation.actions, ...maintenanceActions]
931
- : preparation.actions;
705
+ // C1 (13-bus-factor): fold the per-ref `distill-skipped` rows (~13k/run,
706
+ // ~91% of result_json bytes) into a bounded aggregate BEFORE persistence.
707
+ // The metric total + per-reason breakdown are preserved on `distillSkipped`;
708
+ // the unbounded row list never reaches result_json. Reflect skip counters
709
+ // below still read `finalActions` (reflect skips are not folded).
710
+ const { actions: persistedActions, aggregate: distillSkippedAggregate } = foldDistillSkipped(finalActions);
932
711
  const result = {
933
712
  schemaVersion: 1,
934
713
  ok: true,
@@ -959,7 +738,8 @@ export async function akmImprove(options = {}) {
959
738
  : {}),
960
739
  plannedRefs: preparation.actionableRefs,
961
740
  ...(profileFilteredRefs.length > 0 ? { profileFilteredRefs } : {}),
962
- actions: finalActions,
741
+ actions: persistedActions,
742
+ ...(distillSkippedAggregate ? { distillSkipped: distillSkippedAggregate } : {}),
963
743
  ...(preparation.validationFailures.length > 0 ? { validationFailures: preparation.validationFailures } : {}),
964
744
  ...(preparation.schemaRepairs.length > 0 ? { schemaRepairs: preparation.schemaRepairs } : {}),
965
745
  ...(consolidation.processed > 0 || consolidation.warnings.length > 0 ? { consolidation } : {}),
@@ -986,18 +766,20 @@ export async function akmImprove(options = {}) {
986
766
  // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1k / §3.
987
767
  ...(memoryInferenceDurationMs > 0 ? { memoryInferenceDurationMs } : {}),
988
768
  ...(graphExtractionDurationMs > 0 ? { graphExtractionDurationMs } : {}),
989
- ...(stalenessDetection ? { stalenessDetection } : {}),
769
+ ...(recombination ? { recombination } : {}),
770
+ ...(proceduralCompilation ? { proceduralCompilation } : {}),
771
+ ...(cycleMetrics ? { cycleMetrics } : {}),
990
772
  ...(orphansPurged !== undefined ? { orphansPurged } : {}),
991
773
  ...(proposalsExpired !== undefined && proposalsExpired > 0 ? { proposalsExpired } : {}),
992
774
  reflectCooldownActions: finalActions.filter((a) => a.mode === "reflect-cooldown").length,
993
775
  reflectSkippedActions: finalActions.filter((a) => a.mode === "reflect-skipped").length,
994
776
  reflectGuardRejectedActions: finalActions.filter((a) => a.mode === "reflect-guard-rejected").length,
995
777
  ...(() => {
996
- const t = preparation.gateAutoAcceptedCount + loopGateCount + postLoopGateCount;
778
+ const t = prepGateCount + loopGateCount + postLoopGateCount;
997
779
  return t > 0 ? { gateAutoAcceptedCount: t } : {};
998
780
  })(),
999
781
  ...(() => {
1000
- const f = preparation.gateAutoAcceptFailedCount + loopGateFailedCount + postLoopGateFailedCount;
782
+ const f = prepGateFailedCount + loopGateFailedCount + postLoopGateFailedCount;
1001
783
  return f > 0 ? { gateAutoAcceptFailedCount: f } : {};
1002
784
  })(),
1003
785
  ...(triageDrain
@@ -1011,6 +793,9 @@ export async function akmImprove(options = {}) {
1011
793
  }
1012
794
  : {}),
1013
795
  ...(preparation.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
796
+ // #616 — report cycles run only when >1 so the default single-pass
797
+ // serialized envelope stays byte-identical to pre-#616 (AC1).
798
+ ...(cyclesRun > 1 ? { cyclesRun } : {}),
1014
799
  ...(options.runId !== undefined ? { runId: options.runId } : {}),
1015
800
  };
1016
801
  if (!result.dryRun)
@@ -1021,57 +806,18 @@ export async function akmImprove(options = {}) {
1021
806
  warningCount: allWarnings.length,
1022
807
  orphansPurged: orphansPurged ?? 0,
1023
808
  }, eventsCtx);
1024
- // End-of-run BATCH auto-sync. Recognition is decoupled from the per-write
1025
- // path (see write-source.ts case-3): the primary stash writes as a
1026
- // filesystem source during the run, then is committed in one shot here via
1027
- // the same `saveGitStash` that `akm sync` calls. Gated on a non-dry-run, a
1028
- // git-backed primary stash (by `.git`, not by remote), and sync not
1029
- // disabled. A sync failure is NON-FATAL it never fails a successful run
1030
- // (mirrors the contradiction-detection best-effort pattern).
1031
- const effectiveSync = { ...improveProfile.sync, ...options.sync };
1032
- if (!result.dryRun && primaryStashDir && effectiveSync.enabled !== false && isGitBackedStash(primaryStashDir)) {
1033
- const saveGitStashFn = options.saveGitStashFn ?? saveGitStash;
1034
- // Reuse the config resolved at the top of the run (`_earlyConfig`) instead
1035
- // of a second loadConfig(); the writable derivation is shared with
1036
- // `akm sync` via resolveWritableOverride().
1037
- const writableOverride = resolveWritableOverride(_earlyConfig);
1038
- const push = effectiveSync.push !== false;
1039
- // `sync.message` may contain `{token}` placeholders (timestamp/date/time/
1040
- // scope/refs/accepted) expanded against this run's results; the default
1041
- // template has no tokens so it renders verbatim.
1042
- const message = renderSyncCommitMessage(effectiveSync.message ?? "akm improve auto-sync", result, Date.now());
1043
- try {
1044
- // Pass primaryStashDir as the explicit commit target so the gate above
1045
- // (which validated primaryStashDir via isGitBackedStash) and the commit
1046
- // operate on the SAME directory — avoids divergence when a caller passes
1047
- // a non-default options.stashDir (FIX 9).
1048
- const syncResult = saveGitStashFn(undefined, message, writableOverride, { push, repoDir: primaryStashDir });
1049
- result.sync = {
1050
- committed: syncResult.committed,
1051
- pushed: syncResult.pushed,
1052
- skipped: syncResult.skipped,
1053
- ...(syncResult.reason !== undefined ? { reason: syncResult.reason } : {}),
1054
- };
1055
- appendEvent({
1056
- eventType: "stash_synced",
1057
- metadata: {
1058
- committed: syncResult.committed,
1059
- pushed: syncResult.pushed,
1060
- skipped: syncResult.skipped,
1061
- reason: syncResult.reason ?? null,
1062
- },
1063
- }, eventsCtx);
1064
- }
1065
- catch (syncErr) {
1066
- const reason = syncErr instanceof Error ? syncErr.message : String(syncErr);
1067
- warn(`improve: end-of-run stash sync failed (non-fatal): ${reason}`);
1068
- result.sync = { committed: false, pushed: false, skipped: true, reason };
1069
- appendEvent({
1070
- eventType: "stash_synced",
1071
- metadata: { committed: false, pushed: false, skipped: true, reason },
1072
- }, eventsCtx);
1073
- }
1074
- }
809
+ // End-of-run BATCH auto-sync the converged commit. Recognition is
810
+ // decoupled from the per-write path (see write-source.ts case-3): the primary
811
+ // stash writes as a filesystem source during the run, then is committed via
812
+ // the same `saveGitStash` that `akm sync` calls. The gating (git-backed
813
+ // primary stash, sync not disabled) and the NON-FATAL guarantee now live in
814
+ // `commitStashBatch` (#662); the inter-cycle and catch-path calls reuse it.
815
+ // dry-run already returned above, so this always runs on a completed live
816
+ // run. `result.sync` reflects this final commit (for a one-cycle run it is
817
+ // the only commit; for a multi-cycle run the earlier cycles were banked by
818
+ // the inter-cycle calls and this records the last batch). `result` carries
819
+ // the full `{accepted}`/`{refs}`/`{triage_*}` token data for the message.
820
+ result.sync = commitStashBatch(result);
1075
821
  return result;
1076
822
  }
1077
823
  catch (err) {
@@ -1080,10 +826,17 @@ export async function akmImprove(options = {}) {
1080
826
  eventType: "improve_failed",
1081
827
  ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
1082
828
  metadata: {
1083
- error: err instanceof Error ? err.message : String(err),
829
+ error: errMessage(err),
1084
830
  durationMs: Date.now() - startMs,
1085
831
  },
1086
832
  }, eventsCtx);
833
+ // #662 crash/abort safety net: commit whatever this run already wrote to the
834
+ // primary stash BEFORE rethrowing, so an interrupted run (mid-cycle crash or
835
+ // a cooperative budget abort that surfaces as a throw) does not leave its
836
+ // writes uncommitted until a later clean run sweeps them up. Best-effort —
837
+ // `commitStashBatch` swallows its own errors and no-ops a clean tree, so this
838
+ // never masks or supersedes the original failure being rethrown below.
839
+ commitStashBatch({ scope, plannedRefs, runId: options.runId });
1087
840
  throw err;
1088
841
  }
1089
842
  finally {
@@ -1096,9 +849,15 @@ export async function akmImprove(options = {}) {
1096
849
  // #607: release any per-process locks still held (backstop for error paths;
1097
850
  // the normal path already released each lock after its stage completed).
1098
851
  releaseAllProcessLocks();
1099
- // Drop the process.exit backstop so it does not fire later (or accumulate
1100
- // across repeated in-process calls).
1101
- process.removeAllListeners("exit");
852
+ // Drop ONLY our own process.exit backstop so it does not fire later (or
853
+ // accumulate across repeated in-process calls). Must NOT use
854
+ // removeAllListeners("exit") here: in the in-process model (tests and
855
+ // programmatic callers import cli.ts) that would silently destroy exit
856
+ // handlers owned by the host or other commands.
857
+ if (exitBackstop) {
858
+ process.removeListener("exit", exitBackstop);
859
+ exitBackstop = undefined;
860
+ }
1102
861
  // I1: close the long-lived state.db connection opened at the top of the run.
1103
862
  try {
1104
863
  eventsDb?.close();
@@ -1125,7 +884,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1125
884
  // Coarse audit buckets, derived from the SAME classifyImproveAction the
1126
885
  // persisted metrics_json uses (state-db.ts#computeImproveRunMetrics) so the
1127
886
  // emitted event and the stored row can never disagree.
1128
- const classCounts = { accepted: 0, rejected: 0, error: 0, noop: 0 };
887
+ const classCounts = { accepted: 0, rejected: 0, skipped: 0, error: 0, noop: 0 };
1129
888
  for (const action of result.actions ?? []) {
1130
889
  classCounts[classifyImproveAction(action.mode)] += 1;
1131
890
  // Per-variant counters for the event metadata. The default arm makes any
@@ -1170,6 +929,13 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1170
929
  assertNever(action.mode);
1171
930
  }
1172
931
  }
932
+ // C1: distill-skipped rows are no longer in `result.actions` (folded into the
933
+ // bounded `distillSkipped` aggregate at assembly). Add the aggregate total to
934
+ // the per-variant counter AND the coarse `skipped` bucket so the emitted event
935
+ // still reports the true skipped volume.
936
+ const distillSkippedTotal = result.distillSkipped?.total ?? 0;
937
+ actionCounts.distillSkipped += distillSkippedTotal;
938
+ classCounts.skipped += distillSkippedTotal;
1173
939
  appendEvent({
1174
940
  eventType: "improve_completed",
1175
941
  ref: result.scope.mode === "ref"
@@ -1192,6 +958,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1192
958
  reflectGuardRejectedActions: actionCounts.reflectGuardRejected,
1193
959
  acceptedActions: classCounts.accepted,
1194
960
  rejectedActions: classCounts.rejected,
961
+ skippedActions: classCounts.skipped,
1195
962
  noopActions: classCounts.noop,
1196
963
  reflectsWithErrorContext: result.reflectsWithErrorContext ?? 0,
1197
964
  coverageGapCount: result.coverageGaps?.length ?? 0,
@@ -1230,2007 +997,6 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1230
997
  },
1231
998
  }, eventsCtx);
1232
999
  }
1233
- /**
1234
- * Run (or gate-skip) the memory consolidation pass.
1235
- *
1236
- * #551 — two coordinated changes live here:
1237
- *
1238
- * 1. STRUCTURAL: this runs before extract in the improve pipeline (see
1239
- * `runImprovePreparationStage`). Consolidation therefore only ever judges
1240
- * PRIOR-run memories; current-run extract promotions are invisible to it.
1241
- *
1242
- * 2. SMARTER POOL-DELTA GATE: even among on-disk files, a memory whose only
1243
- * post-`lastConsolidateTs` mtime bump came from its OWN auto-accept
1244
- * promotion (i.e. it was just promoted by extract in the immediately
1245
- * preceding run and has not had a full improve cycle to settle) does NOT
1246
- * count as "work to do". We exclude those paths from the pool-delta check
1247
- * using the `promoted` events already emitted with each promotion's
1248
- * `assetPath`. A genuinely-settled prior memory — one edited by feedback,
1249
- * reflect, manual edit, or simply older than the last consolidate — still
1250
- * triggers the run. This is gate-option (a) from the issue (same-run /
1251
- * adjacent-run promotion exclusion), chosen over option (b) because there
1252
- * is no `extract_completed` event in the data model to gate against;
1253
- * `promoted` events with `assetPath` already carry exactly the signal we
1254
- * need, so the fix is non-invasive and provably correct.
1255
- */
1256
- async function runConsolidationPass(args) {
1257
- const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
1258
- const baseConfig = options.config ?? loadConfig();
1259
- const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
1260
- const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
1261
- const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
1262
- // When volume triggers a consolidation pass, force-enable the consolidate
1263
- // process on the default improve profile so the gate accepts the run even
1264
- // if the user's config disabled it. We synthesise a new profile override
1265
- // rather than mutating connection settings.
1266
- const consolidationConfig = volumeTriggered
1267
- ? {
1268
- ...baseConfig,
1269
- profiles: {
1270
- ...(baseConfig.profiles ?? {}),
1271
- improve: {
1272
- ...(baseConfig.profiles?.improve ?? {}),
1273
- default: {
1274
- ...(baseConfig.profiles?.improve?.default ?? {}),
1275
- processes: {
1276
- ...(baseConfig.profiles?.improve?.default?.processes ?? {}),
1277
- consolidate: {
1278
- ...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
1279
- enabled: true,
1280
- },
1281
- },
1282
- },
1283
- },
1284
- },
1285
- }
1286
- : baseConfig;
1287
- // 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
1288
- // memory file has been updated since the most recent successful
1289
- // consolidate_completed event. Time-based cooldowns produced the same
1290
- // synchronised-wave failure mode the reflect/distill cooldowns did; the
1291
- // pool-delta gate ties consolidation to actual work-to-do.
1292
- const recentConsolidations = readEvents({ type: "consolidate_completed" });
1293
- const lastConsolidation = recentConsolidations.events
1294
- .filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
1295
- .sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
1296
- const lastConsolidateTs = lastConsolidation?.ts;
1297
- // #551 smarter gate: build the set of memory asset paths whose only delta
1298
- // since the last consolidate is their OWN auto-accept promotion. Those files
1299
- // have not had a full improve cycle to settle, so they offer no merge /
1300
- // contradiction candidates yet — excluding them stops the gate firing on
1301
- // freshly-promoted single-source memories. We read `promoted` events emitted
1302
- // after the last consolidate; each carries the written `assetPath`.
1303
- const promotedSinceConsolidate = (() => {
1304
- const paths = new Set();
1305
- try {
1306
- const promoted = readEvents({
1307
- type: "promoted",
1308
- ...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
1309
- }).events;
1310
- for (const e of promoted) {
1311
- const ap = e.metadata?.assetPath;
1312
- if (typeof ap === "string" && ap.length > 0)
1313
- paths.add(path.resolve(ap));
1314
- }
1315
- }
1316
- catch {
1317
- // best-effort: if the events query fails, fall back to no exclusions
1318
- // (preserves pre-#551 behaviour rather than over-skipping).
1319
- }
1320
- return paths;
1321
- })();
1322
- // Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do,
1323
- // EXCEPT files whose only post-consolidate change was their own promotion.
1324
- // Using file mtime keeps this query DB-free and matches what the indexer
1325
- // already uses as the canonical `memory.updated_at` proxy.
1326
- //
1327
- // Bootstrap: when no successful consolidate_completed event has ever been
1328
- // recorded, we cannot evaluate the pool-delta — treat as eligible so a
1329
- // fresh stash runs consolidate once before the steady-state gate kicks in.
1330
- const memoryUpdatedAfterLastConsolidate = (() => {
1331
- if (volumeTriggered)
1332
- return true; // volume override forces the run regardless.
1333
- if (!lastConsolidateTs)
1334
- return true; // bootstrap path: never consolidated.
1335
- if (!primaryStashDir)
1336
- return false;
1337
- const memoriesDir = path.join(primaryStashDir, "memories");
1338
- if (!fs.existsSync(memoriesDir))
1339
- return false;
1340
- try {
1341
- return fs.readdirSync(memoriesDir).some((f) => {
1342
- if (!f.endsWith(".md"))
1343
- return false;
1344
- const filePath = path.join(memoriesDir, f);
1345
- // #551: skip files that were only touched by their own promotion this
1346
- // cohort — they have no settled merge/contradiction candidates yet.
1347
- if (promotedSinceConsolidate.has(path.resolve(filePath)))
1348
- return false;
1349
- try {
1350
- return fs.statSync(filePath).mtime.toISOString() > lastConsolidateTs;
1351
- }
1352
- catch {
1353
- return false;
1354
- }
1355
- });
1356
- }
1357
- catch {
1358
- return false;
1359
- }
1360
- })();
1361
- const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
1362
- // Profile gate: if profile explicitly disables consolidate, skip the entire pass.
1363
- const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
1364
- // #553 minPoolSize guard: skip consolidation when the eligible memory pool is
1365
- // below a minimum size, rather than spending an LLM pass on a handful of
1366
- // memories. This is an INDEPENDENT skip condition from #551's mtime pool-delta
1367
- // gate — either can skip. Default 500; `minPoolSize: 0` disables the guard.
1368
- // Evaluated against the eligible-pool count BEFORE entering the LLM loop so a
1369
- // skip costs ZERO LLM calls.
1370
- const CONSOLIDATE_DEFAULT_MIN_POOL_SIZE = 500;
1371
- const configuredMinPoolSize = improveProfile?.processes?.consolidate?.minPoolSize;
1372
- const minPoolSize = typeof configuredMinPoolSize === "number" ? configuredMinPoolSize : CONSOLIDATE_DEFAULT_MIN_POOL_SIZE;
1373
- const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
1374
- // volumeTriggered means the pool already exceeds the volume threshold (100),
1375
- // so a force-triggered run never trips the pool-size guard. The guard only
1376
- // engages when minPoolSize > 0 and the eligible pool is strictly below it.
1377
- const poolBelowMinSize = !volumeTriggered && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
1378
- let consolidation = {
1379
- schemaVersion: 1,
1380
- ok: true,
1381
- shape: "consolidate-result",
1382
- dryRun: false,
1383
- previewOnly: false,
1384
- target: "",
1385
- processed: 0,
1386
- merged: 0,
1387
- deleted: 0,
1388
- promoted: [],
1389
- contradicted: 0,
1390
- warnings: [],
1391
- durationMs: 0,
1392
- };
1393
- let gateAutoAcceptedCount = 0;
1394
- let gateAutoAcceptFailedCount = 0;
1395
- const consolidateGateCfg = makeGateConfig("consolidate", {
1396
- globalThreshold: options.autoAccept,
1397
- dryRun: options.dryRun ?? false,
1398
- stashDir: primaryStashDir,
1399
- config: consolidationConfig,
1400
- eventsCtx,
1401
- }, { minimumThreshold: 95 });
1402
- if (consolidateDisabledByProfile) {
1403
- info("[improve] consolidation skipped (disabled by improve profile)");
1404
- }
1405
- else if (poolBelowMinSize) {
1406
- // #553: eligible pool below the configured minimum — skip with zero LLM
1407
- // calls. Reuse the #551 `improve_skipped` emission path so health surfaces
1408
- // it via the dynamic skipReasons aggregation under `pool_below_min_size`.
1409
- appendEvent({
1410
- eventType: "improve_skipped",
1411
- ref: "memory:_consolidation",
1412
- metadata: {
1413
- reason: "pool_below_min_size",
1414
- poolSize: eligiblePoolSize,
1415
- minPoolSize,
1416
- },
1417
- }, eventsCtx);
1418
- info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
1419
- }
1420
- else if (!consolidationOnCooldown) {
1421
- consolidation = await withLlmStage("consolidate", () => akmConsolidate({
1422
- ...options.consolidateOptions,
1423
- config: consolidationConfig,
1424
- stashDir: options.stashDir,
1425
- autoTriggered: volumeTriggered,
1426
- // Tie consolidate proposals back to this improve invocation so
1427
- // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
1428
- sourceRun: `consolidate-${Date.now()}`,
1429
- // Pass profile-configured options. incrementalSince narrows the pool to
1430
- // recently-changed memories + graph neighbours — use this for frequent
1431
- // passes (quick-shredder). Leave absent in the nightly default profile for
1432
- // a full-pool sweep that catches stale-but-unmerged duplicates.
1433
- incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
1434
- limit: improveProfile?.processes?.consolidate?.limit,
1435
- neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
1436
- maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
1437
- // Honor profile.autoAccept (already merged into options.autoAccept at the
1438
- // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
1439
- // is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
1440
- // (which maps to undefined) from disabling consolidation auto-accept.
1441
- // options.consolidateOptions.autoAccept (if explicitly provided by caller)
1442
- // still wins because the spread above runs first.
1443
- autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
1444
- }));
1445
- {
1446
- const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
1447
- try {
1448
- if (!primaryStashDir)
1449
- return { proposalId, confidence: undefined };
1450
- const proposal = getProposal(primaryStashDir, proposalId);
1451
- return { proposalId, confidence: proposal.confidence };
1452
- }
1453
- catch {
1454
- return { proposalId, confidence: undefined };
1455
- }
1456
- }), consolidateGateCfg);
1457
- gateAutoAcceptedCount += consolidateGr.promoted.length;
1458
- gateAutoAcceptFailedCount += consolidateGr.failed.length;
1459
- }
1460
- if (consolidation.processed > 0) {
1461
- appendEvent({
1462
- eventType: "consolidate_completed",
1463
- ref: "memory:_consolidation",
1464
- metadata: {
1465
- processed: consolidation.processed,
1466
- merged: consolidation.merged,
1467
- deleted: consolidation.deleted,
1468
- contradicted: consolidation.contradicted,
1469
- failedChunks: consolidation.failedChunks ?? 0,
1470
- durationMs: consolidation.durationMs,
1471
- },
1472
- }, eventsCtx);
1473
- }
1474
- }
1475
- else {
1476
- appendEvent({
1477
- eventType: "improve_skipped",
1478
- ref: "memory:_consolidation",
1479
- metadata: {
1480
- reason: "consolidation_no_memory_updates",
1481
- lastEventTs: lastConsolidation?.ts ?? null,
1482
- },
1483
- }, eventsCtx);
1484
- info("[improve] consolidation skipped (no memory updates since last run)");
1485
- }
1486
- // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
1487
- const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
1488
- return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
1489
- }
1490
- async function runImprovePreparationStage(args) {
1491
- const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
1492
- const actions = [];
1493
- const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
1494
- // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
1495
- let memoryIndexHealth;
1496
- if (primaryStashDir) {
1497
- const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
1498
- if (fs.existsSync(memoryMdPath)) {
1499
- try {
1500
- const lines = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
1501
- const overBudget = lines >= 180;
1502
- memoryIndexHealth = { lineCount: lines, overBudget };
1503
- if (overBudget) {
1504
- cleanupWarnings.push(`MEMORY.md has ${lines} lines (budget: 200). Consolidation strongly recommended.`);
1505
- }
1506
- }
1507
- catch {
1508
- // best-effort
1509
- }
1510
- }
1511
- }
1512
- // Phase 0.3 — memory consolidation pass (#551).
1513
- //
1514
- // Consolidation runs BEFORE the session-extract pass. This is the structural
1515
- // half of the #551 fix: extract auto-accept writes brand-new memory .md files
1516
- // on every run, which previously made the consolidation pool-delta gate fire
1517
- // unconditionally (any new file => "memory updated since last consolidate").
1518
- // By running consolidation first, the gate and akmConsolidate only ever see
1519
- // memories that existed at the start of the run — current-run extract
1520
- // promotions are not on disk yet. The complementary smarter-gate logic
1521
- // (excluding adjacent-run promotions) lives in `runConsolidationPass`.
1522
- const consolidationPass = await runConsolidationPass({
1523
- options,
1524
- primaryStashDir,
1525
- memorySummary,
1526
- improveProfile,
1527
- eventsCtx,
1528
- });
1529
- // Phase 0.4 — session-extract pass.
1530
- //
1531
- // Reads native session files (claude-code JSONL, opencode storage tree)
1532
- // through the SessionLogHarness registry, pre-filters noise, and asks a
1533
- // bounded in-tree LLM to produce candidate memory/lesson/knowledge
1534
- // proposals for content the agent did NOT preserve via inline `akm remember`
1535
- // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
1536
- // hook with an on-demand pull pipeline.
1537
- //
1538
- // Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
1539
- // (#593: the gate respects the resolved improve profile, not just the
1540
- // hardcoded `default` profile path the legacy feature flag reads).
1541
- // Each available harness gets one call with the default --since window;
1542
- // already-seen sessions (tracked in state.db.extract_sessions_seen) are
1543
- // skipped automatically so re-runs don't burn LLM calls on unchanged data.
1544
- //
1545
- // Failures are non-fatal — one harness throwing doesn't abort improve.
1546
- // The extract envelope's own `warnings` field surfaces what went wrong.
1547
- let extractResults;
1548
- // Seed the preparation-stage gate counters with consolidation's auto-accept
1549
- // gate results (#551: consolidation now runs in this stage), then accumulate
1550
- // extract's gate results on top.
1551
- let gateAutoAcceptedCount = consolidationPass.gateAutoAcceptedCount;
1552
- let gateAutoAcceptFailedCount = consolidationPass.gateAutoAcceptFailedCount;
1553
- const extractConfig = options.config ?? loadConfig();
1554
- const extractGateCfg = makeGateConfig("extract", {
1555
- globalThreshold: options.autoAccept,
1556
- dryRun: options.dryRun ?? false,
1557
- stashDir: primaryStashDir,
1558
- config: extractConfig,
1559
- eventsCtx,
1560
- });
1561
- // #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
1562
- // already done upstream; here we elide every akmExtract/processSession call)
1563
- // when the NEW (unseen, in-window) candidate-session pool is below a minimum.
1564
- // 22% of improve runs produce zero memory-inference writes because extract
1565
- // finds no new sessions, yet still burns the full extract pipeline. Default 0
1566
- // (disabled) preserves existing always-run behaviour; only opted-in profiles
1567
- // (e.g. `frequent`) set it. Evaluated BEFORE any LLM call so a skip costs zero
1568
- // LLM work AND writes nothing — which also means no extract auto-accept bumps
1569
- // memory mtimes, so a skipped extract never flags work for the NEXT run's
1570
- // consolidation mtime-gate (the downstream trigger #554 asks us to suppress).
1571
- const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
1572
- const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
1573
- const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
1574
- // #593: gate on BOTH the legacy feature flag (which only reads
1575
- // `profiles.improve.default.processes.extract.enabled` — kept for back-compat
1576
- // with users who disable extract via the default-profile path) AND the active
1577
- // resolved profile. Without the second check a non-default profile setting
1578
- // `extract.enabled: false` (e.g. the built-in `quick`) was silently ignored
1579
- // and extract ran on every improve call regardless.
1580
- if (isLlmFeatureEnabled(extractConfig, "session_extraction") && resolveProcessEnabled("extract", improveProfile)) {
1581
- const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
1582
- // The guard engages only when minNewSessions > 0; 0 disables it entirely.
1583
- let belowMinNewSessions = false;
1584
- if (minNewSessions > 0 && availableHarnesses.length > 0) {
1585
- const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
1586
- const newCandidateCount = countFn(extractConfig, {
1587
- ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1588
- // C2: pin the candidate-count state.db open to the boundary-resolved path.
1589
- ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1590
- });
1591
- if (newCandidateCount < minNewSessions) {
1592
- belowMinNewSessions = true;
1593
- // Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
1594
- // skipReasons aggregation surfaces this under `below_min_new_sessions`.
1595
- appendEvent({
1596
- eventType: "improve_skipped",
1597
- ref: "memory:_extract",
1598
- metadata: {
1599
- reason: "below_min_new_sessions",
1600
- newSessions: newCandidateCount,
1601
- minNewSessions,
1602
- },
1603
- }, eventsCtx);
1604
- info(`[improve] extract skipped (new sessions ${newCandidateCount} < minNewSessions ${minNewSessions})`);
1605
- }
1606
- }
1607
- if (!belowMinNewSessions && availableHarnesses.length > 0) {
1608
- extractResults = [];
1609
- for (const h of availableHarnesses) {
1610
- try {
1611
- const result = await withLlmStage("session-extraction", () => akmExtract({
1612
- type: h.name,
1613
- ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
1614
- config: extractConfig,
1615
- dryRun: options.dryRun ?? false,
1616
- ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1617
- // C2: pin extract's skip-tracking state.db open to the boundary path.
1618
- ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1619
- }));
1620
- extractResults.push(result);
1621
- {
1622
- const gr = await runAutoAcceptGate(primaryStashDir
1623
- ? result.proposals.map((proposalId) => {
1624
- const proposal = getProposal(primaryStashDir, proposalId);
1625
- return { proposalId, confidence: resolveExtractConfidence(proposal) };
1626
- })
1627
- : [], extractGateCfg);
1628
- gateAutoAcceptedCount += gr.promoted.length;
1629
- gateAutoAcceptFailedCount += gr.failed.length;
1630
- }
1631
- }
1632
- catch (err) {
1633
- const msg = err instanceof Error ? err.message : String(err);
1634
- cleanupWarnings.push(`extract(${h.name}) failed: ${msg}`);
1635
- }
1636
- }
1637
- if (extractResults.length === 0) {
1638
- // All harnesses threw — clear so the envelope's `extract` field is
1639
- // absent rather than misleadingly empty.
1640
- extractResults = undefined;
1641
- }
1642
- }
1643
- }
1644
- // Backlog drain: gate any pending extract proposals that weren't created in
1645
- // this run (i.e. pre-date the gate or were produced by a run that timed out
1646
- // before the gate fired). Without this, eligible proposals accumulate
1647
- // indefinitely — the fresh-gate only covers the current run's output.
1648
- if (primaryStashDir && !options.dryRun && options.autoAccept !== undefined) {
1649
- const freshIds = new Set((extractResults ?? []).flatMap((r) => r.proposals));
1650
- const backlog = listProposals(primaryStashDir, { status: "pending" }).filter((p) => p.source === "extract" && !freshIds.has(p.id));
1651
- if (backlog.length > 0) {
1652
- const backlogCandidates = backlog.map((p) => ({
1653
- proposalId: p.id,
1654
- confidence: resolveExtractConfidence(p),
1655
- }));
1656
- const backlogGr = await runAutoAcceptGate(backlogCandidates, extractGateCfg);
1657
- gateAutoAcceptedCount += backlogGr.promoted.length;
1658
- gateAutoAcceptFailedCount += backlogGr.failed.length;
1659
- }
1660
- }
1661
- // eligibleCount = raw pre-filter count (before cooldown/signal/cleanup filters).
1662
- // improve_completed.plannedRefs = post-filter count of refs that actually entered the loop.
1663
- appendEvent({
1664
- eventType: "improve_invoked",
1665
- ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
1666
- metadata: { scope, dryRun: options.dryRun ?? false, eligibleCount: plannedRefs.length },
1667
- }, eventsCtx);
1668
- // ensureIndex now runs in akmImprove() BEFORE collectEligibleRefs so the
1669
- // eligible-ref query sees a populated `entries` table on the very first
1670
- // pass after a DB version upgrade (#339). Any failure messages from that
1671
- // earlier call were threaded in via args.initialCleanupWarnings.
1672
- let appliedCleanup;
1673
- try {
1674
- appliedCleanup =
1675
- primaryStashDir && memoryCleanupPlan ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan) : undefined;
1676
- }
1677
- catch (err) {
1678
- cleanupWarnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
1679
- }
1680
- const archivedRefs = appliedCleanup?.archived.map((record) => record.ref) ?? [];
1681
- const removed = new Set(archivedRefs);
1682
- const postCleanupRefs = archivedRefs.length === 0 ? plannedRefs : plannedRefs.filter((r) => !removed.has(r.ref));
1683
- // ── Phase 1: validation pass + schema repair (run on full postCleanupRefs) ──
1684
- // Identifies refs whose on-disk asset has structural problems. Validation
1685
- // failures are excluded from every downstream bucket. Run early so the
1686
- // cooldown partition operates on a clean set.
1687
- if (appliedCleanup) {
1688
- for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
1689
- const archived = appliedCleanup.archived.find((record) => record.ref === candidate.ref);
1690
- if (!archived)
1691
- continue;
1692
- actions.push({
1693
- ref: candidate.ref,
1694
- mode: "memory-prune",
1695
- result: { ok: true, pruned: true, reason: candidate.reason },
1696
- });
1697
- }
1698
- if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
1699
- try {
1700
- await reindexFn({ stashDir: primaryStashDir });
1701
- }
1702
- catch (err) {
1703
- cleanupWarnings.push(`reindex after cleanup failed: ${err instanceof Error ? err.message : String(err)}`);
1704
- }
1705
- }
1706
- }
1707
- const validationFailures = [];
1708
- for (const candidate of postCleanupRefs) {
1709
- try {
1710
- // #591: use the path pre-resolved at planning time when it is still on
1711
- // disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
1712
- // stash. Fall back to findAssetFilePath only for refs that bypassed
1713
- // collectEligibleRefs' index scan or whose file moved since planning.
1714
- const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
1715
- ? candidate.filePath
1716
- : await findAssetFilePath(candidate.ref, options.stashDir);
1717
- if (!filePath) {
1718
- validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
1719
- continue;
1720
- }
1721
- if (path.extname(filePath).toLowerCase() !== ".md") {
1722
- continue;
1723
- }
1724
- if (isLessonCandidate(candidate.ref)) {
1725
- const raw = fs.readFileSync(filePath, "utf8");
1726
- const fm = parseFrontmatter(raw).data;
1727
- if (!fm.description)
1728
- validationFailures.push({ ref: candidate.ref, reason: "missing description" });
1729
- }
1730
- }
1731
- catch (e) {
1732
- validationFailures.push({ ref: candidate.ref, reason: String(e) });
1733
- }
1734
- }
1735
- if (validationFailures.length > 0) {
1736
- info(`[improve] ${validationFailures.length} assets have validation issues (will attempt schema repair):`);
1737
- for (const f of validationFailures)
1738
- info(` ${f.ref}: ${f.reason}`);
1739
- }
1740
- let schemaRepairs = [];
1741
- let repairedRefs = new Set();
1742
- // Schema repair pass: attempt to fix validation failures via LLM before skipping.
1743
- if (validationFailures.length > 0 && options.repairValidationFailures !== false) {
1744
- const baseConfigForRepair = options.config ?? loadConfig();
1745
- const llmCfg = getDefaultLlmConfig(baseConfigForRepair);
1746
- if (llmCfg) {
1747
- const result = await runSchemaRepairPass(validationFailures, {
1748
- startMs,
1749
- budgetMs,
1750
- llmConfig: llmCfg,
1751
- stashDir: options.stashDir,
1752
- findFilePath: findAssetFilePath,
1753
- isLessonCandidateFn: isLessonCandidate,
1754
- });
1755
- schemaRepairs = result.repairs;
1756
- repairedRefs = result.repairedRefs;
1757
- }
1758
- }
1759
- const validationFailureRefs = new Set(validationFailures.filter((f) => !repairedRefs.has(f.ref)).map((f) => f.ref));
1760
- if (repairedRefs.size > 0) {
1761
- info(`[improve] schema repair fixed ${repairedRefs.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
1762
- }
1763
- // Phase 0.5 — structural hygiene pass
1764
- let lintSummary;
1765
- if (primaryStashDir) {
1766
- try {
1767
- const lintResult = akmLint({ fix: true, dir: primaryStashDir });
1768
- lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
1769
- }
1770
- catch {
1771
- // lint is best-effort; never block improve
1772
- }
1773
- }
1774
- // O-5 / #378: Per-originator rolling error windows.
1775
- // Reflexion (arXiv:2303.11366) warns that cross-task verbal critique
1776
- // contamination degrades below single-shot baseline. Each originator key
1777
- // ("schema-repair", "reflect") maintains its own rolling window so that
1778
- // schema-repair failures are not injected as avoidPatterns into reflect calls.
1779
- const recentErrors = {};
1780
- const RECENT_ERRORS_CAP = 3;
1781
- // Helper: push an error onto an originator's rolling window.
1782
- function pushRecentError(originator, msg) {
1783
- if (!recentErrors[originator])
1784
- recentErrors[originator] = [];
1785
- recentErrors[originator].push(msg);
1786
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
1787
- recentErrors[originator].shift();
1788
- }
1789
- // Seed schema-repair originator window from any schema-repair errors.
1790
- for (const repair of schemaRepairs) {
1791
- if (repair.outcome === "error") {
1792
- const errMsg = repair.error ?? `schema repair error: ${repair.reason}`;
1793
- pushRecentError("schema-repair", errMsg);
1794
- }
1795
- }
1796
- // ── Phase 2: signal-delta eligibility sets built EARLY ────────────────────
1797
- // 0.8.0 replaces the flat time-based cooldowns (which produced synchronised
1798
- // waves whenever many refs cooled at the same instant — see the 2026-05-26
1799
- // 54-ref simultaneous-reflect incident) with a *signal-delta* gate:
1800
- //
1801
- // reflectEligible(ref) ≡ latestFeedbackTs(ref) > lastReflectProposalTs(ref)
1802
- // distillEligible(ref) ≡ latestFeedbackTs(ref) > lastDistillProposalTs(ref)
1803
- //
1804
- // i.e. a ref is re-eligible iff new feedback has landed since the last
1805
- // proposal was generated for it. Stable content with no new signal stays
1806
- // out of the queue regardless of clock time; a sudden burst of feedback
1807
- // surfaces only the refs that the burst actually touches.
1808
- //
1809
- // The 30-day FEEDBACK_SIGNAL_WINDOW_DAYS bound still applies — only feedback
1810
- // events newer than that count as "current signal". Ancient one-off
1811
- // negatives don't permanently lock a ref into every run.
1812
- //
1813
- // High-retrieval refs (P0-A path) use a simpler "eligible once" rule: a
1814
- // ref with no feedback signal but retrievalCount ≥ threshold is eligible
1815
- // exactly once (no prior reflect proposal). Subsequent re-eligibility for
1816
- // those refs requires either a new feedback event (then the normal
1817
- // signal-delta gate applies) or human action. Documented limitation: this
1818
- // path does not re-fire on retrieval-count growth alone in 0.8.0; storing
1819
- // the retrieval count in proposal metadata for proper delta-tracking is
1820
- // captured as future work.
1821
- const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
1822
- const feedbackSinceCutoff = new Date(Date.now() - daysToMs(FEEDBACK_SIGNAL_WINDOW_DAYS)).toISOString();
1823
- // Build the three timestamp maps once across the entire postCleanupRefs set.
1824
- // Per-ref queries would be N+1 and the planner is already the hottest path
1825
- // in `akm improve`.
1826
- const candidateRefs = postCleanupRefs.filter((r) => !validationFailureRefs.has(r.ref)).map((r) => r.ref);
1827
- const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff);
1828
- const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect");
1829
- const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill");
1830
- // Refs the distill signal-delta gate rejected at planning time. The main
1831
- // loop reads this to skip distill for these refs without re-checking
1832
- // eligibility per iteration.
1833
- const distillCooledRefs = new Set();
1834
- const preCooldownCount = postCleanupRefs.length;
1835
- // ── Phase 3: partition postCleanupRefs by signal-delta eligibility ────────
1836
- // Three buckets (validation failures are excluded entirely):
1837
- // eligibleRefs — reflect signal-delta passes (full reflect+distill
1838
- // loop path; distill guard remains in the loop for
1839
- // refs that fail the distill signal-delta gate).
1840
- // distillOnlyRefs — reflect blocked but distill signal-delta passes
1841
- // AND ref is a distill candidate.
1842
- // noFeedbackPool — neither signal-delta gate passes *and* the ref has
1843
- // no recent feedback signal at all. These are NOT
1844
- // skipped here: they are handed to the high-retrieval
1845
- // fallback (P0-A) below so frequently-retrieved but
1846
- // never-rated assets can still be improved. Only refs
1847
- // that P0-A declines are ultimately fully skipped.
1848
- // fullySkippedCount — has stale feedback but no signal delta → genuine
1849
- // skip (counted, aggregated event emitted post-loop),
1850
- // excluded from sort.
1851
- const eligibleRefs = [];
1852
- const distillOnlyRefs = [];
1853
- // Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
1854
- const noFeedbackPool = [];
1855
- let fullySkippedCount = 0;
1856
- // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
1857
- const scopeRefBypass = scope.mode === "ref";
1858
- for (const r of postCleanupRefs) {
1859
- if (validationFailureRefs.has(r.ref))
1860
- continue;
1861
- if (scopeRefBypass) {
1862
- eligibleRefs.push(r);
1863
- continue;
1864
- }
1865
- const reflectOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastReflectProposalTs);
1866
- const distillOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastDistillProposalTs);
1867
- const isDistillCandidate = isDistillCandidateRef(r.ref, options.stashDir);
1868
- if (reflectOk) {
1869
- if (!distillOk && isDistillCandidate) {
1870
- // Reflect passes the gate, distill does not — emit the synthetic
1871
- // distill-skipped action and event up-front so the in-loop guard
1872
- // does not have to re-derive eligibility.
1873
- distillCooledRefs.add(r.ref);
1874
- actions.push({ ref: r.ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
1875
- appendEvent({
1876
- eventType: "improve_skipped",
1877
- ref: r.ref,
1878
- metadata: { reason: "distill_no_new_signal" },
1879
- }, eventsCtx);
1880
- }
1881
- else if (!distillOk) {
1882
- // Not a distill candidate AND distill gate doesn't pass — just mark
1883
- // distillCooled so the loop's distill section is a no-op.
1884
- distillCooledRefs.add(r.ref);
1885
- }
1886
- eligibleRefs.push(r);
1887
- }
1888
- else if (distillOk && isDistillCandidate) {
1889
- // Reflect blocked but distill passes → distill-only bucket.
1890
- distillOnlyRefs.push(r);
1891
- }
1892
- else if (!latestFeedbackTs.has(r.ref)) {
1893
- // Neither signal-delta gate passes AND there is no recent feedback signal
1894
- // at all. Rather than skip outright, defer to the high-retrieval fallback
1895
- // (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
1896
- // what that path is meant to rescue. Refs P0-A declines are skipped there.
1897
- noFeedbackPool.push(r);
1898
- }
1899
- else {
1900
- // Has feedback on record but no signal delta since the last proposal —
1901
- // genuinely fully skipped. Counted here; a single aggregated
1902
- // improve_skipped event is emitted after the loop (mirrors
1903
- // profile_filtered_all_passes) instead of one event per ref.
1904
- fullySkippedCount++;
1905
- actions.push({
1906
- ref: r.ref,
1907
- mode: "distill-skipped",
1908
- result: { ok: true, reason: "no new signal since last proposal" },
1909
- });
1910
- }
1911
- }
1912
- // Emit ONE aggregated skip event for the fully-skipped bucket rather than one
1913
- // improve_skipped event per ref (#592 pattern, mirrors
1914
- // profile_filtered_all_passes above). The per-ref loop previously produced
1915
- // ~11K state.db writes per run on a large stash, the dominant contributor to
1916
- // 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
1917
- // run summary; no downstream consumer needs a per-ref DB audit trail (health's
1918
- // skip histogram reads the `no_new_signal` counter from the count field).
1919
- if (fullySkippedCount > 0) {
1920
- appendEvent({
1921
- eventType: "improve_skipped",
1922
- ref: undefined,
1923
- metadata: {
1924
- reason: "no_new_signal",
1925
- count: fullySkippedCount,
1926
- },
1927
- }, eventsCtx);
1928
- }
1929
- // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
1930
- // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
1931
- // deferred noFeedbackPool that may be rescued by the high-retrieval fallback
1932
- // (P0-A). The fully-skipped bucket has already been routed and its aggregated
1933
- // event emitted; we deliberately avoid spending DB/CPU on refs that the
1934
- // signal-delta gate rejected with feedback already on record.
1935
- const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
1936
- // Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
1937
- // partition above could not place these in a reflect/distill bucket, but they
1938
- // may still qualify if they have been retrieved often enough. Two disjoint
1939
- // sources feed this set:
1940
- // 1. noFeedbackPool — refs with no recent feedback that the partition loop
1941
- // deliberately deferred here (otherwise they would never reach P0-A).
1942
- // 2. processableRefs entries that turn out to carry no recent feedback
1943
- // *signal* once feedbackSummary is computed below.
1944
- // (1) is added here; (2) is folded in after feedbackSummary is built.
1945
- // Gap 6: only surface feedback signals from the last 30 days so that
1946
- // ancient one-off feedback events don't permanently lock an asset into
1947
- // every improve run. Assets with only stale signals fall through to the
1948
- // high-retrieval path (P0-A) or are skipped until new signals arrive.
1949
- // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
1950
- // Phase 2 above for the signal-delta gate; we reuse them here.)
1951
- // Pre-compute feedback summary per ref in a single pass so we don't issue
1952
- // two readEvents({type:"feedback", ref}) per asset (one for signal filtering,
1953
- // one for ratio computation).
1954
- // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
1955
- // ratios are available for any noFeedbackPool ref that P0-A rescues below.
1956
- const feedbackSummary = new Map();
1957
- for (const candidate of [...processableRefs, ...noFeedbackPool]) {
1958
- if (feedbackSummary.has(candidate.ref))
1959
- continue;
1960
- const { events } = readEvents({ type: "feedback", ref: candidate.ref });
1961
- let hasSignal = false;
1962
- let positive = 0;
1963
- let negative = 0;
1964
- for (const e of events) {
1965
- if (!hasSignal &&
1966
- (e.ts ?? "") >= feedbackSinceCutoff &&
1967
- e.metadata !== undefined &&
1968
- (typeof e.metadata.signal === "string" || typeof e.metadata.note === "string")) {
1969
- hasSignal = true;
1970
- }
1971
- if (e.metadata?.signal === "positive")
1972
- positive++;
1973
- else if (e.metadata?.signal === "negative")
1974
- negative++;
1975
- }
1976
- feedbackSummary.set(candidate.ref, { hasSignal, positive, negative });
1977
- }
1978
- const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
1979
- // P0-A: also surface zero-feedback assets that have been retrieved many times.
1980
- const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
1981
- const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
1982
- // Zero-feedback candidates for P0-A: processableRefs without a recent signal,
1983
- // plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
1984
- // disjoint by construction, but guard against overlap defensively).
1985
- const noFeedbackSeen = new Set();
1986
- const noFeedbackCandidates = [];
1987
- for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
1988
- if (noFeedbackSeen.has(r.ref))
1989
- continue;
1990
- noFeedbackSeen.add(r.ref);
1991
- noFeedbackCandidates.push(r);
1992
- }
1993
- let highRetrievalRefs = [];
1994
- // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
1995
- // proactive-maintenance selector below can reuse them without a second DB pass.
1996
- let retrievalCounts = new Map();
1997
- let dbForRetrieval;
1998
- try {
1999
- dbForRetrieval = openExistingDatabase();
2000
- const showEventCount = countUsageEventsByType(dbForRetrieval, "show");
2001
- if (showEventCount === 0) {
2002
- warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
2003
- }
2004
- retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
2005
- // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
2006
- // ref qualifies exactly once — when it has actually been retrieved
2007
- // (retrievalCount ≥ 1) AND retrievalCount ≥ threshold AND no prior reflect
2008
- // proposal exists for it. Once a reflect proposal is on record, subsequent
2009
- // re-eligibility requires explicit feedback (which flows through the normal
2010
- // signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
2011
- // from rescuing genuinely never-retrieved assets — the fallback is for
2012
- // *retrieved* assets, not silent ones. Tracking growth in retrieval count
2013
- // would require persisting the count in proposal metadata; deferred to a
2014
- // follow-up.
2015
- highRetrievalRefs = noFeedbackCandidates.filter((r) => {
2016
- const count = retrievalCounts.get(r.ref) ?? 0;
2017
- return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
2018
- });
2019
- }
2020
- catch (err) {
2021
- rethrowIfTestIsolationError(err);
2022
- // best-effort: if DB unavailable, highRetrievalRefs stays empty
2023
- }
2024
- finally {
2025
- if (dbForRetrieval)
2026
- closeDatabase(dbForRetrieval);
2027
- }
2028
- // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
2029
- // The signal-delta gate and P0-A only surface assets with fresh feedback or a
2030
- // raw-retrieval spike. Neither revisits a stable, high-value asset on a
2031
- // schedule, so on a quiet stash useful assets drift stale and are never
2032
- // refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
2033
- // and the run is whole-stash / type scope, this selector ranks the eligible
2034
- // population by a composite maintenance priority, gates on staleness ("due"),
2035
- // bounds to top-N, and folds the winners into the SAME candidate set the other
2036
- // two sources feed — so they flow through the existing #580 empty-diff /
2037
- // cosmetic suppression and additive-distill gates. It adds no new mutation
2038
- // logic of its own. The due gate doubles as the rotation cooldown: a freshly
2039
- // reflected asset is excluded until it ages back past `dueDays`, so successive
2040
- // runs rotate through the due pool rather than re-selecting the same heads.
2041
- let proactiveRefs = [];
2042
- let proactiveMaintenanceSummary;
2043
- const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
2044
- if (proactiveEnabled) {
2045
- const pmCfg = improveProfile.processes?.proactiveMaintenance;
2046
- const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
2047
- const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
2048
- const importanceWeights = pmCfg?.importanceWeights;
2049
- // Candidate population: the zero-feedback / non-signal pool — exactly the
2050
- // assets the other two sources would NOT pick this run. Exclude any P0-A
2051
- // rescued this run so we never double-select the same ref.
2052
- const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
2053
- const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
2054
- const selection = selectProactiveMaintenanceRefs({
2055
- candidates: pmCandidates,
2056
- lastReflectTs: lastReflectProposalTs,
2057
- lastDistillTs: lastDistillProposalTs,
2058
- retrievalCounts,
2059
- sizeBytesOf: (r) => {
2060
- const fp = r.filePath;
2061
- if (!fp)
2062
- return undefined;
2063
- try {
2064
- return fs.statSync(fp).size;
2065
- }
2066
- catch {
2067
- return undefined;
2068
- }
2069
- },
2070
- dueDays,
2071
- maxPerRun,
2072
- importanceWeights,
2073
- });
2074
- proactiveRefs = selection.selected;
2075
- proactiveMaintenanceSummary = {
2076
- selected: selection.selected.length,
2077
- dueTotal: selection.dueTotal,
2078
- neverReflected: selection.neverReflected,
2079
- };
2080
- // Aggregated observability event (never per-ref — avoids the event flood the
2081
- // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
2082
- appendEvent({
2083
- eventType: "proactive_selected",
2084
- ref: undefined,
2085
- metadata: {
2086
- count: selection.selected.length,
2087
- dueTotal: selection.dueTotal,
2088
- neverReflected: selection.neverReflected,
2089
- },
2090
- }, eventsCtx);
2091
- if (selection.selected.length > 0) {
2092
- info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
2093
- `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
2094
- }
2095
- }
2096
- // Record an in-memory skip action for every zero-feedback ref that the
2097
- // partition loop deferred to P0-A but P0-A then declined (retrievalCount below
2098
- // threshold, or a prior reflect proposal already on record). These never make
2099
- // it into mergedRefs, so without this they would silently vanish from the run
2100
- // summary. No DB event is written here — these refs carry no signal at all, so
2101
- // there is nothing for the skip histogram to aggregate; the action log alone
2102
- // preserves the per-ref audit trail (mirrors the fully-skipped action above).
2103
- const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
2104
- for (const r of noFeedbackPool) {
2105
- if (rescuedSet.has(r.ref))
2106
- continue;
2107
- actions.push({
2108
- ref: r.ref,
2109
- mode: "distill-skipped",
2110
- result: { ok: true, reason: "no new signal since last proposal" },
2111
- });
2112
- }
2113
- // If the user explicitly scoped to a single ref, always act on it —
2114
- // skip the signal/retrieval filter entirely. The filter exists to avoid
2115
- // noisy "improve everything" runs; it should not gate an intentional
2116
- // per-ref invocation where the user's explicit choice is the signal.
2117
- //
2118
- // For type/all scope: only process refs with usage signals (recent feedback
2119
- // or sufficient retrievals). A stash with no signals has 0 eligible refs —
2120
- // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
2121
- // to bring them into the eligible pool.
2122
- // Layer-2 proactive refs join the eligible set alongside feedback-signal and
2123
- // high-retrieval (P0-A) refs. The three sources are disjoint by construction
2124
- // (proactive draws from noFeedbackCandidates with the P0-A picks removed), but
2125
- // dedupe defensively so a ref can never enter the loop twice. `requireFeedbackSignal`
2126
- // still suppresses both fallback sources for callers that want feedback-only runs.
2127
- const signalAndRetrievalRefs = dedupeRefs([...signalFiltered, ...highRetrievalRefs, ...proactiveRefs]);
2128
- const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2129
- // ── Attribution tagging: stamp each ref with the eligibility lane that
2130
- // selected it ──────────────────────────────────────────────────────────────
2131
- // Every reflect/distill proposal must record WHICH lane chose its source asset
2132
- // so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
2133
- // (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
2134
- // lane map here — the one place all four lanes are known — and stamp it onto
2135
- // each ImproveEligibleRef object. Because the ref objects are shared by
2136
- // reference across buckets, the stamp travels with the ref through the sort,
2137
- // disk-check, and loop stages down to the reflect/distill event emit sites and
2138
- // createProposal calls. See EligibilitySource for the lane vocabulary.
2139
- //
2140
- // Precedence (prefer the most specific reactive signal):
2141
- // scope > signal-delta > high-retrieval > proactive
2142
- // A ref with real feedback is attributed to feedback even if it was also due
2143
- // for proactive maintenance. We apply lanes weakest-first so the strongest
2144
- // overwrites; the explicit --scope <ref> bypass wins outright (user intent).
2145
- const eligibilitySourceByRef = new Map();
2146
- for (const r of proactiveRefs)
2147
- eligibilitySourceByRef.set(r.ref, "proactive");
2148
- for (const r of highRetrievalRefs)
2149
- eligibilitySourceByRef.set(r.ref, "high-retrieval");
2150
- for (const r of signalFiltered)
2151
- eligibilitySourceByRef.set(r.ref, "signal-delta");
2152
- if (scope.mode === "ref") {
2153
- // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
2154
- // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
2155
- for (const r of processableRefs)
2156
- eligibilitySourceByRef.set(r.ref, "scope");
2157
- }
2158
- for (const r of mergedRefs) {
2159
- // "unknown" is a genuine fallback, never a silent alias for signal-delta:
2160
- // only refs we truly cannot attribute land here (none in practice, since
2161
- // mergedRefs is always a subset of the four lanes above).
2162
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2163
- }
2164
- const utilityMap = buildUtilityMap(mergedRefs);
2165
- // Load feedback ratio per ref from the pre-computed summary (no extra DB pass).
2166
- const feedbackRatios = new Map();
2167
- for (const ref of mergedRefs) {
2168
- const summary = feedbackSummary.get(ref.ref);
2169
- const positive = summary?.positive ?? 0;
2170
- const negative = summary?.negative ?? 0;
2171
- const total = positive + negative;
2172
- // ratio = negative proportion (high = needs more improvement)
2173
- feedbackRatios.set(ref.ref, total > 0 ? negative / total : 0);
2174
- }
2175
- // Sort: combine utility (desc) with feedback negativity (desc) — high-negative assets rank higher
2176
- const sorted = [...mergedRefs].sort((a, b) => {
2177
- const utilA = utilityMap.get(a.ref) ?? 0;
2178
- const utilB = utilityMap.get(b.ref) ?? 0;
2179
- const ratioA = feedbackRatios.get(a.ref) ?? 0;
2180
- const ratioB = feedbackRatios.get(b.ref) ?? 0;
2181
- // Combined score: 70% utility, 30% negative ratio
2182
- const scoreA = utilA * 0.7 + ratioA * 0.3;
2183
- const scoreB = utilB * 0.7 + ratioB * 0.3;
2184
- return scoreB - scoreA;
2185
- });
2186
- // Phase 0: surface coverage gaps from zero-result search queries
2187
- let coverageGaps = [];
2188
- try {
2189
- const dbForGaps = openExistingDatabase();
2190
- try {
2191
- coverageGaps = getZeroResultSearches(dbForGaps);
2192
- }
2193
- finally {
2194
- closeDatabase(dbForGaps);
2195
- }
2196
- }
2197
- catch (err) {
2198
- rethrowIfTestIsolationError(err);
2199
- // best-effort
2200
- }
2201
- // actionableRefs is the post-cooldown, post-validation, post-signal, post-sort
2202
- // set — i.e. the genuinely processable refs in priority order. Note: this is
2203
- // a semantic shift from earlier code where actionableRefs was the pre-cooldown
2204
- // sorted set; the new meaning matches reality and is documented on
2205
- // ImprovePreparationResult.actionableRefs.
2206
- //
2207
- // Final guard: drop any candidate whose backing file is no longer on disk.
2208
- // Phase 1 validation captures missing files at the start of preparation, but
2209
- // the gap between that check and dispatch can be minutes on large stashes —
2210
- // long enough for a checkpoint / git checkout / external cleanup to delete
2211
- // the asset. Empirically (improve-critical-review 2026-05-20) the single
2212
- // biggest reject category was "Asset no longer exists on disk" (604/1407 =
2213
- // 43%), meaning reflect/distill was producing proposals against deleted refs.
2214
- // A cheap existsSync per surviving candidate eliminates that wasted work.
2215
- const assetMissingOnDisk = [];
2216
- const existsCheckedActionable = [];
2217
- for (const candidate of sorted) {
2218
- // #591: prefer the path pre-resolved at planning time (synchronous
2219
- // existsSync) over a serial async DB lookup per ref.
2220
- const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
2221
- ? candidate.filePath
2222
- : await findAssetFilePath(candidate.ref, options.stashDir);
2223
- if (filePath && fs.existsSync(filePath)) {
2224
- existsCheckedActionable.push(candidate);
2225
- }
2226
- else {
2227
- assetMissingOnDisk.push(candidate.ref);
2228
- }
2229
- }
2230
- // #592 audit: one summary event instead of one per missing ref. Normally
2231
- // tiny, but a stash deletion racing the run could make this O(n) sequential
2232
- // state.db writes. `refs` is capped so the metadata row stays bounded.
2233
- if (assetMissingOnDisk.length > 0) {
2234
- appendEvent({
2235
- eventType: "improve_skipped",
2236
- ref: undefined,
2237
- metadata: {
2238
- reason: "asset_missing_on_disk",
2239
- count: assetMissingOnDisk.length,
2240
- refs: assetMissingOnDisk.slice(0, 50),
2241
- },
2242
- }, eventsCtx);
2243
- }
2244
- const actionableRefs = existsCheckedActionable;
2245
- // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
2246
- // preserving sort order. distillOnlyRefs participate in the sort so --limit
2247
- // picks them by score, not by arbitrary position.
2248
- const distillOnlyRefSetForSort = new Set(distillOnlyRefs.map((r) => r.ref));
2249
- const reflectAndDistillRefsAfterSort = [];
2250
- const distillOnlyRefsAfterSort = [];
2251
- for (const r of actionableRefs) {
2252
- if (distillOnlyRefSetForSort.has(r.ref)) {
2253
- distillOnlyRefsAfterSort.push(r);
2254
- }
2255
- else {
2256
- reflectAndDistillRefsAfterSort.push(r);
2257
- }
2258
- }
2259
- // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
2260
- const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
2261
- const loopRefs = options.limit ? allLoopRefs.slice(0, options.limit) : allLoopRefs;
2262
- // Update the returned distillOnlyRefs to the sorted order so callers see the
2263
- // ranked view (loop stage uses it as a Set so order is irrelevant, but the
2264
- // shape change keeps downstream consumers consistent).
2265
- const distillOnlyRefsResult = distillOnlyRefsAfterSort;
2266
- const totalReflectBlocked = fullySkippedCount + distillOnlyRefs.length;
2267
- if (totalReflectBlocked > 0) {
2268
- info(`[improve] ${totalReflectBlocked} of ${preCooldownCount} indexed refs blocked by reflect signal-delta ` +
2269
- `(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
2270
- }
2271
- if (signalAndRetrievalRefs.length > 0) {
2272
- info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval)`);
2273
- }
2274
- if (validationFailureRefs.size > 0) {
2275
- info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
2276
- }
2277
- if (assetMissingOnDisk.length > 0) {
2278
- info(`[improve] ${assetMissingOnDisk.length} candidates dropped — file not on disk`);
2279
- }
2280
- const deferredCount = actionableRefs.length - loopRefs.length;
2281
- info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
2282
- (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
2283
- return {
2284
- actions,
2285
- cleanupWarnings,
2286
- appliedCleanup,
2287
- memoryIndexHealth,
2288
- extract: extractResults,
2289
- actionableRefs,
2290
- signalBearingSet,
2291
- validationFailures,
2292
- schemaRepairs,
2293
- lintSummary,
2294
- loopRefs,
2295
- distillCooledRefs,
2296
- distillOnlyRefs: distillOnlyRefsResult,
2297
- coverageGaps,
2298
- recentErrors,
2299
- utilityMap,
2300
- gateAutoAcceptedCount,
2301
- gateAutoAcceptFailedCount,
2302
- consolidation: consolidationPass.consolidation,
2303
- consolidationRan: consolidationPass.consolidationRan,
2304
- ...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
2305
- };
2306
- }
2307
- async function runImproveLoopStage(args) {
2308
- const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, utilityMap, startMs, budgetMs, eventsCtx, improveProfile, } = args;
2309
- // O-1 (#364): compute remaining budget at call time so each sub-call
2310
- // receives only its fair share of the wall-clock budget.
2311
- const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
2312
- const RECENT_ERRORS_CAP = 3;
2313
- // requirePlannedRefs guard: when the distill profile sets this flag, skip
2314
- // distill for distill-only refs if the reflect phase produced no planned refs.
2315
- // Prevents the distill loop from generating hundreds of distill-skipped events
2316
- // on quiet passes (all refs on reflect cooldown, no new signal to distill).
2317
- const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
2318
- const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
2319
- const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
2320
- const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
2321
- // R-2 / #389: Self-Consistency multi-sample voting helpers.
2322
- // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
2323
- const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
2324
- const SC_N = Math.min(Math.max(2, options.selfConsistencyN ?? 3), 5);
2325
- /**
2326
- * Compute Jaccard token overlap between two strings.
2327
- * Tokenizes by whitespace; returns 0 when both are empty.
2328
- */
2329
- function jaccardSimilarity(a, b) {
2330
- const tokensA = new Set(a.split(/\s+/).filter(Boolean));
2331
- const tokensB = new Set(b.split(/\s+/).filter(Boolean));
2332
- if (tokensA.size === 0 && tokensB.size === 0)
2333
- return 1;
2334
- let intersection = 0;
2335
- for (const t of tokensA) {
2336
- if (tokensB.has(t))
2337
- intersection++;
2338
- }
2339
- const union = tokensA.size + tokensB.size - intersection;
2340
- return union > 0 ? intersection / union : 0;
2341
- }
2342
- /**
2343
- * Given N reflect results, return the one with the highest average Jaccard
2344
- * similarity to all other successful results (majority-vote winner).
2345
- * Falls back to the first successful result when N < 2.
2346
- */
2347
- function pickMajorityVote(results) {
2348
- const successful = results.filter((r) => r.ok);
2349
- if (successful.length === 0)
2350
- return (results[0] ?? {
2351
- schemaVersion: 1,
2352
- ok: false,
2353
- reason: "non_zero_exit",
2354
- error: "all samples failed",
2355
- exitCode: null,
2356
- });
2357
- if (successful.length === 1)
2358
- return successful[0];
2359
- let bestIdx = 0;
2360
- let bestScore = -1;
2361
- for (let i = 0; i < successful.length; i++) {
2362
- let totalSim = 0;
2363
- for (let j = 0; j < successful.length; j++) {
2364
- if (i === j)
2365
- continue;
2366
- totalSim += jaccardSimilarity(successful[i].proposal.payload.content ?? "", successful[j].proposal.payload.content ?? "");
2367
- }
2368
- const avgSim = totalSim / (successful.length - 1);
2369
- if (avgSim > bestScore) {
2370
- bestScore = avgSim;
2371
- bestIdx = i;
2372
- }
2373
- }
2374
- return successful[bestIdx] ?? successful[0];
2375
- }
2376
- // O-5 / #378: helper to push per-originator errors into the rolling window.
2377
- function pushRecentError(originator, msg) {
2378
- if (!recentErrors[originator])
2379
- recentErrors[originator] = [];
2380
- recentErrors[originator].push(msg);
2381
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
2382
- recentErrors[originator].shift();
2383
- }
2384
- // Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
2385
- const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
2386
- let completedCount = 0;
2387
- let reflectsWithErrorContext = 0;
2388
- const memoryRefsForInference = new Set();
2389
- // Pre-load all pending proposals once instead of querying per asset in the loop.
2390
- const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
2391
- const pendingProposalRefSet = new Set(dedupeStashDirForProposals
2392
- ? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
2393
- : []);
2394
- let gateAutoAcceptedCount = 0;
2395
- let gateAutoAcceptFailedCount = 0;
2396
- const reflectGateCfg = makeGateConfig("reflect", {
2397
- globalThreshold: options.autoAccept,
2398
- dryRun: options.dryRun ?? false,
2399
- stashDir: primaryStashDir,
2400
- config: options.config ?? loadConfig(),
2401
- eventsCtx,
2402
- });
2403
- const distillGateCfg = makeGateConfig("distill", {
2404
- globalThreshold: options.autoAccept,
2405
- dryRun: options.dryRun ?? false,
2406
- stashDir: primaryStashDir,
2407
- config: options.config ?? loadConfig(),
2408
- eventsCtx,
2409
- });
2410
- for (const planned of loopRefs) {
2411
- if (Date.now() - startMs >= budgetMs) {
2412
- const remaining = loopRefs.length - completedCount;
2413
- info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
2414
- appendEvent({
2415
- eventType: "improve_skipped",
2416
- ref: planned.ref,
2417
- metadata: {
2418
- reason: "budget_exhausted",
2419
- remaining,
2420
- },
2421
- }, eventsCtx);
2422
- // B11: Emit improve_skipped for all remaining assets that will not be processed.
2423
- for (const remainingRef of loopRefs.slice(completedCount + 1)) {
2424
- appendEvent({
2425
- eventType: "improve_skipped",
2426
- ref: remainingRef.ref,
2427
- metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
2428
- }, eventsCtx);
2429
- }
2430
- actions.push({
2431
- ref: planned.ref,
2432
- mode: "error",
2433
- result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
2434
- });
2435
- break;
2436
- }
2437
- try {
2438
- // Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
2439
- // Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
2440
- // have their synthetic actions emitted in runImprovePreparationStage.
2441
- const isDistillOnly = distillOnlyRefSet.has(planned.ref);
2442
- const parsedPlannedRef = parseAssetRef(planned.ref);
2443
- // B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
2444
- // shouldDistillMemoryRef already returns false for .derived refs, so the distill
2445
- // path is also a no-op for them — we just avoid unnecessary agent spawns.
2446
- // D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
2447
- if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
2448
- // Type guard: skip reflect for unsupported types (script, env, task, etc.)
2449
- // and raw wiki directories, driven by the active improve profile.
2450
- const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
2451
- if (reflectSkip.skip) {
2452
- actions.push({
2453
- ref: planned.ref,
2454
- mode: "reflect-skipped",
2455
- result: { ok: true, reason: reflectSkip.reason },
2456
- });
2457
- }
2458
- else {
2459
- // O-5 / #378: only inject reflect-originator errors into the reflect call.
2460
- // Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
2461
- const reflectErrors = recentErrors.reflect ?? [];
2462
- if (reflectErrors.length > 0)
2463
- reflectsWithErrorContext++;
2464
- // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
2465
- // bounded by the wall-clock deadline rather than the default per-profile timeout.
2466
- const reflectBudgetMs = remainingBudgetMs();
2467
- // Wire profile.processes.reflect.{mode, profile, timeoutMs} into the reflect
2468
- // dispatch when present. Falls back to akmReflect's own config-based resolution
2469
- // (profiles.improve.<name>.processes.reflect → defaults.llm) when the profile
2470
- // does not specify.
2471
- const reflectProfileRunner = resolveImproveProcessRunnerFromProfile(improveProfile.processes?.reflect, options.config ?? loadConfig());
2472
- const reflectCallArgs = {
2473
- ref: planned.ref,
2474
- task: options.task,
2475
- ...(options.stashDir ? { stashDir: options.stashDir } : {}),
2476
- ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
2477
- agentProcess: options.agentProcess ?? "reflect",
2478
- eventSource: "improve",
2479
- ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
2480
- ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
2481
- // Attribution: carry the eligibility lane so reflect stamps it on
2482
- // the reflect_invoked event and the persisted proposal.
2483
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2484
- };
2485
- // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
2486
- // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
2487
- const refUtility = utilityMap.get(planned.ref) ?? 0;
2488
- const useConsistency = refUtility >= SC_THRESHOLD && SC_N >= 2;
2489
- let reflectResult;
2490
- if (useConsistency) {
2491
- const samples = [];
2492
- for (let s = 0; s < SC_N; s++) {
2493
- if (remainingBudgetMs() <= 0)
2494
- break;
2495
- // draftMode: skip DB write so each sample doesn't create a proposal.
2496
- samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
2497
- }
2498
- const winner = pickMajorityVote(samples.length > 0
2499
- ? samples
2500
- : [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
2501
- // Persist only the majority-vote winner as a single real proposal.
2502
- if (winner.ok && primaryStashDir) {
2503
- const persistResult = createProposal(primaryStashDir, {
2504
- ref: winner.proposal.ref,
2505
- source: "reflect",
2506
- sourceRun: `reflect-sc-${Date.now()}`,
2507
- payload: winner.proposal.payload,
2508
- // Attribution: the self-consistency path persists the winner here
2509
- // (draftMode skips reflect's own createProposal), so stamp the lane.
2510
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2511
- });
2512
- reflectResult = isProposalSkipped(persistResult)
2513
- ? {
2514
- schemaVersion: 1,
2515
- ok: false,
2516
- reason: "cooldown",
2517
- error: `SC proposal skipped: ${persistResult.message}`,
2518
- ref: winner.ref,
2519
- exitCode: null,
2520
- }
2521
- : { ...winner, proposal: persistResult };
2522
- }
2523
- else {
2524
- reflectResult = winner;
2525
- }
2526
- }
2527
- else {
2528
- reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
2529
- }
2530
- const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
2531
- // Content-policy guard hits (reflect size-rail rejections) are NOT
2532
- // LLM faults — the agent responded fine, the downstream guard
2533
- // blocked the output. Route them to a distinct `reflect-guard-rejected`
2534
- // mode so health metrics can split deterministic guard hits out of
2535
- // true LLM failures. See
2536
- // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
2537
- const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
2538
- // Type-guard rejection (reflect refused a script/env/task ref) is
2539
- // also NOT an LLM failure — the LLM is never invoked. Route to the
2540
- // existing `reflect-skipped` bucket so it does not inflate the
2541
- // failure-rate numerator. ~9% of `reflect-failed` events in the
2542
- // user's stack were this case; see review §1a row "Reflect refused
2543
- // asset type".
2544
- const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
2545
- // Noise-gate suppression (#580): the candidate edit was an empty
2546
- // diff or a cosmetic-only reformat of the current asset. Like
2547
- // `unsupported_type`, this is a deterministic skip — not an LLM
2548
- // fault — so it routes to the `reflect-skipped` bucket and stays
2549
- // out of recentErrors/avoidPatterns.
2550
- const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
2551
- actions.push({
2552
- ref: planned.ref,
2553
- mode: reflectResult.ok
2554
- ? "reflect"
2555
- : isCooldown
2556
- ? "reflect-cooldown"
2557
- : isGuardReject
2558
- ? "reflect-guard-rejected"
2559
- : isTypeRefused || isNoChange
2560
- ? "reflect-skipped"
2561
- : "reflect-failed",
2562
- result: reflectResult,
2563
- });
2564
- // Cooldown skips, guard rejects, type-refused skips, and noise-gate
2565
- // skips are not failures — do not pollute recentErrors with them
2566
- // (those get injected as `avoidPatterns` into the next reflect
2567
- // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
2568
- // so the next iteration sees "your last expansion was too large";
2569
- // type-refused and no-change are deterministic and add no learning
2570
- // signal.
2571
- if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
2572
- const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
2573
- pushRecentError("reflect", errMsg);
2574
- }
2575
- // improve_reflect_outcome — per-asset metric for tuning the reflect path.
2576
- appendEvent({
2577
- eventType: "improve_reflect_outcome",
2578
- ref: planned.ref,
2579
- metadata: {
2580
- ok: reflectResult.ok,
2581
- durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
2582
- agentProfile: reflectResult.ok ? reflectResult.agentProfile : undefined,
2583
- reason: reflectResult.ok ? undefined : reflectResult.reason,
2584
- },
2585
- }, eventsCtx);
2586
- if (reflectResult.ok) {
2587
- const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
2588
- gateAutoAcceptedCount += reflectGr.promoted.length;
2589
- gateAutoAcceptFailedCount += reflectGr.failed.length;
2590
- }
2591
- } // end else (reflect type/profile check)
2592
- }
2593
- else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
2594
- // B6: .derived refs skip reflect; record synthetic skip action.
2595
- actions.push({
2596
- ref: planned.ref,
2597
- mode: "distill-skipped",
2598
- result: { ok: true, reason: "derived-memory-reflect-skipped" },
2599
- });
2600
- appendEvent({
2601
- eventType: "improve_skipped",
2602
- ref: planned.ref,
2603
- metadata: { reason: "derived_memory_reflect_skipped" },
2604
- }, eventsCtx);
2605
- }
2606
- // isDistillOnly refs: no reflect action emitted — proceed directly to distill path below.
2607
- const hasRecentFeedbackSignal = signalBearingSet.has(planned.ref);
2608
- const explicitRefScope = scope.mode === "ref";
2609
- // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
2610
- // distill so callers who configure `profile.processes.distill.allowedTypes`
2611
- // or land on raw-wiki refs get a recorded skip action instead of silently
2612
- // proceeding.
2613
- const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
2614
- if (distillSkip.skip) {
2615
- actions.push({
2616
- ref: planned.ref,
2617
- mode: "distill-skipped",
2618
- result: { ok: true, reason: distillSkip.reason },
2619
- });
2620
- completedCount++;
2621
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2622
- continue;
2623
- }
2624
- // requirePlannedRefs guard: skip distill for distill-only refs when no
2625
- // reflect-eligible refs were planned this run, preventing mass skip events.
2626
- if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
2627
- actions.push({
2628
- ref: planned.ref,
2629
- mode: "distill-skipped",
2630
- result: { ok: true, reason: "require_planned_refs" },
2631
- });
2632
- completedCount++;
2633
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2634
- continue;
2635
- }
2636
- // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
2637
- // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
2638
- // it will refuse.
2639
- const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
2640
- const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
2641
- // distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
2642
- // refs; non-candidate refs in the set are blocked here.
2643
- // O-2 (#365): bypass the distill cooldown when the user explicitly targeted
2644
- // this ref via --scope — their intent overrides unattended-run policies.
2645
- if (shouldAttemptDistill &&
2646
- !skipMemoryDistillForWeakSignal &&
2647
- (!distillCooledRefs.has(planned.ref) || explicitRefScope)) {
2648
- // TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
2649
- const lessonRef = deriveLessonRef(planned.ref);
2650
- const knowledgeRef = deriveKnowledgeRef(planned.ref);
2651
- const dedupeStashDir = primaryStashDir ?? options.stashDir;
2652
- if (dedupeStashDir) {
2653
- // B2: check both lesson ref and knowledge ref since auto-promoted memories
2654
- // create knowledge: proposals, not lesson: proposals.
2655
- const hasExistingPending = pendingProposalRefSet.has(lessonRef) || pendingProposalRefSet.has(knowledgeRef);
2656
- if (hasExistingPending) {
2657
- actions.push({
2658
- ref: planned.ref,
2659
- mode: "distill-skipped",
2660
- result: { ok: true, reason: "pending proposal exists" },
2661
- });
2662
- appendEvent({
2663
- eventType: "improve_skipped",
2664
- ref: planned.ref,
2665
- metadata: { reason: "pending_proposal_exists" },
2666
- }, eventsCtx);
2667
- completedCount++;
2668
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2669
- continue;
2670
- }
2671
- // D-2 (#370): reject-aware cooldown for distill. When the reviewer
2672
- // recently rejected a distilled lesson or knowledge proposal for this
2673
- // asset, skip re-distillation for a 1-day grace window. Prevents the
2674
- // same rejected proposal from being regenerated immediately. The
2675
- // window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
2676
- // signal-delta gates and dropped --distill-cooldown-days; a short
2677
- // reject grace is preserved here so a fresh rejection isn't
2678
- // overridden by the same run).
2679
- // References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
2680
- const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
2681
- const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
2682
- (rejectedProposalsByRef.has(lessonRef) || rejectedProposalsByRef.has(knowledgeRef));
2683
- if (recentlyRejectedLesson) {
2684
- const rejectedEntry = rejectedProposalsByRef.get(lessonRef) ?? rejectedProposalsByRef.get(knowledgeRef);
2685
- const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
2686
- if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
2687
- actions.push({
2688
- ref: planned.ref,
2689
- mode: "distill-skipped",
2690
- result: { ok: true, reason: "distill reject grace window" },
2691
- });
2692
- appendEvent({
2693
- eventType: "improve_skipped",
2694
- ref: planned.ref,
2695
- metadata: {
2696
- reason: "distill_reject_grace_window",
2697
- },
2698
- }, eventsCtx);
2699
- completedCount++;
2700
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2701
- continue;
2702
- }
2703
- }
2704
- }
2705
- const distillResult = await withLlmStage("distill", () => distillFn({
2706
- ref: planned.ref,
2707
- ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
2708
- ...(options.stashDir ? { stashDir: options.stashDir } : {}),
2709
- // Attribution: carry the eligibility lane so distill stamps it on the
2710
- // distill_invoked event and the persisted proposal.
2711
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2712
- }));
2713
- actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
2714
- if (distillResult.outcome === "queued" && distillResult.proposal) {
2715
- const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
2716
- gateAutoAcceptedCount += distillGr.promoted.length;
2717
- gateAutoAcceptFailedCount += distillGr.failed.length;
2718
- }
2719
- if (parsedPlannedRef.type === "memory") {
2720
- const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
2721
- if (!promotedToKnowledge)
2722
- memoryRefsForInference.add(planned.ref);
2723
- }
2724
- if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
2725
- const slug = planned.ref
2726
- .replace(/[^a-z0-9]/gi, "-")
2727
- .toLowerCase()
2728
- .slice(0, 60);
2729
- writeEvalCase(primaryStashDir, {
2730
- ref: planned.ref,
2731
- failureReason: distillResult.reason ?? "quality gate rejected",
2732
- assetType: parseAssetRef(planned.ref).type ?? "unknown",
2733
- rejectedAt: Date.now(),
2734
- source: "distill_quality_rejected",
2735
- slug: `${slug}-${Date.now()}`,
2736
- });
2737
- }
2738
- // D6: use pre-loaded map instead of per-iteration DB query
2739
- const rejectedProposalEvent = rejectedProposalsByRef.get(planned.ref);
2740
- if (rejectedProposalEvent && primaryStashDir) {
2741
- const slug = planned.ref
2742
- .replace(/[^a-z0-9]/gi, "-")
2743
- .toLowerCase()
2744
- .slice(0, 60);
2745
- writeEvalCase(primaryStashDir, {
2746
- ref: planned.ref,
2747
- failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
2748
- assetType: parseAssetRef(planned.ref).type ?? "unknown",
2749
- rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
2750
- source: "proposal_rejected",
2751
- slug: `${slug}-rejected`,
2752
- });
2753
- }
2754
- }
2755
- else if (skipMemoryDistillForWeakSignal) {
2756
- actions.push({
2757
- ref: planned.ref,
2758
- mode: "distill-skipped",
2759
- result: { ok: true, reason: "memory requires recent feedback signal" },
2760
- });
2761
- appendEvent({
2762
- eventType: "improve_skipped",
2763
- ref: planned.ref,
2764
- metadata: { reason: "memory_distill_requires_feedback" },
2765
- }, eventsCtx);
2766
- }
2767
- }
2768
- catch (err) {
2769
- // B7: UsageError thrown by akmDistill on validation_failed should be recorded
2770
- // as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
2771
- // The distill_invoked event was already emitted inside akmDistill before the throw.
2772
- if (err instanceof UsageError) {
2773
- actions.push({
2774
- ref: planned.ref,
2775
- mode: "distill",
2776
- result: { ok: false, outcome: "validation_failed", error: err.message },
2777
- });
2778
- }
2779
- else {
2780
- actions.push({
2781
- ref: planned.ref,
2782
- mode: "error",
2783
- result: { ok: false, error: err instanceof Error ? err.message : String(err) },
2784
- });
2785
- }
2786
- }
2787
- completedCount++;
2788
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2789
- }
2790
- return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
2791
- }
2792
- async function runImprovePostLoopStage(args) {
2793
- const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, consolidationRan, } = args;
2794
- const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
2795
- info("[improve] post-loop maintenance starting");
2796
- const maintenanceResult = await runImproveMaintenancePasses({
2797
- options,
2798
- primaryStashDir,
2799
- actionableRefs,
2800
- memoryRefsForInference,
2801
- allWarnings,
2802
- reindexFn,
2803
- consolidationRan,
2804
- // O-1 (#364): forward the budget signal to memory inference + graph extraction.
2805
- budgetSignal,
2806
- eventsCtx,
2807
- improveProfile,
2808
- });
2809
- let deadUrls;
2810
- if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
2811
- try {
2812
- const knowledgeEntries = actionableRefs
2813
- .filter((r) => {
2814
- try {
2815
- return parseAssetRef(r.ref).type === "knowledge";
2816
- }
2817
- catch {
2818
- return false;
2819
- }
2820
- })
2821
- .slice(0, 10)
2822
- .map((r) => ({ ref: r.ref, body: "" }));
2823
- if (knowledgeEntries.length > 0) {
2824
- info(`[improve] checking URLs in ${knowledgeEntries.length} knowledge refs`);
2825
- deadUrls = await checkDeadUrls(primaryStashDir, knowledgeEntries);
2826
- info(`[improve] URL check complete (${deadUrls.length} dead/timeout URLs)`);
2827
- }
2828
- }
2829
- catch {
2830
- // best-effort
2831
- }
2832
- }
2833
- return {
2834
- allWarnings,
2835
- deadUrls,
2836
- ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
2837
- ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
2838
- ...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
2839
- ...(maintenanceResult.actions && maintenanceResult.actions.length > 0
2840
- ? { maintenanceActions: maintenanceResult.actions }
2841
- : {}),
2842
- memoryInferenceDurationMs: maintenanceResult.memoryInferenceDurationMs,
2843
- graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
2844
- orphansPurged: maintenanceResult.orphansPurged,
2845
- proposalsExpired: maintenanceResult.proposalsExpired,
2846
- // Consolidation's auto-accept gate counts now accrue in the preparation
2847
- // stage (#551); post-loop no longer runs an auto-accept gate of its own.
2848
- gateAutoAcceptedCount: 0,
2849
- gateAutoAcceptFailedCount: 0,
2850
- };
2851
- }
2852
- // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
2853
- // Exported for tests (#584/#585 DB-locking regression coverage); production
2854
- // callers reach it only through akmImprove → runImprovePostLoopStage.
2855
- export async function runImproveMaintenancePasses(args) {
2856
- const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
2857
- if (!primaryStashDir)
2858
- return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
2859
- const config = options.config ?? loadConfig();
2860
- const sources = resolveSourceEntries(options.stashDir, config);
2861
- const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
2862
- const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
2863
- const stalenessDetectionFn = options.stalenessDetectionFn ?? runStalenessDetectionPass;
2864
- let db;
2865
- let memoryInference;
2866
- let graphExtraction;
2867
- let stalenessDetection;
2868
- let reindexedAfterInference = false;
2869
- const actions = [];
2870
- let memoryInferenceDurationMs = 0;
2871
- let graphExtractionDurationMs = 0;
2872
- let orphansPurged = 0;
2873
- let proposalsExpired = 0;
2874
- const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2875
- // #584: reindexFn opens its own write handle on the same index.db WAL file.
2876
- // Holding our handle across that call produced SQLITE_BUSY / "database is
2877
- // locked" failures in production, so the handle is closed BEFORE every
2878
- // reindex and reopened after — the fresh handle also sees the post-reindex
2879
- // state that graph extraction and staleness detection below rely on. The
2880
- // reopen runs in `finally` so a failed reindex still leaves a usable handle.
2881
- const reindexWithIndexDbReleased = async (stashDir) => {
2882
- if (db) {
2883
- closeDatabase(db);
2884
- db = undefined;
2885
- }
2886
- try {
2887
- await reindexFn({ stashDir });
2888
- }
2889
- finally {
2890
- db = openIndexDb();
2891
- }
2892
- };
2893
- await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
2894
- try {
2895
- db = openIndexDb();
2896
- // Memory inference candidate-discovery (post-Item 9 fix from
2897
- // memory:akm-improve-critical-review-2026-05-20). Previously this pass
2898
- // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
2899
- // candidateRefs filter. But memoryRefsForInference is populated from refs
2900
- // distilled THIS RUN — by the time that happens, those parents are
2901
- // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
2902
- // them. The genuinely-pending parents in the stash never entered the
2903
- // filter. Result: 0/0/0 for 25 consecutive runs.
2904
- //
2905
- // Fix: always run the pass when the feature is enabled; let the pass's
2906
- // own `collectPendingMemories` + `isPendingMemory` predicate find
2907
- // candidates from the filesystem-of-truth. The this-run set is still
2908
- // logged as a hint but no longer used as a filter.
2909
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
2910
- const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
2911
- const pendingBelowMinCount = (() => {
2912
- if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
2913
- return false;
2914
- const pending = collectPendingMemories(primaryStashDir).length;
2915
- if (pending < minPendingCount) {
2916
- info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
2917
- return true;
2918
- }
2919
- return false;
2920
- })();
2921
- if (memoryInferenceDisabledByProfile) {
2922
- info("[improve] memory inference skipped (disabled by improve profile)");
2923
- }
2924
- else if (pendingBelowMinCount) {
2925
- // skipped — message already emitted above
2926
- }
2927
- else {
2928
- const hintRefs = memoryRefsForInference.size;
2929
- info(hintRefs > 0
2930
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
2931
- : "[improve] memory inference starting (discovering pending parents)");
2932
- const inferenceStart = Date.now();
2933
- try {
2934
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
2935
- memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
2936
- config,
2937
- sources,
2938
- signal: budgetSignal,
2939
- db,
2940
- reEnrich: false,
2941
- onProgress: (event) => {
2942
- const current = event.currentRef ? ` ${event.currentRef}` : "";
2943
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
2944
- },
2945
- }));
2946
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2947
- actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
2948
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2949
- }
2950
- catch (err) {
2951
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2952
- allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2953
- }
2954
- }
2955
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
2956
- info("[improve] reindexing after memory inference writes");
2957
- try {
2958
- await reindexWithIndexDbReleased(primaryStashDir);
2959
- reindexedAfterInference = true;
2960
- info("[improve] reindex after memory inference complete");
2961
- }
2962
- catch (err) {
2963
- allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2964
- }
2965
- }
2966
- const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
2967
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
2968
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
2969
- // Build the set of refs actually touched this run.
2970
- const touchedRefs = new Set();
2971
- for (const r of args.actionableRefs)
2972
- touchedRefs.add(r.ref);
2973
- for (const r of memoryRefsForInference)
2974
- touchedRefs.add(r);
2975
- // INVARIANT: graph extraction normally runs only on files touched by
2976
- // actionable refs (candidatePaths). Full-corpus scans are opt-in via
2977
- // profile.processes.graphExtraction.fullScan = true (used by the
2978
- // `graph-refresh` built-in profile and its weekly scheduled task).
2979
- // The empty-Set fallback is intentional when no refs were touched —
2980
- // the extractor's filter rejects every file and returns empty, keeping
2981
- // the pass invoked so the action is recorded and tests stay exercised.
2982
- if (graphExtractionDisabledByProfile) {
2983
- info("[improve] graph extraction skipped (disabled by improve profile)");
2984
- }
2985
- else if (sources.length > 0 && graphEnabled) {
2986
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
2987
- const extractionStart = Date.now();
2988
- try {
2989
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
2990
- // so graph extraction sees current DB state after consolidation writes.
2991
- if (consolidationRan && !reindexedAfterInference) {
2992
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
2993
- try {
2994
- await reindexWithIndexDbReleased(primaryStashDir);
2995
- reindexedAfterInference = true;
2996
- info("[improve] reindex after consolidation complete");
2997
- }
2998
- catch (err) {
2999
- allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
3000
- }
3001
- }
3002
- // #584: no close/reopen needed here — reindexWithIndexDbReleased
3003
- // already swapped in a fresh post-reindex handle.
3004
- // Resolve touched refs to absolute file paths. Skipped for fullScan
3005
- // (candidatePaths stays undefined → extractor processes all files).
3006
- let candidatePaths;
3007
- if (!graphExtractionFullScan) {
3008
- candidatePaths = new Set();
3009
- if (primaryStashDir && touchedRefs.size > 0) {
3010
- const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
3011
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
3012
- for (const p of resolved) {
3013
- if (typeof p === "string" && p.length > 0)
3014
- candidatePaths.add(p);
3015
- }
3016
- }
3017
- }
3018
- const progressHandler = (event) => {
3019
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
3020
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
3021
- };
3022
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
3023
- graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
3024
- config,
3025
- sources,
3026
- signal: budgetSignal,
3027
- db,
3028
- reEnrich: false,
3029
- onProgress: progressHandler,
3030
- options: { candidatePaths },
3031
- }));
3032
- graphExtractionDurationMs = Date.now() - extractionStart;
3033
- actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
3034
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
3035
- }
3036
- catch (err) {
3037
- graphExtractionDurationMs = Date.now() - extractionStart;
3038
- allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
3039
- }
3040
- }
3041
- else if (sources.length > 0 && !graphEnabled) {
3042
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
3043
- }
3044
- // Orphan proposal purge — reject pending reflect proposals whose target
3045
- // asset no longer exists on disk. Runs after graph extraction so newly
3046
- // promoted assets from accept flows during this run are already present.
3047
- if (primaryStashDir) {
3048
- try {
3049
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
3050
- orphansPurged = purgeResult.rejected;
3051
- if (purgeResult.rejected > 0) {
3052
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
3053
- }
3054
- appendEvent({
3055
- eventType: "proposal_orphan_purge",
3056
- ref: "proposals:_orphan-purge",
3057
- metadata: {
3058
- checked: purgeResult.checked,
3059
- rejected: purgeResult.rejected,
3060
- durationMs: purgeResult.durationMs,
3061
- byType: purgeResult.byType,
3062
- orphans: purgeResult.orphans.map((o) => o.ref),
3063
- },
3064
- }, eventsCtx);
3065
- }
3066
- catch (err) {
3067
- allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
3068
- }
3069
- // Phase 6B (Advantage D6b): expire pending proposals that have aged past
3070
- // the retention window. Runs AFTER orphan purge so we never double-archive
3071
- // a proposal that orphan-purge already moved. `expireStaleProposals` emits
3072
- // its own per-proposal `proposal_expired` events; we additionally emit a
3073
- // single roll-up event here for parity with the orphan-purge surface.
3074
- try {
3075
- const expireResult = expireStaleProposals(primaryStashDir, config);
3076
- proposalsExpired = expireResult.expired;
3077
- if (expireResult.expired > 0) {
3078
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
3079
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
3080
- }
3081
- appendEvent({
3082
- eventType: "proposal_expiration_pass",
3083
- ref: "proposals:_expiration",
3084
- metadata: {
3085
- checked: expireResult.checked,
3086
- expired: expireResult.expired,
3087
- durationMs: expireResult.durationMs,
3088
- retentionDays: expireResult.retentionDays,
3089
- expiredProposals: expireResult.expiredProposals,
3090
- },
3091
- }, eventsCtx);
3092
- }
3093
- catch (err) {
3094
- allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
3095
- }
3096
- }
3097
- // Fix #2 (observability 0.8.0): trim the events table in state.db so it
3098
- // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
3099
- // invocation, and every command surface emits at least one event besides —
3100
- // without this trim, state.db is a permanent append-only log. Config key
3101
- // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
3102
- // window. The purge runs against state.db (a different SQLite file from
3103
- // the index `db` above).
3104
- {
3105
- const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
3106
- if (retentionDays > 0) {
3107
- // #585: reuse the long-lived eventsCtx.db connection when akmImprove
3108
- // opened one — opening a second state.db write connection while
3109
- // eventsDb is still live made two simultaneous writers contend on the
3110
- // same WAL file ("database is locked"). Only the eventsCtx.dbPath
3111
- // fallback path (state.db failed to open up-front) opens — and then
3112
- // owns and closes — its own handle. C2 still holds: the fallback uses
3113
- // the boundary-pinned path, never a live `process.env` re-read.
3114
- const ownsStateDb = !eventsCtx?.db;
3115
- let stateDb;
3116
- try {
3117
- stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
3118
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
3119
- if (purgedCount > 0) {
3120
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
3121
- }
3122
- appendEvent({
3123
- eventType: "events_purged",
3124
- ref: "events:_purge",
3125
- metadata: { purgedCount, retentionDays },
3126
- }, eventsCtx);
3127
- // improve_runs uses the same retention window as events — both are
3128
- // observability/audit data, both grow append-only, both have a
3129
- // dedicated purge helper. Mirroring the events purge here means a
3130
- // single retention knob (improve.eventRetentionDays) governs both.
3131
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
3132
- if (improveRunsPurged > 0) {
3133
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
3134
- }
3135
- appendEvent({
3136
- eventType: "improve_runs_purged",
3137
- ref: "improve_runs:_purge",
3138
- metadata: { purgedCount: improveRunsPurged, retentionDays },
3139
- }, eventsCtx);
3140
- }
3141
- catch (err) {
3142
- allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
3143
- }
3144
- finally {
3145
- if (ownsStateDb && stateDb) {
3146
- try {
3147
- stateDb.close();
3148
- }
3149
- catch {
3150
- // best-effort
3151
- }
3152
- }
3153
- }
3154
- // task_logs in logs.db (#579) shares the same retention window as
3155
- // events/improve_runs — all three are observability data governed by
3156
- // the single improve.eventRetentionDays knob. Separate try/finally
3157
- // because logs.db is a different file: a locked/missing logs.db must
3158
- // not block the state.db purges above.
3159
- let logsDb;
3160
- try {
3161
- logsDb = openLogsDatabase();
3162
- const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
3163
- if (taskLogsPurged > 0) {
3164
- info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
3165
- }
3166
- appendEvent({
3167
- eventType: "task_logs_purged",
3168
- ref: "task_logs:_purge",
3169
- metadata: { purgedCount: taskLogsPurged, retentionDays },
3170
- }, eventsCtx);
3171
- }
3172
- catch (err) {
3173
- allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
3174
- }
3175
- finally {
3176
- if (logsDb) {
3177
- try {
3178
- logsDb.close();
3179
- }
3180
- catch {
3181
- // best-effort
3182
- }
3183
- }
3184
- }
3185
- }
3186
- }
3187
- // Phase 4A (staleness detection). Activates the `deprecated` belief-state
3188
- // machinery shipped in Phase 1A. Default OFF — gated by
3189
- // `features.index.staleness_detection.enabled`. Runs after orphan purge
3190
- // and before the URL check (which lives in the outer caller).
3191
- if (sources.length > 0) {
3192
- try {
3193
- stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
3194
- if (stalenessDetection.considered > 0) {
3195
- info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
3196
- `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
3197
- `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
3198
- }
3199
- for (const w of stalenessDetection.warnings)
3200
- allWarnings.push(`[improve] staleness detection: ${w}`);
3201
- }
3202
- catch (err) {
3203
- allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
3204
- }
3205
- }
3206
- }
3207
- finally {
3208
- if (db)
3209
- closeDatabase(db);
3210
- }
3211
- });
3212
- return {
3213
- ...(memoryInference ? { memoryInference } : {}),
3214
- ...(graphExtraction ? { graphExtraction } : {}),
3215
- ...(stalenessDetection ? { stalenessDetection } : {}),
3216
- ...(actions.length > 0 ? { actions } : {}),
3217
- memoryInferenceDurationMs,
3218
- graphExtractionDurationMs,
3219
- orphansPurged,
3220
- proposalsExpired,
3221
- };
3222
- }
3223
- function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStashDir) {
3224
- if (!primaryStashDir || eligibleMemories === 0)
3225
- return false;
3226
- if (scope.mode === "all")
3227
- return true;
3228
- if (scope.mode === "type")
3229
- return scope.value === "memory";
3230
- if (!scope.value)
3231
- return false;
3232
- return parseAssetRef(scope.value).type === "memory";
3233
- }
3234
1000
  function shapeMemoryCleanup(plan) {
3235
1001
  return {
3236
1002
  analyzedDerived: plan.analyzedDerived,
@@ -3241,48 +1007,3 @@ function shapeMemoryCleanup(plan) {
3241
1007
  ...(plan.relativeDateCandidates.length > 0 ? { relativeDateCandidates: plan.relativeDateCandidates } : {}),
3242
1008
  };
3243
1009
  }
3244
- function buildUtilityMap(refs) {
3245
- const map = new Map();
3246
- if (refs.length === 0)
3247
- return map;
3248
- const refSet = new Set(refs.map((r) => r.ref));
3249
- let db;
3250
- try {
3251
- db = openExistingDatabase();
3252
- const allDbEntries = getAllEntries(db);
3253
- const idToRef = new Map();
3254
- for (const indexed of allDbEntries) {
3255
- const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
3256
- if (refSet.has(ref))
3257
- idToRef.set(indexed.id, ref);
3258
- }
3259
- const ids = [...idToRef.keys()];
3260
- if (ids.length > 0) {
3261
- const { global: scores } = getUtilityScoresByIds(db, ids);
3262
- for (const [id, score] of scores) {
3263
- const ref = idToRef.get(id);
3264
- if (ref)
3265
- map.set(ref, score.utility);
3266
- }
3267
- }
3268
- }
3269
- catch (err) {
3270
- rethrowIfTestIsolationError(err);
3271
- // best-effort: if DB unavailable, all utilities default to 0
3272
- }
3273
- finally {
3274
- if (db)
3275
- closeDatabase(db);
3276
- }
3277
- return map;
3278
- }
3279
- async function findAssetFilePath(ref, stashDir, writableDirSet) {
3280
- return resolveAssetPath(ref, {
3281
- stashDir,
3282
- mode: "disk-only",
3283
- writableDirSet,
3284
- directoryIndexNames: ["SKILL.md"],
3285
- preserveDirectNameFallback: true,
3286
- honorOrigin: false,
3287
- });
3288
- }