akm-cli 0.9.16 → 0.9.17-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/CHANGELOG.md +2101 -0
  2. package/STABILITY.md +11 -10
  3. package/dist/akm +124 -193
  4. package/dist/akm-migrate +38 -19
  5. package/dist/assets/hints/cli-hints-full.md +6 -7
  6. package/dist/assets/improve-strategies/catchup.json +0 -3
  7. package/dist/assets/improve-strategies/consolidate.json +0 -1
  8. package/dist/assets/improve-strategies/default.json +1 -2
  9. package/dist/assets/improve-strategies/proactive-maintenance.json +1 -2
  10. package/dist/assets/improve-strategies/quick.json +1 -2
  11. package/dist/assets/improve-strategies/reflect-distill.json +1 -2
  12. package/dist/assets/improve-strategies/thorough.json +0 -3
  13. package/dist/assets/prompts/consolidate-pair.md +20 -0
  14. package/dist/assets/prompts/consolidate-system.md +4 -11
  15. package/dist/assets/prompts/retrieval-relevance-judge.md +6 -0
  16. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +20 -20
  17. package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -2
  18. package/dist/assets/templates/html/health.html +3 -5
  19. package/dist/cli/retired-commands.js +1 -1
  20. package/dist/cli/shared.js +6 -2
  21. package/dist/cli/unknown-flags.js +24 -1
  22. package/dist/cli.js +68 -10
  23. package/dist/commands/agent/agent-dispatch.js +1 -1
  24. package/dist/commands/command/command-execution.js +24 -62
  25. package/dist/commands/feedback-cli.js +0 -1
  26. package/dist/commands/health/accept-rate.js +6 -0
  27. package/dist/commands/health/archive-usage.js +92 -0
  28. package/dist/commands/health/checks.js +83 -74
  29. package/dist/commands/health/config-skew.js +38 -0
  30. package/dist/commands/health/data-dir-usage.js +25 -13
  31. package/dist/commands/health/egress.js +54 -0
  32. package/dist/commands/health/html-report.js +1 -42
  33. package/dist/commands/health/improve-metrics.js +136 -591
  34. package/dist/commands/health/md-report.js +1 -6
  35. package/dist/commands/health/plugin-staleness.js +53 -3
  36. package/dist/commands/health/renderers.js +12 -4
  37. package/dist/commands/health/report-view-model.js +14 -120
  38. package/dist/commands/health/types-improve.js +4 -19
  39. package/dist/commands/health/windows.js +64 -74
  40. package/dist/commands/health.js +145 -143
  41. package/dist/commands/improve/consolidate/chunking.js +26 -117
  42. package/dist/commands/improve/consolidate/continuity-check.js +137 -0
  43. package/dist/commands/improve/consolidate/pair-pass.js +791 -0
  44. package/dist/commands/improve/consolidate/sanitize.js +54 -149
  45. package/dist/commands/improve/consolidate.js +589 -1127
  46. package/dist/commands/improve/content-hash.js +16 -24
  47. package/dist/commands/improve/distill/content-repair.js +18 -100
  48. package/dist/commands/improve/distill-guards.js +20 -81
  49. package/dist/commands/improve/distill-promotion-policy.js +23 -243
  50. package/dist/commands/improve/distill.js +608 -1041
  51. package/dist/commands/improve/eligibility.js +126 -390
  52. package/dist/commands/improve/execution.js +8 -10
  53. package/dist/commands/improve/extract-prompt.js +1 -2
  54. package/dist/commands/improve/extract.js +487 -1046
  55. package/dist/commands/improve/feedback-valence.js +0 -25
  56. package/dist/commands/improve/improve-cli.js +75 -169
  57. package/dist/commands/improve/improve-result-file.js +10 -66
  58. package/dist/commands/improve/improve-strategies.js +52 -4
  59. package/dist/commands/improve/improve-usage-report.js +18 -64
  60. package/dist/commands/improve/improve.js +480 -1074
  61. package/dist/commands/improve/ledger.js +119 -0
  62. package/dist/commands/improve/locks.js +2 -8
  63. package/dist/commands/improve/loop-stages.js +415 -1073
  64. package/dist/commands/improve/memory/derived-ref.js +12 -77
  65. package/dist/commands/improve/memory/memory-belief.js +16 -118
  66. package/dist/commands/improve/memory/memory-improve.js +266 -14
  67. package/dist/commands/improve/outcome-loop.js +28 -156
  68. package/dist/commands/improve/planner.js +5 -15
  69. package/dist/commands/improve/preparation.js +779 -2319
  70. package/dist/commands/improve/proactive-maintenance.js +34 -101
  71. package/dist/commands/improve/reflect-noise.js +104 -280
  72. package/dist/commands/improve/reflect.js +642 -1353
  73. package/dist/commands/improve/retrieval-gate.js +127 -0
  74. package/dist/commands/improve/retrieval-scope.js +92 -0
  75. package/dist/commands/improve/salience.js +41 -240
  76. package/dist/commands/improve/session-asset.js +19 -100
  77. package/dist/commands/improve/stage.js +322 -0
  78. package/dist/commands/lint/base-linter.js +37 -15
  79. package/dist/commands/proposal/drain.js +261 -578
  80. package/dist/commands/proposal/proposal-cli.js +19 -20
  81. package/dist/commands/proposal/proposal-types.js +31 -24
  82. package/dist/commands/proposal/proposal.js +38 -8
  83. package/dist/commands/proposal/propose.js +134 -160
  84. package/dist/commands/proposal/repository.js +1097 -1394
  85. package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
  86. package/dist/commands/proposal/validators/proposal-validators.js +1 -1
  87. package/dist/commands/proposal/validators/proposals.js +22 -89
  88. package/dist/commands/read/curate.js +105 -462
  89. package/dist/commands/read/knowledge.js +3 -2
  90. package/dist/commands/read/search-cli.js +16 -33
  91. package/dist/commands/read/search.js +17 -23
  92. package/dist/commands/read/show.js +57 -108
  93. package/dist/commands/sources/bundle-cli.js +25 -2
  94. package/dist/commands/sources/bundle-config-ops.js +4 -0
  95. package/dist/commands/sources/dangerous-env-audit.js +1 -2
  96. package/dist/commands/sources/info.js +127 -29
  97. package/dist/commands/sources/installed-stashes.js +197 -746
  98. package/dist/commands/sources/schema-repair.js +98 -129
  99. package/dist/commands/sources/source-add.js +62 -12
  100. package/dist/commands/sources/source-manage.js +9 -2
  101. package/dist/commands/sources/stash-cli.js +24 -4
  102. package/dist/commands/tasks/explain.js +10 -13
  103. package/dist/commands/tasks/tasks-cli.js +12 -13
  104. package/dist/commands/tasks/tasks.js +350 -936
  105. package/dist/commands/tasks/validate.js +26 -24
  106. package/dist/commands/workflow/plan.js +22 -29
  107. package/dist/commands/workflow-cli.js +4 -4
  108. package/dist/core/adapter/adapters/akm-adapter.js +2 -1
  109. package/dist/core/adapter/adapters/akm-lint.js +2 -3
  110. package/dist/core/adapter/adapters/akm-metadata.js +42 -12
  111. package/dist/core/adapter/adapters/akm-task-adapter.js +29 -8
  112. package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
  113. package/dist/core/adapter/execution-source.js +17 -29
  114. package/dist/core/asset/asset-placement.js +4 -13
  115. package/dist/core/asset/frontmatter.js +106 -1
  116. package/dist/core/asset/resolve-ref.js +1 -1
  117. package/dist/core/bundle-id.js +42 -5
  118. package/dist/core/bundle-rename.js +285 -0
  119. package/dist/core/config/config-io.js +1 -2
  120. package/dist/core/config/config-schema.js +9 -34
  121. package/dist/core/config/config-walker.js +1 -1
  122. package/dist/core/config/config.js +184 -111
  123. package/dist/core/config/engine-semantics.js +0 -2
  124. package/dist/core/config/legacy-source-shape-shim.js +38 -9
  125. package/dist/core/config/schema/embedding.js +20 -5
  126. package/dist/core/config/schema/engines.js +5 -0
  127. package/dist/core/config/schema/execution.js +1 -1
  128. package/dist/core/config/schema/experimental.js +1 -1
  129. package/dist/core/config/schema/improve-processes.js +54 -125
  130. package/dist/core/config/schema/improve.js +4 -42
  131. package/dist/core/config/schema/index-config.js +9 -48
  132. package/dist/core/config/schema/scheduler.js +12 -12
  133. package/dist/core/config/schema/search.js +6 -22
  134. package/dist/core/env-secret-ref.js +0 -1
  135. package/dist/core/errors.js +8 -9
  136. package/dist/core/file-change.js +13 -5
  137. package/dist/core/file-lock.js +76 -173
  138. package/dist/core/improve-result.js +35 -7
  139. package/dist/core/improve-types.js +0 -1
  140. package/dist/core/logs-db.js +2 -2
  141. package/dist/core/loopback.js +7 -12
  142. package/dist/core/non-task-input.js +20 -0
  143. package/dist/core/parse.js +13 -16
  144. package/dist/core/paths.js +0 -24
  145. package/dist/core/redaction.js +109 -2
  146. package/dist/core/run-lock.js +2 -5
  147. package/dist/core/spawn-env.js +1 -1
  148. package/dist/core/state/migrations.js +123 -61
  149. package/dist/core/state-db-scope.js +2 -4
  150. package/dist/core/state-db.js +126 -692
  151. package/dist/core/time.js +0 -20
  152. package/dist/core/type-presentation.js +1 -9
  153. package/dist/core/write-source.js +294 -1005
  154. package/dist/execution/input-contract.js +1 -1
  155. package/dist/execution/resolved-request.js +135 -689
  156. package/dist/execution/source.js +63 -257
  157. package/dist/execution/target-ref.js +1 -1
  158. package/dist/indexer/bundle-identity-guard.js +2 -2
  159. package/dist/indexer/db/llm-cache.js +2 -2
  160. package/dist/indexer/ensure-index.js +77 -73
  161. package/dist/indexer/index-rebuild-lock.js +3 -11
  162. package/dist/indexer/index-writer-lock.js +8 -17
  163. package/dist/indexer/index-written-assets.js +141 -154
  164. package/dist/indexer/indexer.js +400 -1124
  165. package/dist/indexer/links/declared-links.js +90 -0
  166. package/dist/indexer/materialize-embeddings.js +60 -397
  167. package/dist/indexer/passes/memory-inference.js +96 -90
  168. package/dist/indexer/passes/metadata.js +132 -219
  169. package/dist/indexer/read-preflight.js +0 -7
  170. package/dist/indexer/scan/doc-to-entry.js +2 -3
  171. package/dist/indexer/scan/drain-dir.js +1 -1
  172. package/dist/indexer/search/db-search.js +190 -590
  173. package/dist/indexer/search/fts-query.js +30 -41
  174. package/dist/indexer/search/ranking.js +28 -154
  175. package/dist/indexer/search/search-attribution.js +12 -32
  176. package/dist/indexer/search/search-fields.js +11 -15
  177. package/dist/indexer/search/search-hit-enrichers.js +54 -85
  178. package/dist/indexer/search/search-source.js +1 -4
  179. package/dist/indexer/usage/usage-events.js +36 -7
  180. package/dist/indexer/walk/walker.js +3 -4
  181. package/dist/integrations/agent/engine-fallback.js +23 -40
  182. package/dist/integrations/agent/engine-resolution.js +93 -183
  183. package/dist/integrations/agent/execution.js +507 -0
  184. package/dist/integrations/agent/model-map.js +28 -156
  185. package/dist/integrations/agent/request-lowering.js +66 -141
  186. package/dist/integrations/agent/runner-dispatch.js +143 -321
  187. package/dist/integrations/agent/runner.js +54 -14
  188. package/dist/integrations/lockfile.js +53 -101
  189. package/dist/llm/client.js +18 -6
  190. package/dist/llm/embedders/deterministic.js +2 -3
  191. package/dist/llm/embedders/profile.js +71 -0
  192. package/dist/llm/embedders/remote.js +11 -17
  193. package/dist/llm/feature-gate.js +0 -8
  194. package/dist/llm/index-passes.js +3 -5
  195. package/dist/llm/memory-infer.js +1 -2
  196. package/dist/llm/structured-call.js +5 -24
  197. package/dist/output/generic-render.js +23 -11
  198. package/dist/output/html-render.js +13 -10
  199. package/dist/output/render-registry.js +3 -32
  200. package/dist/output/shapes/helpers.js +25 -38
  201. package/dist/output/shapes/passthrough.js +1 -9
  202. package/dist/{indexer/graph/graph-types.js → output/text/bundle-rename.js} +4 -1
  203. package/dist/output/text/command-format.js +69 -31
  204. package/dist/output/text/helpers.js +1 -1
  205. package/dist/output/text/migrate.js +5 -14
  206. package/dist/output/text/proposal-format.js +48 -3
  207. package/dist/output/text/show-format.js +13 -17
  208. package/dist/output/text/workflow-format.js +0 -32
  209. package/dist/output/text.js +2 -0
  210. package/dist/registry/factory.js +4 -19
  211. package/dist/registry/network.js +66 -220
  212. package/dist/registry/providers/index.js +0 -2
  213. package/dist/registry/providers/skills-sh.js +3 -14
  214. package/dist/registry/providers/static-index.js +24 -26
  215. package/dist/registry/resolve.js +55 -131
  216. package/dist/scripts/akm-migrate-node.js +42948 -92369
  217. package/dist/scripts/akm-migrate.js +42935 -92354
  218. package/dist/setup/registry-stash-loader.js +4 -13
  219. package/dist/setup/semantic-assets.js +3 -44
  220. package/dist/setup/setup.js +1 -1
  221. package/dist/setup/steps/connection.js +5 -6
  222. package/dist/setup/steps/platforms.js +2 -2
  223. package/dist/setup/steps/tasks.js +25 -15
  224. package/dist/sources/provider-factory.js +17 -18
  225. package/dist/sources/providers/filesystem.js +2 -3
  226. package/dist/sources/providers/git-install.js +7 -1
  227. package/dist/sources/providers/git-provider.js +0 -3
  228. package/dist/sources/providers/git-stash.js +83 -21
  229. package/dist/sources/providers/npm.js +2 -4
  230. package/dist/sources/providers/provider-utils.js +5 -10
  231. package/dist/sources/providers/website.js +0 -2
  232. package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
  233. package/dist/sources/website-url.js +2 -2
  234. package/dist/storage/database.js +9 -35
  235. package/dist/storage/repositories/improve-ledger-repository.js +209 -0
  236. package/dist/storage/repositories/index-connection.js +39 -72
  237. package/dist/storage/repositories/index-entries-repository.js +131 -129
  238. package/dist/storage/repositories/index-entry-mapper.js +1 -2
  239. package/dist/storage/repositories/index-entry-schema.js +101 -268
  240. package/dist/storage/repositories/index-fts-repository.js +86 -256
  241. package/dist/storage/repositories/index-links-repository.js +143 -0
  242. package/dist/storage/repositories/index-llm-cache-repository.js +7 -9
  243. package/dist/storage/repositories/index-meta-repository.js +6 -4
  244. package/dist/storage/repositories/index-schema.js +257 -325
  245. package/dist/storage/repositories/index-utility-repository.js +8 -29
  246. package/dist/storage/repositories/index-vec-repository.js +133 -414
  247. package/dist/storage/repositories/outcome-repository.js +2 -1
  248. package/dist/storage/repositories/proposals-repository.js +104 -1
  249. package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
  250. package/dist/storage/repositories/salience-repository.js +1 -19
  251. package/dist/storage/repositories/task-history-repository.js +26 -4
  252. package/dist/storage/repositories/workflow-runs-repository.js +53 -244
  253. package/dist/storage/sqlite-migrations.js +136 -0
  254. package/dist/storage/sqlite-pragmas.js +11 -9
  255. package/dist/storage/sqlite-transaction.js +170 -0
  256. package/dist/storage/state-db-integrity.js +130 -0
  257. package/dist/tasks/activation-config.js +134 -62
  258. package/dist/tasks/backends/cron.js +191 -302
  259. package/dist/tasks/backends/exec-utils.js +2 -5
  260. package/dist/tasks/backends/launchd.js +141 -748
  261. package/dist/tasks/backends/schtasks.js +119 -623
  262. package/dist/tasks/prepare/prepare-support.js +5 -15
  263. package/dist/tasks/prepare/prepare.js +0 -2
  264. package/dist/tasks/resolve-akm-bin.js +20 -79
  265. package/dist/tasks/run/attempt-lifecycle.js +0 -1
  266. package/dist/tasks/run/load-task.js +1 -1
  267. package/dist/tasks/scheduler-binding.js +20 -238
  268. package/dist/tasks/scheduler-invocation.js +136 -244
  269. package/dist/tasks/scheduler-lock.js +53 -0
  270. package/dist/tasks/scheduler-sync.js +368 -679
  271. package/dist/tasks/source/parse-task-source.js +55 -9
  272. package/dist/tasks/source/task-source-v3-frozen.js +3 -4
  273. package/dist/tasks/source/task-to-v4.js +464 -88
  274. package/dist/workflows/authoring/authoring.js +3 -12
  275. package/dist/workflows/compile.js +211 -0
  276. package/dist/workflows/concurrency-policy.js +13 -74
  277. package/dist/workflows/exec/child-invocation.js +3 -17
  278. package/dist/workflows/exec/child-workflow.js +32 -141
  279. package/dist/workflows/exec/dispatch-redaction.js +13 -53
  280. package/dist/workflows/exec/environment.js +98 -0
  281. package/dist/workflows/exec/exec-unit.js +33 -140
  282. package/dist/workflows/exec/frozen-judge.js +7 -59
  283. package/dist/workflows/exec/native-executor.js +82 -341
  284. package/dist/workflows/exec/param-secrets.js +29 -47
  285. package/dist/workflows/exec/run-workflow.js +154 -387
  286. package/dist/workflows/exec/scheduler.js +9 -36
  287. package/dist/workflows/exec/step-work.js +127 -430
  288. package/dist/workflows/exec/unit-dispatch.js +11 -63
  289. package/dist/workflows/exec/unit-writer.js +8 -52
  290. package/dist/workflows/exec/worktree.js +39 -273
  291. package/dist/workflows/freeze/child-output-references.js +4 -15
  292. package/dist/workflows/freeze/environment.js +99 -92
  293. package/dist/workflows/freeze/freeze.js +172 -0
  294. package/dist/workflows/freeze/step-values.js +19 -21
  295. package/dist/workflows/freeze/targets/child-workflow.js +23 -92
  296. package/dist/workflows/freeze/targets/command.js +10 -33
  297. package/dist/workflows/freeze/targets/script.js +5 -12
  298. package/dist/workflows/freeze/targets/shell.js +3 -6
  299. package/dist/workflows/freeze/targets/task.js +25 -80
  300. package/dist/workflows/freeze/task-bindings.js +20 -67
  301. package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
  302. package/dist/workflows/ir/params.js +6 -51
  303. package/dist/workflows/ir/plan-hash.js +2 -34
  304. package/dist/workflows/parser.js +140 -43
  305. package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
  306. package/dist/workflows/renderer.js +36 -69
  307. package/dist/workflows/resource-limits.js +12 -120
  308. package/dist/workflows/runtime/agent-identity.js +8 -40
  309. package/dist/workflows/runtime/run-outputs.js +3 -6
  310. package/dist/workflows/runtime/run-plan.js +316 -0
  311. package/dist/workflows/runtime/runs.js +48 -200
  312. package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
  313. package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
  314. package/dist/workflows/validate-summary.js +2 -7
  315. package/docs/integration/bundling-akm.md +49 -42
  316. package/docs/migration/README.md +1 -0
  317. package/docs/migration/release-notes/0.9.17.md +43 -0
  318. package/docs/migration/v0.9.1-to-v0.9.2.md +23 -7
  319. package/docs/reference/cli.md +232 -135
  320. package/docs/reference/configuration.md +71 -57
  321. package/docs/reference/data-and-telemetry.md +20 -21
  322. package/docs/reference/tasks.md +105 -39
  323. package/docs/reference/workflow-schema.md +14 -18
  324. package/docs/reference/workflows.md +6 -9
  325. package/package.json +1 -1
  326. package/schemas/akm-config.json +115 -738
  327. package/schemas/akm-workflow.json +1 -0
  328. package/dist/assets/improve-strategies/graph-refresh.json +0 -15
  329. package/dist/assets/prompts/contradiction-judge.md +0 -33
  330. package/dist/assets/prompts/graph-extract-system.md +0 -1
  331. package/dist/assets/prompts/graph-extract-user-prompt.md +0 -35
  332. package/dist/assets/prompts/metadata-enhance-system.md +0 -1
  333. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +0 -4
  334. package/dist/commands/health/advisories.js +0 -150
  335. package/dist/commands/health/metrics.js +0 -329
  336. package/dist/commands/health/surfaces.js +0 -102
  337. package/dist/commands/improve/anti-collapse.js +0 -83
  338. package/dist/commands/improve/collapse-detector.js +0 -432
  339. package/dist/commands/improve/consolidate/eligibility.js +0 -48
  340. package/dist/commands/improve/consolidate/merge.js +0 -149
  341. package/dist/commands/improve/distill/promote-memory.js +0 -291
  342. package/dist/commands/improve/distill/quality-gate.js +0 -337
  343. package/dist/commands/improve/eval-cases.js +0 -52
  344. package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
  345. package/dist/commands/improve/proposal-envelope.js +0 -31
  346. package/dist/commands/improve/run-context.js +0 -123
  347. package/dist/commands/improve/shared.js +0 -31
  348. package/dist/commands/improve/source-identity.js +0 -28
  349. package/dist/commands/improve/triage.js +0 -96
  350. package/dist/commands/proposal/drain-policies.js +0 -151
  351. package/dist/commands/sources/update-transaction.js +0 -220
  352. package/dist/core/action-contributors.js +0 -28
  353. package/dist/core/config/config-version-shim.js +0 -101
  354. package/dist/core/fs-txn.js +0 -405
  355. package/dist/core/lexical-score.js +0 -25
  356. package/dist/core/maintenance-barrier.js +0 -167
  357. package/dist/execution/executable-identity.js +0 -105
  358. package/dist/execution/guarded-source.js +0 -427
  359. package/dist/indexer/db/graph-db.js +0 -444
  360. package/dist/indexer/graph/graph-boost.js +0 -427
  361. package/dist/indexer/graph/graph-dedup.js +0 -95
  362. package/dist/indexer/graph/graph-extraction.js +0 -1108
  363. package/dist/indexer/search/name-match.js +0 -35
  364. package/dist/indexer/search/ranking-contributors.js +0 -515
  365. package/dist/indexer/search/ranking-types.js +0 -4
  366. package/dist/indexer/walk/project-context.js +0 -192
  367. package/dist/integrations/agent/execution-cascade.js +0 -566
  368. package/dist/integrations/agent/execution-definitions.js +0 -202
  369. package/dist/integrations/agent/execution-lowering.js +0 -841
  370. package/dist/integrations/agent/execution-preparation.js +0 -98
  371. package/dist/integrations/agent/inline-execution.js +0 -74
  372. package/dist/llm/graph-extract.js +0 -728
  373. package/dist/llm/metadata-enhance.js +0 -96
  374. package/dist/registry/create-provider-registry.js +0 -29
  375. package/dist/registry/pinned-request-helper.js +0 -247
  376. package/dist/registry/pinned-transport.js +0 -717
  377. package/dist/sources/providers/index.js +0 -14
  378. package/dist/storage/engines/sqlite-migrations.js +0 -271
  379. package/dist/storage/repositories/canaries-repository.js +0 -107
  380. package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
  381. package/dist/storage/repositories/registry-cache.js +0 -113
  382. package/dist/tasks/scheduler-sync-preview.js +0 -52
  383. package/dist/tasks/source/task-to-v3.js +0 -507
  384. package/dist/workflows/freeze/resolve-steps.js +0 -86
  385. package/dist/workflows/freeze/source-freeze.js +0 -64
  386. package/dist/workflows/ir/compile.js +0 -321
  387. package/dist/workflows/ir/environment-v4.js +0 -330
  388. package/dist/workflows/ir/freeze-v4.js +0 -153
  389. package/dist/workflows/ir/schema-v4.js +0 -745
  390. package/dist/workflows/ir/schema.js +0 -354
  391. package/dist/workflows/program/schema.js +0 -77
  392. package/dist/workflows/runtime/checkin.js +0 -57
  393. package/dist/workflows/runtime/plan-classifier.js +0 -196
  394. package/dist/workflows/runtime/unit-checkin.js +0 -45
  395. package/dist/workflows/runtime/unit-phases.js +0 -20
  396. package/dist/workflows/schema.js +0 -4
  397. package/dist/workflows/source-ir/compile.js +0 -200
  398. package/dist/workflows/source-ir/program.js +0 -50
  399. package/dist/workflows/source-ir/result.js +0 -26
  400. package/dist/workflows/source-ir/schema.js +0 -786
  401. package/dist/workflows/source-ir/triggers.js +0 -79
  402. package/dist/workflows/source-ir/uses.js +0 -40
  403. package/dist/workflows/validator.js +0 -60
@@ -1,114 +1,95 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /** The improve loop (reflect + distill per ref), the post-loop checks and the maintenance passes. */
4
5
  import fs from "node:fs";
5
- import path from "node:path";
6
6
  import { parseRefInput } from "../../core/asset/resolve-ref.js";
7
7
  import { daysToMs } from "../../core/common.js";
8
- import { DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE, loadConfig } from "../../core/config/config.js";
8
+ import { loadConfig } from "../../core/config/config.js";
9
9
  import { UsageError } from "../../core/errors.js";
10
10
  import { appendEvent } from "../../core/events.js";
11
11
  import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
12
12
  import { getDbPath, getTaskLogDir } from "../../core/paths.js";
13
13
  import { withStateDb } from "../../core/state-db.js";
14
14
  import { info } from "../../core/warn.js";
15
- import { DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES, runGraphExtractionPass, } from "../../indexer/graph/graph-extraction.js";
16
- import { deriveWritableBundleIds } from "../../indexer/installations.js";
15
+ import { indexWrittenAssets } from "../../indexer/index-written-assets.js";
17
16
  import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
18
17
  import { resolveSourceEntries } from "../../indexer/search/search-source.js";
19
- import { isProcessEnabled } from "../../llm/feature-gate.js";
20
- import { withLlmStage } from "../../llm/usage-telemetry.js";
21
- import { purgeOldCycleMetrics } from "../../storage/repositories/canaries-repository.js";
22
18
  import { purgeOldEvents } from "../../storage/repositories/events-repository.js";
23
19
  import { purgeOldImproveRuns } from "../../storage/repositories/improve-runs-repository.js";
24
20
  import { closeDatabase, openIndexDatabase } from "../../storage/repositories/index-connection.js";
25
- import { getEntryByRef } from "../../storage/repositories/index-entries-repository.js";
21
+ import { getLiveRefSnapshot, isRefLiveInSnapshot, } from "../../storage/repositories/index-entries-repository.js";
26
22
  import { clearAssetOutcomeMissing, countAssetOutcomeMissing, deleteAssetOutcomeMissingBefore, listAssetOutcomeMissingState, stampAssetOutcomeMissing, } from "../../storage/repositories/outcome-repository.js";
27
23
  import { clearAssetSalienceMissing, countAssetSalienceMissing, deleteAssetSalienceMissingBefore, listAssetSalienceMissingState, stampAssetSalienceMissing, } from "../../storage/repositories/salience-repository.js";
24
+ import { readFreelistInfo, STATE_DB_VACUUMED_EVENT, vacuumIfReclaimable } from "../../storage/state-db-integrity.js";
28
25
  import { purgeOldTaskLogFiles } from "../../tasks/run/task-log.js";
29
- import { expireStaleProposals, listProposals, purgeOrphanProposals } from "../proposal/repository.js";
26
+ import { expireStaleProposals, purgeOrphanProposals } from "../proposal/repository.js";
30
27
  import { checkDeadUrls } from "../url-checker.js";
31
- import { DEFAULT_RETENTION_DAYS as CYCLE_METRICS_RETENTION_DAYS, runCollapseDetector } from "./collapse-detector.js";
32
- import { deriveLessonRef } from "./distill.js";
33
- import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
34
- // Eligibility / candidate-selection predicates live in ./eligibility.
35
- import { findAssetFilePath, isDistillCandidateRef } from "./eligibility.js";
36
- import { writeEvalCase } from "./eval-cases.js";
28
+ import { isDistillCandidateRef } from "./eligibility.js";
37
29
  import { shouldSkipRef } from "./improve-strategies.js";
30
+ import { recordLedgerAttempt, stateKey, stripBundle } from "./ledger.js";
31
+ import { pushRecentError } from "./preparation.js";
38
32
  import { recordNoOp, resetConsecutiveNoOps } from "./salience.js";
39
- import { errMessage, refSlug } from "./shared.js";
40
- import { bareImproveRef, durableImproveRef } from "./source-identity.js";
41
- // ── improve loop / post-loop / maintenance stages ───────────────────
42
- // The cycle stages run by akmImprove, extracted from improve.ts.
43
- /** O-5 / #378: rolling per-originator error-window cap. */
44
- const RECENT_ERRORS_CAP = 3;
45
- /** O-5 / #378: push a per-originator error into the rolling window. */
46
- function pushRecentError(recentErrors, originator, msg) {
47
- if (!recentErrors[originator])
48
- recentErrors[originator] = [];
49
- recentErrors[originator].push(msg);
50
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
51
- recentErrors[originator].shift();
52
- }
53
- /**
54
- * Build the per-run loop environment from the run context: the derived guards
55
- * and the pending-proposal preload.
56
- */
33
+ import { attributeStage, errMessage } from "./stage.js";
57
34
  export function prepareImproveLoopEnv(args) {
58
- const { ctx, scope, options, reflectFn, distillFn, loopRefs, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, startMs, budgetMs, improveProfile, resolvedPlan, } = args;
59
- // WI-9.10: the legacy dual context's optional eventsCtx/budgetSignal are
60
- // now RunContext's required eventsCtx / optional signal — renamed local
61
- // aliases so the rest of this function (and the ImproveLoopEnv object
62
- // literal below) is unchanged. `primaryStashDir` deliberately comes from
63
- // the state's honest optional field, NOT `ctx.stashDir`: the rare
64
- // unresolvable-primary path must keep skipping the `if (primaryStashDir)`
65
- // guards below (see the field's doc in ./improve-run-types).
66
- const primaryStashDir = args.primaryStashDir;
67
- const eventsCtx = ctx.eventsCtx;
68
- const budgetSignal = ctx.signal;
69
- // O-1 (#364): compute remaining budget at call time so each sub-call
70
- // receives only its fair share of the wall-clock budget.
71
- const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
72
- // Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
73
- const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
74
- // requirePlannedRefs guard: when the distill profile sets this flag, skip
75
- // distill for distill-only refs if the reflect phase produced no planned refs.
76
- // Prevents the distill loop from generating hundreds of distill-skipped events
77
- // on quiet passes (all refs on reflect cooldown, no new signal to distill).
78
- const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
79
- const hasReflectEligibleRefs = loopRefs.some((r) => !distillOnlyRefSet.has(r.ref));
80
- const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
81
- // Pre-load all pending proposals once instead of querying per asset in the loop.
82
- const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
83
- const pendingProposalRefSet = new Set(dedupeStashDirForProposals
84
- ? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
85
- : []);
35
+ const distillOnlyRefSet = new Set(args.distillOnlyRefs.map((r) => r.ref));
36
+ const requirePlannedRefs = args.improveProfile?.processes?.distill?.requirePlannedRefs === true;
86
37
  return {
87
- scope,
88
- options,
89
- primaryStashDir,
90
- reflectFn,
91
- distillFn,
92
- signalBearingSet,
93
- distillCooledRefs,
38
+ scope: args.scope,
39
+ options: args.options,
40
+ primaryStashDir: args.primaryStashDir,
41
+ reflectFn: args.reflectFn,
42
+ distillFn: args.distillFn,
43
+ signalBearingSet: args.signalBearingSet,
44
+ distillCooledRefs: args.distillCooledRefs,
94
45
  distillOnlyRefSet,
95
- recentErrors,
96
- rejectedProposalsByRef,
97
- eventsCtx,
98
- improveProfile,
99
- resolvedPlan,
100
- budgetSignal,
101
- skipDistillDueToRequirePlannedRefs,
102
- pendingProposalRefSet,
103
- remainingBudgetMs,
46
+ recentErrors: args.recentErrors,
47
+ eventsCtx: args.eventsCtx,
48
+ improveProfile: args.improveProfile,
49
+ resolvedPlan: args.resolvedPlan,
50
+ budgetSignal: args.budgetSignal,
51
+ skipDistillDueToRequirePlannedRefs: requirePlannedRefs && args.loopRefs.every((r) => distillOnlyRefSet.has(r.ref)),
52
+ remainingBudgetMs: () => Math.max(0, args.budgetMs - (Date.now() - args.startMs)),
104
53
  };
105
54
  }
106
55
  /**
107
- * One improve-loop iteration for a single planned ref: the reflect pass, then
108
- * the distill pass, with the per-ref error classification (B7) around both.
109
- * `continue` in the old inline loop body is an early `return` inside the
110
- * passes; the orchestrator folds the returned tally and owns the run counters.
56
+ * Record a loop attempt in the improve ledger. Proposals and quality
57
+ * rejections record themselves; the loop records what they cannot see.
111
58
  */
59
+ function recordLoopAttempt(planned, env, source, outcome, detail) {
60
+ const stashDir = env.primaryStashDir ?? env.options.stashDir;
61
+ if (!stashDir || env.options.dryRun)
62
+ return;
63
+ recordLedgerAttempt({ eventsCtx: env.eventsCtx }, {
64
+ stashDir,
65
+ ref: stateKey(planned.ref, planned.itemRef),
66
+ source,
67
+ outcome,
68
+ ...(detail !== undefined ? { detail } : {}),
69
+ });
70
+ }
71
+ /** Plasticity counter: repeated no-ops dampen an asset's selection score; a change lifts it. */
72
+ function recordPlasticity(env, planned, outcome) {
73
+ const db = env.eventsCtx?.db;
74
+ if (!db || !outcome)
75
+ return;
76
+ const key = stateKey(planned.ref, planned.itemRef);
77
+ try {
78
+ if (outcome === "noop")
79
+ recordNoOp(db, key);
80
+ else
81
+ resetConsecutiveNoOps(db, key);
82
+ }
83
+ catch {
84
+ // best-effort
85
+ }
86
+ }
87
+ function recordSkip(tally, ref, reason, event) {
88
+ tally.actions.push({ ref, mode: "distill-skipped", result: { ok: true, reason } });
89
+ if (event)
90
+ appendEvent({ eventType: "improve_skipped", ref, metadata: { reason: event.reason } }, event.env.eventsCtx);
91
+ }
92
+ /** One loop iteration: reflect, then distill. A distill UsageError is a validation failure. */
112
93
  export async function processImproveLoopRef(planned, env) {
113
94
  const tally = {
114
95
  actions: [],
@@ -117,20 +98,15 @@ export async function processImproveLoopRef(planned, env) {
117
98
  memoryRefsForInference: [],
118
99
  };
119
100
  try {
120
- // Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
121
- // Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
122
- // have their synthetic actions emitted in runImprovePreparationStage.
123
101
  const isDistillOnly = env.distillOnlyRefSet.has(planned.ref);
124
- const parsedPlannedRef = parseRefInput(planned.ref);
125
- await runLoopReflectPass(planned, isDistillOnly, env, tally);
126
- // isDistillOnly refs: no reflect action emitted — proceed directly to the distill pass.
127
- await runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally);
102
+ const parsed = parseRefInput(planned.ref);
103
+ if (!isDistillOnly)
104
+ await runLoopReflectPass(planned, env, tally);
105
+ await runLoopDistillPass(planned, parsed.type, isDistillOnly, env, tally);
128
106
  }
129
107
  catch (err) {
130
- // B7: UsageError thrown by akmDistill on validation_failed should be recorded
131
- // as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
132
- // The distill_invoked event was already emitted inside akmDistill before the throw.
133
108
  if (err instanceof UsageError) {
109
+ recordLoopAttempt(planned, env, "distill", "failed", err.message);
134
110
  tally.actions.push({
135
111
  ref: planned.ref,
136
112
  mode: "distill",
@@ -138,448 +114,177 @@ export async function processImproveLoopRef(planned, env) {
138
114
  });
139
115
  }
140
116
  else {
141
- tally.actions.push({
142
- ref: planned.ref,
143
- mode: "error",
144
- result: { ok: false, error: errMessage(err) },
145
- });
117
+ tally.actions.push({ ref: planned.ref, mode: "error", result: { ok: false, error: errMessage(err) } });
146
118
  }
147
119
  }
148
120
  return tally;
149
121
  }
150
- /**
151
- * Reflect half of one loop iteration: type/profile gates, the reflect call with
152
- * recent-error avoidPatterns, outcome classification (cooldown / guard-reject /
153
- * type-refused / noise-gate), and plasticity counters. Records onto the per-ref
154
- * tally only.
155
- */
156
- async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
157
- const { options, primaryStashDir, reflectFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
158
- // B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
159
- // shouldDistillMemoryRef already returns false for .derived refs, so the distill
160
- // path is also a no-op for them — we just avoid unnecessary agent spawns.
161
- // D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
162
- if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
163
- // Type guard: skip reflect for unsupported types (script, env, task, etc.)
164
- // and raw wiki directories, driven by the active improve profile.
165
- const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
166
- if (reflectSkip.skip) {
167
- tally.actions.push({
168
- ref: planned.ref,
169
- mode: "reflect-skipped",
170
- result: { ok: true, reason: reflectSkip.reason },
171
- });
172
- }
173
- else {
174
- // O-5 / #378: only inject reflect-originator errors into the reflect call.
175
- // Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
176
- const reflectErrors = env.recentErrors.reflect ?? [];
177
- if (reflectErrors.length > 0)
178
- tally.reflectsWithErrorContext++;
179
- // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
180
- // bounded by the wall-clock deadline rather than the default per-profile timeout.
181
- const reflectBudgetMs = env.remainingBudgetMs();
182
- // Re-enter canonical named-engine lowering with the config snapshot and
183
- // process profile frozen into the invocation plan. The loop never injects
184
- // a RunnerSpec seam or observes later caller-config mutations.
185
- const reflectCallArgs = {
186
- ref: planned.ref,
187
- // Carry the resolved item_ref so reflect uses the same durable key for
188
- // events and state reads.
189
- ...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
190
- task: options.task,
191
- // Active strategy supplies non-engine process tuning.
192
- ...(improveProfile ? { improveProfile } : {}),
193
- config: resolvedPlan.config,
194
- ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
195
- ...(options.sourceName && primaryStashDir
196
- ? { target: { source: options.sourceName, root: primaryStashDir } }
197
- : {}),
198
- ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
199
- eventSource: "improve",
200
- // #639 — resolve the low-value filter from the ACTIVE improve profile
201
- // (default off when unset), so the running strategy decides.
202
- lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
203
- ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
204
- signal: budgetSignal,
205
- // R25: reflect's event emits reuse the run's long-lived state.db handle.
206
- eventsCtx: env.eventsCtx,
207
- // Attribution: carry the eligibility lane so reflect stamps it on
208
- // the reflect_invoked event and the persisted proposal.
209
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
210
- };
211
- const reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs), {
212
- engine: resolvedPlan.processes.reflect.runner?.engine,
213
- process: "reflect",
214
- });
215
- const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
216
- // Content-policy guard hits (reflect size-rail rejections) are NOT
217
- // LLM faults — the agent responded fine, the downstream guard
218
- // blocked the output. Route them to a distinct `reflect-guard-rejected`
219
- // mode so health metrics can split deterministic guard hits out of
220
- // true LLM failures. See
221
- // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
222
- const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
223
- // Type-guard rejection (reflect refused a script/env/task ref) is
224
- // also NOT an LLM failure — the LLM is never invoked. Route to the
225
- // existing `reflect-skipped` bucket so it does not inflate the
226
- // failure-rate numerator. ~9% of `reflect-failed` events in the
227
- // user's stack were this case; see review §1a row "Reflect refused
228
- // asset type".
229
- const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
230
- // Noise-gate suppression (#580): the candidate edit was an empty
231
- // diff or a cosmetic-only reformat of the current asset. Like
232
- // `unsupported_type`, this is a deterministic skip — not an LLM
233
- // fault — so it routes to the `reflect-skipped` bucket and stays
234
- // out of recentErrors/avoidPatterns.
235
- const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
236
- tally.actions.push({
237
- ref: planned.ref,
238
- mode: reflectResult.ok
239
- ? "reflect"
240
- : isCooldown
241
- ? "reflect-cooldown"
242
- : isGuardReject
243
- ? "reflect-guard-rejected"
244
- : isTypeRefused || isNoChange
245
- ? "reflect-skipped"
246
- : "reflect-failed",
247
- result: reflectResult,
248
- });
249
- // Cooldown skips, guard rejects, type-refused skips, and noise-gate
250
- // skips are not failures — do not pollute recentErrors with them
251
- // (those get injected as `avoidPatterns` into the next reflect
252
- // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
253
- // so the next iteration sees "your last expansion was too large";
254
- // type-refused and no-change are deterministic and add no learning
255
- // signal.
256
- if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
257
- const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
258
- tally.recentErrorPushes.push({ originator: "reflect", message: errMsg });
259
- }
260
- // improve_reflect_outcome — per-asset metric for tuning the reflect path.
261
- appendEvent({
262
- eventType: "improve_reflect_outcome",
263
- ref: planned.ref,
264
- metadata: {
265
- ok: reflectResult.ok,
266
- durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
267
- engine: reflectResult.engine,
268
- reason: reflectResult.ok ? undefined : reflectResult.reason,
269
- },
270
- }, eventsCtx);
271
- // Plasticity counter (plan §WS-1 step 8): record no-ops so the
272
- // WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
273
- // repeatedly-silent assets during consolidation-selection.
274
- // A no_change reflect means the LLM was invoked but found nothing to
275
- // improve — the asset is stable. Track it. A successful reflect means
276
- // the asset changed; reset the counter so the dampener lifts.
277
- // Use the same item_ref-or-conceptId salience key as preparation/distill.
278
- const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
279
- if (isNoChange && eventsCtx?.db) {
280
- try {
281
- recordNoOp(eventsCtx.db, plasticityKey);
282
- }
283
- catch {
284
- // best-effort: plasticity counter failure never blocks the run
285
- }
286
- }
287
- else if (reflectResult.ok && eventsCtx?.db) {
288
- try {
289
- resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
290
- }
291
- catch {
292
- // best-effort
293
- }
294
- }
295
- } // end else (reflect type/profile check)
296
- }
297
- else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
298
- // B6: .derived refs skip reflect; record synthetic skip action.
299
- tally.actions.push({
300
- ref: planned.ref,
301
- mode: "distill-skipped",
302
- result: { ok: true, reason: "derived-memory-reflect-skipped" },
303
- });
304
- appendEvent({
305
- eventType: "improve_skipped",
306
- ref: planned.ref,
307
- metadata: { reason: "derived_memory_reflect_skipped" },
308
- }, eventsCtx);
309
- }
310
- }
311
- /**
312
- * Distill half of one loop iteration: the profile / requirePlannedRefs /
313
- * candidate-type / weak-signal / cooldown gates, then the pending-proposal and
314
- * reject-grace dedup checks, then {@link invokeDistillAndRecord}. Each gate
315
- * that was a `continue` in the old inline loop body is an early `return` here.
316
- */
317
- async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally) {
318
- const { options, primaryStashDir, eventsCtx, improveProfile } = env;
319
- const hasRecentFeedbackSignal = env.signalBearingSet.has(planned.ref);
320
- const explicitRefScope = env.scope.mode === "ref";
321
- // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
322
- // distill so callers who configure `profile.processes.distill.allowedTypes`
323
- // or land on raw-wiki refs get a recorded skip action instead of silently
324
- // proceeding.
325
- const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
326
- if (distillSkip.skip) {
327
- tally.actions.push({
328
- ref: planned.ref,
329
- mode: "distill-skipped",
330
- result: { ok: true, reason: distillSkip.reason },
331
- });
122
+ async function runLoopReflectPass(planned, env, tally) {
123
+ const { options, primaryStashDir, improveProfile, resolvedPlan } = env;
124
+ // Derived memories are machine-generated: never reflected.
125
+ if (planned.ref.endsWith(".derived")) {
126
+ recordSkip(tally, planned.ref, "derived-memory-reflect-skipped", { env, reason: "derived_memory_reflect_skipped" });
332
127
  return;
333
128
  }
334
- // requirePlannedRefs guard: skip distill for distill-only refs when no
335
- // reflect-eligible refs were planned this run, preventing mass skip events.
336
- if (env.skipDistillDueToRequirePlannedRefs && isDistillOnly) {
337
- tally.actions.push({
338
- ref: planned.ref,
339
- mode: "distill-skipped",
340
- result: { ok: true, reason: "require_planned_refs" },
341
- });
129
+ const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
130
+ if (reflectSkip.skip) {
131
+ tally.actions.push({ ref: planned.ref, mode: "reflect-skipped", result: { ok: true, reason: reflectSkip.reason } });
342
132
  return;
343
133
  }
344
- // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
345
- // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
346
- // it will refuse.
347
- const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
348
- const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
349
- // distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
350
- // refs; non-candidate refs in the set are blocked here.
351
- // O-2 (#365): bypass the distill cooldown when the user explicitly targeted
352
- // this ref via --scope — their intent overrides unattended-run policies.
353
- if (shouldAttemptDistill &&
354
- !skipMemoryDistillForWeakSignal &&
355
- (!env.distillCooledRefs.has(planned.ref) || explicitRefScope)) {
356
- // TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
357
- const lessonRef = deriveLessonRef(planned.ref);
358
- const knowledgeRef = deriveKnowledgeRef(planned.ref);
359
- const dedupeStashDir = primaryStashDir ?? options.stashDir;
360
- if (dedupeStashDir) {
361
- // B2: check both lesson ref and knowledge ref since auto-promoted memories
362
- // create knowledge: proposals, not lesson: proposals.
363
- const hasExistingPending = env.pendingProposalRefSet.has(lessonRef) || env.pendingProposalRefSet.has(knowledgeRef);
364
- if (hasExistingPending) {
365
- tally.actions.push({
366
- ref: planned.ref,
367
- mode: "distill-skipped",
368
- result: { ok: true, reason: "pending proposal exists" },
369
- });
370
- appendEvent({
371
- eventType: "improve_skipped",
372
- ref: planned.ref,
373
- metadata: { reason: "pending_proposal_exists" },
374
- }, eventsCtx);
375
- return;
376
- }
377
- // D-2 (#370): reject-aware cooldown for distill. When the reviewer
378
- // recently rejected a distilled lesson or knowledge proposal for this
379
- // asset, skip re-distillation for a 1-day grace window. Prevents the
380
- // same rejected proposal from being regenerated immediately. The
381
- // window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
382
- // signal-delta gates and dropped --distill-cooldown-days; a short
383
- // reject grace is preserved here so a fresh rejection isn't
384
- // overridden by the same run).
385
- // References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
386
- const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
387
- const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
388
- (env.rejectedProposalsByRef.has(lessonRef) || env.rejectedProposalsByRef.has(knowledgeRef));
389
- if (recentlyRejectedLesson) {
390
- const rejectedEntry = env.rejectedProposalsByRef.get(lessonRef) ?? env.rejectedProposalsByRef.get(knowledgeRef);
391
- const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
392
- if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
393
- tally.actions.push({
394
- ref: planned.ref,
395
- mode: "distill-skipped",
396
- result: { ok: true, reason: "distill reject grace window" },
397
- });
398
- appendEvent({
399
- eventType: "improve_skipped",
400
- ref: planned.ref,
401
- metadata: {
402
- reason: "distill_reject_grace_window",
403
- },
404
- }, eventsCtx);
405
- return;
406
- }
407
- }
408
- }
409
- await invokeDistillAndRecord(planned, parsedPlannedRef, env, tally);
410
- }
411
- else if (skipMemoryDistillForWeakSignal) {
412
- tally.actions.push({
413
- ref: planned.ref,
414
- mode: "distill-skipped",
415
- result: { ok: true, reason: "memory requires recent feedback signal" },
416
- });
417
- appendEvent({
418
- eventType: "improve_skipped",
419
- ref: planned.ref,
420
- metadata: { reason: "memory_distill_requires_feedback" },
421
- }, eventsCtx);
422
- }
423
- }
424
- /**
425
- * The distill invocation for one ref that passed every gate: the `distillFn`
426
- * call, memory-inference queueing, plasticity counters, and the
427
- * quality-rejected / proposal-rejected eval-case writes.
428
- */
429
- async function invokeDistillAndRecord(planned, parsedPlannedRef, env, tally) {
430
- const { options, primaryStashDir, distillFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
431
- const distillResult = await withLlmStage("distill", () => distillFn({
134
+ // Only reflect's own recent errors reach its prompt.
135
+ const reflectErrors = env.recentErrors.reflect ?? [];
136
+ if (reflectErrors.length > 0)
137
+ tally.reflectsWithErrorContext++;
138
+ const budgetMs = env.remainingBudgetMs();
139
+ const reflectArgs = {
432
140
  ref: planned.ref,
433
- // Carry the resolved item_ref so distill matches preparation's state key.
434
141
  ...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
435
- ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
436
- ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
437
- // Active profile so distill's per-process reads honor `--profile`.
142
+ task: options.task,
438
143
  ...(improveProfile ? { improveProfile } : {}),
439
- config: options.config,
440
- llmRunner: resolvedPlan.processes.distill.runner,
441
- signal: budgetSignal,
442
- // R25: distill's event emits reuse the run's long-lived state.db handle.
443
- eventsCtx,
444
- // Attribution: carry the eligibility lane so distill stamps it on the
445
- // distill_invoked event and the persisted proposal.
144
+ config: resolvedPlan.config,
145
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
146
+ ...(options.sourceName && primaryStashDir ? { target: { source: options.sourceName, root: primaryStashDir } } : {}),
147
+ ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
148
+ eventSource: "improve",
149
+ lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
150
+ ...(budgetMs > 0 ? { timeoutMs: budgetMs } : {}),
151
+ signal: env.budgetSignal,
152
+ eventsCtx: env.eventsCtx,
446
153
  ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
447
- }), { engine: resolvedPlan.processes.distill.runner?.engine, process: "distill" });
448
- tally.actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
449
- if (parsedPlannedRef.type === "memory") {
450
- const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
451
- if (!promotedToKnowledge)
452
- tally.memoryRefsForInference.push(planned.ref);
453
- }
454
- // Plasticity counter (plan §WS-1 step 8) for the distill path.
455
- // quality_rejected: the LLM ran but produced output that didn't pass the
456
- // quality gate — the asset is not yielding useful distill output.
457
- // queued: a proposal was produced; reset the no-op counter.
458
- if (eventsCtx?.db) {
459
- // Use the same item_ref-or-conceptId key as the distill/preparation writers.
460
- const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
461
- try {
462
- if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
463
- recordNoOp(eventsCtx.db, plasticityKey);
464
- }
465
- else if (distillResult.outcome === "queued") {
466
- resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
467
- }
468
- }
469
- catch {
470
- // best-effort: plasticity counter failure never blocks the run
154
+ };
155
+ const result = await attributeStage(resolvedPlan, "reflect", () => env.reflectFn(reflectArgs));
156
+ const reason = result.ok ? undefined : result.reason;
157
+ // A refused type or an unchanged asset is a deterministic skip, not an LLM
158
+ // fault; a guard rejection (size rail) gets its own bucket for health.
159
+ const skipped = reason === "unsupported_type" || reason === "no_change";
160
+ tally.actions.push({
161
+ ref: planned.ref,
162
+ mode: result.ok
163
+ ? "reflect"
164
+ : reason === "content_policy_reject"
165
+ ? "reflect-guard-rejected"
166
+ : skipped
167
+ ? "reflect-skipped"
168
+ : "reflect-failed",
169
+ result,
170
+ });
171
+ // A quality rejection recorded itself, and the judge's text is no lesson for
172
+ // the next prompt; skips revisit on the `unchanged` cadence.
173
+ if (!result.ok && reason !== "quality_rejected") {
174
+ recordLoopAttempt(planned, env, "reflect", skipped ? "unchanged" : "failed", reason);
175
+ if (!skipped) {
176
+ tally.recentErrorPushes.push({
177
+ originator: "reflect",
178
+ message: result.error ?? reason ?? "unknown reflect error",
179
+ });
471
180
  }
472
181
  }
473
- if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
474
- const slug = refSlug(planned.ref);
475
- writeEvalCase(primaryStashDir, {
476
- ref: planned.ref,
477
- failureReason: distillResult.reason ?? "quality gate rejected",
478
- assetType: parseRefInput(planned.ref).type ?? "unknown",
479
- rejectedAt: Date.now(),
480
- source: "distill_quality_rejected",
481
- slug: `${slug}-${Date.now()}`,
482
- });
483
- }
484
- // D6: use pre-loaded map instead of per-iteration DB query
485
- const rejectedProposalEvent = env.rejectedProposalsByRef.get(planned.ref);
486
- if (rejectedProposalEvent && primaryStashDir) {
487
- const slug = refSlug(planned.ref);
488
- writeEvalCase(primaryStashDir, {
489
- ref: planned.ref,
490
- failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
491
- assetType: parseRefInput(planned.ref).type ?? "unknown",
492
- rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
493
- source: "proposal_rejected",
494
- slug: `${slug}-rejected`,
495
- });
496
- }
497
- }
498
- /**
499
- * Wall-clock budget exhausted mid-loop (O-1 / #364): emit the improve_skipped
500
- * events for the current and remaining refs (B11) and return the terminal
501
- * error action for the orchestrator to record before breaking out of the loop.
502
- */
503
- function recordBudgetExhausted(args) {
504
- const { planned, loopRefs, completedCount, startMs, eventsCtx } = args;
505
- const remaining = loopRefs.length - completedCount;
506
- info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
507
182
  appendEvent({
508
- eventType: "improve_skipped",
183
+ eventType: "improve_reflect_outcome",
509
184
  ref: planned.ref,
510
185
  metadata: {
511
- reason: "budget_exhausted",
512
- remaining,
186
+ ok: result.ok,
187
+ durationMs: result.ok ? result.durationMs : undefined,
188
+ engine: result.engine,
189
+ reason,
513
190
  },
514
- }, eventsCtx);
515
- // B11: Emit improve_skipped for all remaining assets that will not be processed.
516
- for (const remainingRef of loopRefs.slice(completedCount + 1)) {
517
- appendEvent({
518
- eventType: "improve_skipped",
519
- ref: remainingRef.ref,
520
- metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
521
- }, eventsCtx);
191
+ }, env.eventsCtx);
192
+ recordPlasticity(env, planned, reason === "no_change" ? "noop" : result.ok ? "changed" : undefined);
193
+ }
194
+ async function runLoopDistillPass(planned, refType, isDistillOnly, env, tally) {
195
+ const { options, primaryStashDir, improveProfile, resolvedPlan } = env;
196
+ const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
197
+ if (distillSkip.skip)
198
+ return recordSkip(tally, planned.ref, distillSkip.reason);
199
+ if (env.skipDistillDueToRequirePlannedRefs && isDistillOnly) {
200
+ return recordSkip(tally, planned.ref, "require_planned_refs");
522
201
  }
523
- return {
202
+ const explicitRefScope = env.scope.mode === "ref";
203
+ const weakMemorySignal = !isDistillOnly && refType === "memory" && !env.signalBearingSet.has(planned.ref) && !explicitRefScope;
204
+ if (weakMemorySignal) {
205
+ return recordSkip(tally, planned.ref, "memory requires recent feedback signal", {
206
+ env,
207
+ reason: "memory_distill_requires_feedback",
208
+ });
209
+ }
210
+ // The ledger holds cooled refs; an explicit `--scope` ref overrides it.
211
+ if (!isDistillCandidateRef(planned.ref, options.stashDir))
212
+ return;
213
+ if (env.distillCooledRefs.has(planned.ref) && !explicitRefScope)
214
+ return;
215
+ const result = await attributeStage(resolvedPlan, "distill", () => env.distillFn({
524
216
  ref: planned.ref,
525
- mode: "error",
526
- result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
527
- };
217
+ ...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
218
+ ...(refType === "memory" ? { proposalKind: "auto" } : {}),
219
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
220
+ ...(improveProfile ? { improveProfile } : {}),
221
+ config: options.config,
222
+ llmRunner: resolvedPlan.processes.distill.runner,
223
+ signal: env.budgetSignal,
224
+ eventsCtx: env.eventsCtx,
225
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
226
+ }));
227
+ tally.actions.push({ ref: planned.ref, mode: "distill", result });
228
+ // `queued` and the quality outcomes recorded themselves; a transport failure
229
+ // or a disabled process is not an attempt, so the ref stays eligible.
230
+ if (result.outcome === "skipped") {
231
+ recordLoopAttempt(planned, env, "distill", "unchanged", result.skipReason ?? result.message);
232
+ }
233
+ if (refType === "memory" && !(result.outcome === "queued" && result.proposalKind === "knowledge")) {
234
+ tally.memoryRefsForInference.push(planned.ref);
235
+ }
236
+ recordPlasticity(env, planned, result.outcome === "quality_rejected" || result.outcome === "skipped"
237
+ ? "noop"
238
+ : result.outcome === "queued"
239
+ ? "changed"
240
+ : undefined);
528
241
  }
529
242
  export async function runImproveLoopStage(args) {
530
- const { ctx, loopRefs, actions, recentErrors, startMs, budgetMs } = args;
531
- const eventsCtx = ctx.eventsCtx;
243
+ const { loopRefs, actions, recentErrors, startMs, budgetMs, eventsCtx } = args;
532
244
  const env = prepareImproveLoopEnv(args);
533
- let completedCount = 0;
534
245
  let reflectsWithErrorContext = 0;
535
246
  const memoryRefsForInference = new Set();
536
- for (const planned of loopRefs) {
247
+ for (const [index, planned] of loopRefs.entries()) {
537
248
  if (Date.now() - startMs >= budgetMs) {
538
- actions.push(recordBudgetExhausted({ planned, loopRefs, completedCount, startMs, eventsCtx }));
249
+ const remaining = loopRefs.length - index;
250
+ info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
251
+ appendEvent({ eventType: "improve_skipped", ref: planned.ref, metadata: { reason: "budget_exhausted", remaining } }, eventsCtx);
252
+ for (const rest of loopRefs.slice(index + 1)) {
253
+ appendEvent({
254
+ eventType: "improve_skipped",
255
+ ref: rest.ref,
256
+ metadata: { reason: "budget_exhausted_batch", remaining: remaining - 1 },
257
+ }, eventsCtx);
258
+ }
259
+ actions.push({
260
+ ref: planned.ref,
261
+ mode: "error",
262
+ result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
263
+ });
539
264
  break;
540
265
  }
541
266
  const tally = await processImproveLoopRef(planned, env);
542
- // Fold the per-ref tally into run-level state — the passes never touch it.
543
267
  actions.push(...tally.actions);
544
268
  for (const push of tally.recentErrorPushes)
545
269
  pushRecentError(recentErrors, push.originator, push.message);
546
270
  reflectsWithErrorContext += tally.reflectsWithErrorContext;
547
271
  for (const ref of tally.memoryRefsForInference)
548
272
  memoryRefsForInference.add(ref);
549
- completedCount++;
550
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
273
+ info(`[improve] ${index + 1}/${loopRefs.length} ${planned.ref}`);
551
274
  }
552
275
  return { reflectsWithErrorContext, memoryRefsForInference };
553
276
  }
554
277
  export async function runImprovePostLoopStage(args) {
555
- const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, resolvedPlan, consolidationRan, } = args;
556
- const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
278
+ const { scope, primaryStashDir, actionableRefs } = args;
279
+ const allWarnings = [...args.cleanupWarnings, ...(args.appliedCleanup?.warnings ?? [])];
557
280
  info("[improve] post-loop maintenance starting");
558
- const maintenanceResult = await runImproveMaintenancePasses({
559
- options,
560
- primaryStashDir,
561
- actionableRefs,
562
- memoryRefsForInference,
563
- allWarnings,
564
- reindexFn,
565
- consolidationRan,
566
- // O-1 (#364): forward the budget signal to memory inference + graph extraction.
567
- budgetSignal,
568
- eventsCtx,
569
- improveProfile,
570
- resolvedPlan,
571
- });
281
+ const maintenance = await runImproveMaintenancePasses({ ...args, allWarnings });
572
282
  let deadUrls;
573
283
  let deadUrlCoverage;
574
284
  if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
575
285
  try {
576
- // Every actionable knowledge ref is scanned for URLs — there used to be
577
- // a `.slice(0, 10)` here, capping the scan to the first ten refs while
578
- // `deadUrlCoverage.total` counted only those, so a real bundle reported
579
- // checked === total while most refs were never looked at (#892). URL
580
- // extraction (a regex over already-loaded text) is cheap; it is the
581
- // network requests that are expensive, and those are bounded by
582
- // `checkDeadUrls`'s concurrency limit, not by trimming what gets scanned.
286
+ // Every actionable knowledge ref is scanned; checkDeadUrls bounds the
287
+ // network concurrency (#892).
583
288
  const knowledgeEntries = actionableRefs
584
289
  .filter((r) => {
585
290
  try {
@@ -590,9 +295,6 @@ export async function runImprovePostLoopStage(args) {
590
295
  }
591
296
  })
592
297
  .map((r) => {
593
- // The URL scan needs the document body; filePath is pre-resolved on
594
- // eligible refs at planning time (#591). Best-effort — an unreadable
595
- // or unresolved file contributes no URLs, same as before.
596
298
  let body = "";
597
299
  if (r.filePath) {
598
300
  try {
@@ -616,154 +318,77 @@ export async function runImprovePostLoopStage(args) {
616
318
  // best-effort
617
319
  }
618
320
  }
619
- // ── R5: collapse/churn detector ────────────────────────────────────────────
620
- // One snapshot per QUALIFYING cycle: consolidate processed work. Runs AFTER
621
- // the maintenance reindex so FTS sees the post-merge index. Deterministic,
622
- // observe-only, fail-open (the orchestrator catches everything) — and inert
623
- // on the ~9-in-10 default-profile runs that touch no merges.
624
- let cycleMetrics;
625
- if (!options.dryRun && consolidationRan) {
626
- cycleMetrics = runCollapseDetector({
627
- runId: options.runId ?? "improve-adhoc",
628
- ...(improveProfile ? { improveProfile } : {}),
629
- pass: "consolidate",
630
- mergeFloorViolations: args.consolidationMergeFloorViolations ?? 0,
631
- config: options.config ?? loadConfig(),
632
- ...(eventsCtx ? { eventsCtx } : {}),
633
- });
634
- }
635
321
  return {
636
322
  allWarnings,
637
323
  deadUrls,
638
324
  ...(deadUrlCoverage ? { deadUrlCoverage } : {}),
639
- ...(cycleMetrics ? { cycleMetrics } : {}),
640
- ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
641
- ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
642
- ...(maintenanceResult.actions && maintenanceResult.actions.length > 0
643
- ? { maintenanceActions: maintenanceResult.actions }
644
- : {}),
645
- memoryInferenceDurationMs: maintenanceResult.memoryInferenceDurationMs,
646
- graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
647
- orphansPurged: maintenanceResult.orphansPurged,
648
- proposalsExpired: maintenanceResult.proposalsExpired,
325
+ ...(maintenance.memoryInference ? { memoryInference: maintenance.memoryInference } : {}),
326
+ ...(maintenance.actions && maintenance.actions.length > 0 ? { maintenanceActions: maintenance.actions } : {}),
327
+ memoryInferenceDurationMs: maintenance.memoryInferenceDurationMs,
328
+ orphansPurged: maintenance.orphansPurged,
329
+ proposalsExpired: maintenance.proposalsExpired,
649
330
  };
650
331
  }
651
- // Exported for tests (#584/#585 DB-locking regression coverage); production
652
- // callers reach it only through akmImprove → runImprovePostLoopStage.
332
+ /**
333
+ * Memory inference → index what it wrote → proposal hygiene (orphan purge,
334
+ * expiration) → orphan-state GC → retention purges. Warnings go to
335
+ * `allWarnings`.
336
+ */
653
337
  export async function runImproveMaintenancePasses(args) {
654
- const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, budgetSignal, eventsCtx } = args;
655
- if (!primaryStashDir)
656
- return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
657
- if (budgetSignal?.aborted)
658
- return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
338
+ const { options, primaryStashDir, allWarnings, budgetSignal, eventsCtx } = args;
339
+ if (!primaryStashDir || budgetSignal?.aborted)
340
+ return { memoryInferenceDurationMs: 0 };
659
341
  const config = options.config ?? loadConfig();
660
- const sources = resolveSourceEntries(options.stashDir, config);
661
- const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
662
- const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
663
- const openIndexDb = () => openIndexDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
664
- const dbCell = {};
665
- // #584: see the MaintenanceCtx.reindexWithIndexDbReleased doc — close before
666
- // every reindex, reopen in `finally` so a failed reindex still leaves a
667
- // usable handle in the cell.
668
- const reindexWithIndexDbReleased = async (stashDir) => {
669
- if (dbCell.current) {
670
- closeDatabase(dbCell.current);
671
- dbCell.current = undefined;
672
- }
673
- try {
674
- await reindexFn({ stashDir, signal: budgetSignal });
675
- }
676
- finally {
677
- dbCell.current = openIndexDb();
678
- }
679
- };
680
342
  const ctx = {
681
343
  config,
682
- sources,
344
+ sources: resolveSourceEntries(options.stashDir, config),
683
345
  primaryStashDir,
684
346
  eventsCtx,
685
347
  budgetSignal,
686
348
  improveProfile: args.improveProfile,
687
349
  resolvedPlan: args.resolvedPlan,
688
- memoryInferenceFn,
689
- graphExtractionFn,
690
- reindexWithIndexDbReleased,
350
+ memoryInferenceFn: options.memoryInferenceFn ?? runMemoryInferencePass,
691
351
  };
692
- const collected = await runMaintenancePassesUnderLease(ctx, dbCell, {
693
- actionableRefs: args.actionableRefs,
694
- memoryRefsForInference,
695
- consolidationRan: args.consolidationRan,
696
- allWarnings,
697
- openIndexDb,
698
- });
699
- return {
700
- ...(collected.memoryInference ? { memoryInference: collected.memoryInference } : {}),
701
- ...(collected.graphExtraction ? { graphExtraction: collected.graphExtraction } : {}),
702
- ...(collected.actions.length > 0 ? { actions: collected.actions } : {}),
703
- memoryInferenceDurationMs: collected.memoryInferenceDurationMs,
704
- graphExtractionDurationMs: collected.graphExtractionDurationMs,
705
- orphansPurged: collected.orphansPurged,
706
- proposalsExpired: collected.proposalsExpired,
707
- };
708
- }
709
- /**
710
- * The maintenance sequence (formerly the ~389-line anonymous
711
- * `withIndexWriterLease` callback, before #872 removed the index-rebuild
712
- * lease): memory inference → reindex-after-inference → graph extraction →
713
- * proposal hygiene (orphan purge, expiration) → retention purges. Each pass
714
- * returns its results and warnings; this orchestrator folds warnings into the
715
- * caller's `allWarnings` sink at the same points the inline code pushed them.
716
- */
717
- async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
718
- const { allWarnings } = args;
352
+ const openIndexDb = () => openIndexDatabase(getDbPath());
353
+ const dbCell = {};
719
354
  const actions = [];
720
- let reindexedAfterInference = false;
721
355
  try {
722
- dbCell.current = args.openIndexDb();
356
+ dbCell.current = openIndexDb();
723
357
  const inference = await runMemoryInferenceMaintenancePass(ctx, dbCell, args.memoryRefsForInference);
724
358
  if (inference.action)
725
359
  actions.push(inference.action);
726
360
  allWarnings.push(...inference.warnings);
727
- const memoryInference = inference.memoryInference;
728
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
729
- info("[improve] reindexing after memory inference writes");
361
+ const written = inference.memoryInference?.writtenPaths ?? [];
362
+ if (written.length > 0) {
363
+ // Index exactly the files inference wrote. indexWrittenAssets opens its
364
+ // own write handle, so ours closes first and reopens after.
365
+ info(`[improve] indexing ${written.length} file(s) written by memory inference`);
730
366
  try {
731
- await ctx.reindexWithIndexDbReleased(ctx.primaryStashDir);
732
- reindexedAfterInference = true;
733
- info("[improve] reindex after memory inference complete");
367
+ if (dbCell.current)
368
+ closeDatabase(dbCell.current);
369
+ dbCell.current = undefined;
370
+ try {
371
+ await indexWrittenAssets(primaryStashDir, written);
372
+ }
373
+ finally {
374
+ dbCell.current = openIndexDb();
375
+ }
376
+ info("[improve] indexing after memory inference complete");
734
377
  }
735
378
  catch (err) {
736
- allWarnings.push(`reindex after memory inference failed: ${errMessage(err)}`);
379
+ allWarnings.push(`indexing after memory inference failed: ${errMessage(err)}`);
737
380
  }
738
381
  }
739
- const graph = await runGraphExtractionMaintenancePass(ctx, dbCell, {
740
- actionableRefs: args.actionableRefs,
741
- memoryRefsForInference: args.memoryRefsForInference,
742
- consolidationRan: args.consolidationRan,
743
- reindexedAfterInference,
744
- });
745
- if (graph.action)
746
- actions.push(graph.action);
747
- allWarnings.push(...graph.warnings);
748
- const orphan = runOrphanProposalPurgePass(ctx);
749
- allWarnings.push(...orphan.warnings);
750
- // #733: orphan-state GC — stamps/clears/(optionally) collects
751
- // asset_salience/asset_outcome rows whose ref no longer resolves in
752
- // index.db. Needs the SAME already-open index.db handle (dbCell.current)
753
- // the passes above share.
754
- const stateGc = runOrphanStateGcPass(ctx, dbCell);
755
- allWarnings.push(...stateGc.warnings);
756
- const expiration = runProposalExpirationPass(ctx);
757
- allWarnings.push(...expiration.warnings);
382
+ const hygiene = runProposalHygienePass(ctx);
383
+ allWarnings.push(...hygiene.warnings);
384
+ allWarnings.push(...runOrphanStateGcPass(ctx, dbCell).warnings);
758
385
  allWarnings.push(...runRetentionPurgePass(ctx).warnings);
759
386
  return {
760
- memoryInference,
761
- graphExtraction: graph.graphExtraction,
762
- actions,
387
+ ...(inference.memoryInference ? { memoryInference: inference.memoryInference } : {}),
388
+ ...(actions.length > 0 ? { actions } : {}),
763
389
  memoryInferenceDurationMs: inference.durationMs,
764
- graphExtractionDurationMs: graph.durationMs,
765
- orphansPurged: orphan.orphansPurged,
766
- proposalsExpired: expiration.proposalsExpired,
390
+ orphansPurged: hygiene.orphansPurged,
391
+ proposalsExpired: hygiene.proposalsExpired,
767
392
  };
768
393
  }
769
394
  finally {
@@ -771,540 +396,257 @@ async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
771
396
  closeDatabase(dbCell.current);
772
397
  }
773
398
  }
774
- /**
775
- * Memory inference candidate-discovery (post-Item 9 fix from
776
- * memories/akm-improve-critical-review-2026-05-20). Previously this pass
777
- * was gated on memoryRefsForInference.size > 0 AND passed those refs as a
778
- * candidateRefs filter. But memoryRefsForInference is populated from refs
779
- * distilled THIS RUN — by the time that happens, those parents are
780
- * already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
781
- * them. The genuinely-pending parents in the stash never entered the
782
- * filter. Result: 0/0/0 for 25 consecutive runs.
783
- *
784
- * Fix: always run the pass when the feature is enabled; let the pass's
785
- * own `collectPendingMemories` + `isPendingMemory` predicate find
786
- * candidates from the filesystem-of-truth. The this-run set is still
787
- * logged as a hint but no longer used as a filter.
788
- */
789
- export async function runMemoryInferenceMaintenancePass(ctx, dbCell, memoryRefsForInference) {
790
- const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, memoryInferenceFn } = ctx;
791
- const warnings = [];
792
- let memoryInference;
793
- let durationMs = 0;
794
- let action;
795
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
796
- const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
797
- const pendingBelowMinCount = (() => {
798
- if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
799
- return false;
800
- const pending = collectPendingMemories(primaryStashDir).length;
801
- if (pending < minPendingCount) {
802
- info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
803
- return true;
804
- }
805
- return false;
806
- })();
807
- if (memoryInferenceDisabledByProfile) {
808
- info("[improve] memory inference skipped (disabled by improve profile)");
809
- }
810
- else if (pendingBelowMinCount) {
811
- // skipped — message already emitted above
399
+ /** Time one LLM maintenance pass; a throw becomes a `<label> failed: …` warning. */
400
+ async function timedLlmPass(label, run) {
401
+ const start = Date.now();
402
+ try {
403
+ const result = await run();
404
+ return { result, durationMs: Date.now() - start, warnings: [] };
812
405
  }
813
- else {
814
- const hintRefs = memoryRefsForInference.size;
815
- info(hintRefs > 0
816
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
817
- : "[improve] memory inference starting (discovering pending parents)");
818
- const inferenceStart = Date.now();
819
- try {
820
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
821
- memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
822
- config,
823
- ...(resolvedPlan
824
- ? {
825
- llmRunner: resolvedPlan.processes.memoryInference.runner,
826
- }
827
- : {}),
828
- sources,
829
- signal: budgetSignal,
830
- db: dbCell.current,
831
- reEnrich: false,
832
- onProgress: (event) => {
833
- const current = event.currentRef ? ` ${event.currentRef}` : "";
834
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
835
- },
836
- }), { engine: resolvedPlan?.processes.memoryInference.runner?.engine, process: "memoryInference" });
837
- durationMs = Date.now() - inferenceStart;
838
- // Synthetic sentinel ref (ref-grammar decision D-R3): a colon-free
839
- // `<domain>/_<marker>` label on the event row, never parsed as an asset
840
- // ref. The domain is the asset stash-subdir for asset-scoped sentinels
841
- // (`memories/…`) and the subsystem name for maintenance/artifact sentinels
842
- // (`graph/…`, `events/…`, `proposals/…`, `health/…`, …). Readers match the
843
- // event by `eventType`, never by this string.
844
- action = { ref: "memories/_inference", mode: "memory-inference", result: memoryInference };
845
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
846
- }
847
- catch (err) {
848
- durationMs = Date.now() - inferenceStart;
849
- warnings.push(`memory inference failed: ${errMessage(err)}`);
850
- }
406
+ catch (err) {
407
+ return { durationMs: Date.now() - start, warnings: [`${label} failed: ${errMessage(err)}`] };
851
408
  }
852
- return { memoryInference, durationMs, action, warnings };
853
409
  }
854
410
  /**
855
- * Graph-extraction maintenance pass.
856
- *
857
- * INVARIANT: graph extraction normally runs only on files touched by
858
- * actionable refs (candidatePaths). Full-corpus scans are opt-in via
859
- * profile.processes.graphExtraction.fullScan = true (used by the
860
- * `graph-refresh` built-in profile and its weekly scheduled task).
861
- * The empty-Set fallback is intentional when no refs were touched —
862
- * the extractor's filter rejects every file and returns empty, keeping
863
- * the pass invoked so the action is recorded and tests stay exercised.
411
+ * Memory inference over every pending parent in the stash. The pass discovers
412
+ * its own candidates; the refs distilled this run are only logged as a hint.
864
413
  */
865
- export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
866
- const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, graphExtractionFn } = ctx;
867
- const warnings = [];
868
- let graphExtraction;
869
- let durationMs = 0;
870
- let action;
871
- let reindexedAfterInference = args.reindexedAfterInference;
872
- const graphEnabled = resolvedPlan ? true : isProcessEnabled("index", "graph_extraction", config);
873
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
874
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
875
- // #624 P2: optional incremental high-signal-first cap. Unset = process all
876
- // eligible (byte-identical to today; no ranking/slice).
877
- const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
878
- const graphExtractionIncludeTypes = improveProfile?.processes?.graphExtraction?.includeTypes ?? [
879
- ...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES,
880
- ];
881
- const graphExtractionBatchSize = improveProfile?.processes?.graphExtraction?.batchSize ?? DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE;
882
- // Build the set of refs actually touched this run.
883
- const touchedRefs = new Set();
884
- for (const r of args.actionableRefs)
885
- touchedRefs.add(r.ref);
886
- for (const r of args.memoryRefsForInference)
887
- touchedRefs.add(r);
888
- if (graphExtractionDisabledByProfile) {
889
- info("[improve] graph extraction skipped (disabled by improve profile)");
414
+ export async function runMemoryInferenceMaintenancePass(ctx, dbCell, memoryRefsForInference) {
415
+ const { config, sources, primaryStashDir, resolvedPlan } = ctx;
416
+ const settings = ctx.improveProfile?.processes?.memoryInference;
417
+ if (settings?.enabled === false) {
418
+ info("[improve] memory inference skipped (disabled by improve profile)");
419
+ return { durationMs: 0, warnings: [] };
890
420
  }
891
- else if (sources.length > 0 && graphEnabled) {
892
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
893
- const extractionStart = Date.now();
894
- try {
895
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
896
- // so graph extraction sees current DB state after consolidation writes.
897
- if (args.consolidationRan && !reindexedAfterInference) {
898
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
899
- try {
900
- await ctx.reindexWithIndexDbReleased(primaryStashDir);
901
- reindexedAfterInference = true;
902
- info("[improve] reindex after consolidation complete");
903
- }
904
- catch (err) {
905
- warnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
906
- }
907
- }
908
- // #584: no close/reopen needed here — reindexWithIndexDbReleased
909
- // already swapped in a fresh post-reindex handle.
910
- // Resolve touched refs to absolute file paths. Skipped for fullScan
911
- // (candidatePaths stays undefined → extractor processes all files).
912
- let candidatePaths;
913
- if (!graphExtractionFullScan) {
914
- candidatePaths = new Set();
915
- if (primaryStashDir && touchedRefs.size > 0) {
916
- const writableBundleIds = deriveWritableBundleIds(resolveSourceEntries(primaryStashDir));
917
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableBundleIds).catch(() => null)));
918
- for (const p of resolved) {
919
- if (typeof p === "string" && p.length > 0)
920
- candidatePaths.add(p);
921
- }
922
- }
923
- }
924
- const progressHandler = (event) => {
925
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
926
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
927
- };
928
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
929
- graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
930
- config,
931
- ...(resolvedPlan
932
- ? {
933
- llmRunner: resolvedPlan.processes.graphExtraction.runner,
934
- }
935
- : {}),
936
- sources,
937
- signal: budgetSignal,
938
- db: dbCell.current,
939
- reEnrich: false,
940
- onProgress: progressHandler,
941
- options: {
942
- candidatePaths,
943
- includeTypes: graphExtractionIncludeTypes,
944
- batchSize: graphExtractionBatchSize,
945
- ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}),
946
- },
947
- }), { engine: resolvedPlan?.processes.graphExtraction.runner?.engine, process: "graphExtraction" });
948
- durationMs = Date.now() - extractionStart;
949
- // Synthetic sentinel ref (D-R3): `graph` has no asset stash-subdir, so the
950
- // colon-free `graph/_artifact` names the subsystem, per the sentinel
951
- // convention documented at the memory-inference writer above.
952
- action = { ref: "graph/_artifact", mode: "graph-extraction", result: graphExtraction };
953
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
954
- }
955
- catch (err) {
956
- durationMs = Date.now() - extractionStart;
957
- warnings.push(`graph extraction failed: ${errMessage(err)}`);
421
+ const minPendingCount = settings?.minPendingCount;
422
+ if (primaryStashDir && minPendingCount !== undefined && minPendingCount > 0) {
423
+ const pending = collectPendingMemories(primaryStashDir).length;
424
+ if (pending < minPendingCount) {
425
+ info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
426
+ return { durationMs: 0, warnings: [] };
958
427
  }
959
428
  }
960
- else if (sources.length > 0 && !graphEnabled) {
961
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
962
- }
963
- return { graphExtraction, durationMs, action, warnings };
429
+ const hintRefs = memoryRefsForInference.size;
430
+ info(hintRefs > 0
431
+ ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
432
+ : "[improve] memory inference starting (discovering pending parents)");
433
+ const pass = await timedLlmPass("memory inference", () => attributeStage(resolvedPlan, "memoryInference", () => ctx.memoryInferenceFn({
434
+ config,
435
+ ...(resolvedPlan ? { llmRunner: resolvedPlan.processes.memoryInference.runner } : {}),
436
+ sources,
437
+ signal: ctx.budgetSignal,
438
+ db: dbCell.current,
439
+ reEnrich: false,
440
+ onProgress: (event) => {
441
+ const current = event.currentRef ? ` ${event.currentRef}` : "";
442
+ info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
443
+ },
444
+ })));
445
+ const memoryInference = pass.result;
446
+ if (!memoryInference)
447
+ return { durationMs: pass.durationMs, warnings: pass.warnings };
448
+ info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
449
+ return {
450
+ memoryInference,
451
+ durationMs: pass.durationMs,
452
+ // Sentinel refs (`<domain>/_<marker>`) label maintenance events; never parsed as assets.
453
+ action: { ref: "memories/_inference", mode: "memory-inference", result: memoryInference },
454
+ warnings: pass.warnings,
455
+ };
964
456
  }
965
457
  /**
966
- * Orphan proposal purge — reject pending reflect proposals whose target
967
- * asset no longer exists on disk. Runs after graph extraction so newly
968
- * promoted assets from accept flows during this run are already present.
458
+ * Reject pending proposals whose target no longer exists, then expire pending
459
+ * proposals past the retention window; each emits a roll-up event.
969
460
  */
970
- function runOrphanProposalPurgePass(ctx) {
971
- const { primaryStashDir, sources, eventsCtx } = ctx;
461
+ function runProposalHygienePass(ctx) {
462
+ const { primaryStashDir, eventsCtx } = ctx;
972
463
  const warnings = [];
973
464
  let orphansPurged = 0;
465
+ let proposalsExpired = 0;
974
466
  try {
975
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
976
- orphansPurged = purgeResult.rejected;
977
- if (purgeResult.rejected > 0) {
978
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
467
+ const purge = purgeOrphanProposals(primaryStashDir, ctx.sources.map((s) => s.path));
468
+ orphansPurged = purge.rejected;
469
+ if (purge.rejected > 0) {
470
+ info(`[improve] orphan purge: ${purge.rejected}/${purge.checked} orphaned proposals rejected (${purge.durationMs}ms)`);
979
471
  }
980
472
  appendEvent({
981
473
  eventType: "proposal_orphan_purge",
982
474
  ref: "proposals/_orphan-purge",
983
475
  metadata: {
984
- checked: purgeResult.checked,
985
- rejected: purgeResult.rejected,
986
- durationMs: purgeResult.durationMs,
987
- byType: purgeResult.byType,
988
- orphans: purgeResult.orphans.map((o) => o.ref),
476
+ checked: purge.checked,
477
+ rejected: purge.rejected,
478
+ durationMs: purge.durationMs,
479
+ byType: purge.byType,
480
+ orphans: purge.orphans.map((o) => o.ref),
989
481
  },
990
482
  }, eventsCtx);
991
483
  }
992
484
  catch (err) {
993
485
  warnings.push(`orphan purge failed: ${errMessage(err)}`);
994
486
  }
995
- return { orphansPurged, warnings };
996
- }
997
- /**
998
- * Phase 6B (Advantage D6b): expire pending proposals that have aged past
999
- * the retention window. Runs AFTER orphan purge so we never double-archive
1000
- * a proposal that orphan-purge already moved. `expireStaleProposals` emits
1001
- * its own per-proposal `proposal_expired` events; we additionally emit a
1002
- * single roll-up event here for parity with the orphan-purge surface.
1003
- */
1004
- function runProposalExpirationPass(ctx) {
1005
- const { primaryStashDir, config, eventsCtx } = ctx;
1006
- const warnings = [];
1007
- let proposalsExpired = 0;
1008
487
  try {
1009
- const expireResult = expireStaleProposals(primaryStashDir, config);
1010
- proposalsExpired = expireResult.expired;
1011
- if (expireResult.expired > 0) {
1012
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
1013
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
488
+ const expiry = expireStaleProposals(primaryStashDir, ctx.config);
489
+ proposalsExpired = expiry.expired;
490
+ if (expiry.expired > 0) {
491
+ info(`[improve] expiration: ${expiry.expired}/${expiry.checked} pending proposals expired ` +
492
+ `(retention=${expiry.retentionDays}d, ${expiry.durationMs}ms)`);
1014
493
  }
1015
494
  appendEvent({
1016
495
  eventType: "proposal_expiration_pass",
1017
496
  ref: "proposals/_expiration",
1018
497
  metadata: {
1019
- checked: expireResult.checked,
1020
- expired: expireResult.expired,
1021
- durationMs: expireResult.durationMs,
1022
- retentionDays: expireResult.retentionDays,
1023
- expiredProposals: expireResult.expiredProposals,
498
+ checked: expiry.checked,
499
+ expired: expiry.expired,
500
+ durationMs: expiry.durationMs,
501
+ retentionDays: expiry.retentionDays,
502
+ expiredProposals: expiry.expiredProposals,
1024
503
  },
1025
504
  }, eventsCtx);
1026
505
  }
1027
506
  catch (err) {
1028
507
  warnings.push(`proposal expiration failed: ${errMessage(err)}`);
1029
508
  }
1030
- return { proposalsExpired, warnings };
509
+ return { orphansPurged, proposalsExpired, warnings };
1031
510
  }
1032
511
  /**
1033
- * Fix #2 (observability 0.8.0): trim the events table in state.db so it
1034
- * doesn't grow unbounded. `akm health` writes a `health_probe` row on every
1035
- * invocation, and every command surface emits at least one event besides —
1036
- * without this trim, state.db is a permanent append-only log. Config key
1037
- * `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
1038
- * window. The purge runs against state.db (a different SQLite file from
1039
- * the index handle the other passes use).
512
+ * Trim the observability data that grows append-only — state.db events and
513
+ * improve_runs, logs.db task_logs, and the per-run task log files — to
514
+ * `improve.eventRetentionDays` (default 90; 0 disables), then VACUUM state.db
515
+ * when enough pages are free. Each store fails on its own. state.db work
516
+ * borrows the run's long-lived handle: a second writer on the same WAL file
517
+ * locks (#585).
1040
518
  */
1041
519
  export function runRetentionPurgePass(ctx) {
1042
520
  const { config, eventsCtx } = ctx;
1043
521
  const warnings = [];
1044
522
  const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
1045
- if (retentionDays > 0) {
1046
- // #585: reuse the long-lived eventsCtx.db connection when akmImprove
1047
- // opened one — opening a second state.db write connection while
1048
- // eventsDb is still live made two simultaneous writers contend on the
1049
- // same WAL file ("database is locked"). Only the eventsCtx.dbPath
1050
- // fallback path (state.db failed to open up-front) opens — and then
1051
- // owns and closes — its own handle. C2 still holds: the fallback uses
1052
- // the boundary-pinned path, never a live `process.env` re-read.
1053
- try {
1054
- withStateDb((stateDb) => {
1055
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
1056
- if (purgedCount > 0) {
1057
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
1058
- }
1059
- appendEvent({
1060
- eventType: "events_purged",
1061
- ref: "events/_purge",
1062
- metadata: { purgedCount, retentionDays },
1063
- }, eventsCtx);
1064
- // improve_runs uses the same retention window as events — both are
1065
- // observability/audit data, both grow append-only, both have a
1066
- // dedicated purge helper. Mirroring the events purge here means a
1067
- // single retention knob (improve.eventRetentionDays) governs both.
1068
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
1069
- if (improveRunsPurged > 0) {
1070
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
1071
- }
1072
- appendEvent({
1073
- eventType: "improve_runs_purged",
1074
- ref: "improve_runs/_purge",
1075
- metadata: { purgedCount: improveRunsPurged, retentionDays },
1076
- }, eventsCtx);
1077
- // R5: improve_cycle_metrics has its OWN retention window
1078
- // (default 365d — a slow collapse needs a longer trend than
1079
- // the 90d events window). canary_queries rows are never purged.
1080
- const cycleRetention = config.improve?.collapseDetector?.retentionDays ?? CYCLE_METRICS_RETENTION_DAYS;
1081
- const cycleMetricsPurged = purgeOldCycleMetrics(stateDb, cycleRetention);
1082
- if (cycleMetricsPurged > 0) {
1083
- info(`[improve] cycle-metrics purge: ${cycleMetricsPurged} row(s) older than ${cycleRetention}d removed from state.db`);
1084
- appendEvent({
1085
- // Dedicated type (mirrors improve_runs_purged) so consumers
1086
- // never have to disambiguate purge targets via the ref string.
1087
- eventType: "improve_cycle_metrics_purged",
1088
- ref: "improve_cycle_metrics/_purge",
1089
- metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
1090
- }, eventsCtx);
1091
- }
1092
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1093
- }
1094
- catch (err) {
1095
- warnings.push(`events purge failed: ${errMessage(err)}`);
1096
- }
1097
- // task_logs in logs.db (#579) shares the same retention window as
1098
- // events/improve_runs — all three are observability data governed by
1099
- // the single improve.eventRetentionDays knob. Separate try/finally
1100
- // because logs.db is a different file: a locked/missing logs.db must
1101
- // not block the state.db purges above.
1102
- let logsDb;
1103
- try {
1104
- logsDb = openLogsDatabase();
1105
- const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
1106
- if (taskLogsPurged > 0) {
1107
- info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
1108
- }
1109
- appendEvent({
1110
- eventType: "task_logs_purged",
1111
- ref: "task_logs/_purge",
1112
- metadata: { purgedCount: taskLogsPurged, retentionDays },
1113
- }, eventsCtx);
1114
- }
1115
- catch (err) {
1116
- warnings.push(`task_logs purge failed: ${errMessage(err)}`);
1117
- }
1118
- finally {
1119
- if (logsDb) {
1120
- try {
1121
- logsDb.close();
1122
- }
1123
- catch {
1124
- // best-effort
1125
- }
1126
- }
1127
- }
1128
- // Per-run flat log files under getTaskLogDir() (#951): logs.db above is
1129
- // the durable record and already retention-purged, so the transitional
1130
- // `<taskId>/<timestamp>.log` tail files can be deleted on the same
1131
- // window without losing anything. A separate try/catch — a filesystem
1132
- // problem here must not block the DB purges above.
523
+ if (retentionDays <= 0)
524
+ return { warnings };
525
+ const report = (eventType, ref, purgedCount, what) => {
526
+ if (purgedCount > 0)
527
+ info(`[improve] ${eventType}: ${purgedCount} ${what} older than ${retentionDays}d removed`);
528
+ appendEvent({ eventType, ref, metadata: { purgedCount, retentionDays } }, eventsCtx);
529
+ };
530
+ try {
531
+ withStateDb((stateDb) => {
532
+ report("events_purged", "events/_purge", purgeOldEvents(stateDb, retentionDays), "event(s)");
533
+ report("improve_runs_purged", "improve_runs/_purge", purgeOldImproveRuns(stateDb, retentionDays), "run(s)");
534
+ const vacuum = vacuumIfReclaimable(stateDb, readFreelistInfo(stateDb), { eventType: STATE_DB_VACUUMED_EVENT }, eventsCtx);
535
+ if (vacuum.ran)
536
+ info(`[improve] state.db vacuum: ${vacuum.pagesBefore} -> ${vacuum.pagesAfter} pages`);
537
+ }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
538
+ }
539
+ catch (err) {
540
+ warnings.push(`events purge failed: ${errMessage(err)}`);
541
+ }
542
+ let logsDb;
543
+ try {
544
+ logsDb = openLogsDatabase();
545
+ report("task_logs_purged", "task_logs/_purge", purgeOldTaskLogs(logsDb, retentionDays), "log line(s)");
546
+ }
547
+ catch (err) {
548
+ warnings.push(`task_logs purge failed: ${errMessage(err)}`);
549
+ }
550
+ finally {
1133
551
  try {
1134
- const taskLogFilesPurged = purgeOldTaskLogFiles(undefined, retentionDays);
1135
- if (taskLogFilesPurged > 0) {
1136
- info(`[improve] task log files purge: ${taskLogFilesPurged} file(s) older than ${retentionDays}d removed from ${getTaskLogDir()}`);
1137
- }
1138
- appendEvent({
1139
- eventType: "task_log_files_purged",
1140
- ref: "task_log_files/_purge",
1141
- metadata: { purgedCount: taskLogFilesPurged, retentionDays },
1142
- }, eventsCtx);
552
+ logsDb?.close();
1143
553
  }
1144
- catch (err) {
1145
- warnings.push(`task log files purge failed: ${errMessage(err)}`);
554
+ catch {
555
+ // best-effort
1146
556
  }
1147
557
  }
558
+ try {
559
+ report("task_log_files_purged", "task_log_files/_purge", purgeOldTaskLogFiles(undefined, retentionDays), `file(s) under ${getTaskLogDir()}`);
560
+ }
561
+ catch (err) {
562
+ warnings.push(`task log files purge failed: ${errMessage(err)}`);
563
+ }
1148
564
  return { warnings };
1149
565
  }
1150
- // ── #733 — orphan-state GC pass (Workstream C) ──────────────────────────────
1151
- //
1152
- // Deliberately lean: one maintenance pass, one additive migration (021), one
1153
- // event type (asset_state_gc), one config gate (improve.stateGc.collect,
1154
- // default false). See docs/architecture/specs/0.9.0-close-out-plan.md
1155
- // Workstream C for the full design rationale. No quarantine archive, no
1156
- // circuit breaker, no health-advisory plumbing, no new tables.
1157
566
  /**
1158
- * Grace window (ms) before an unresolved `asset_salience` / `asset_outcome`
1159
- * row becomes delete-eligible — only when `improve.stateGc.collect` is true.
1160
- * A named constant, not a config knob (owner ruling — see the close-out
1161
- * plan's Workstream C). Mirrors `TXN_SWEEP_GRACE_MS` (src/core/fs-txn.ts:298).
567
+ * Grace window before an unresolved salience/outcome row may be deleted (only
568
+ * when `improve.stateGc.collect` is true).
1162
569
  */
1163
570
  export const STATE_GC_GRACE_MS = daysToMs(7);
1164
571
  /**
1165
- * Resolve one state-table's stored `asset_ref` against the live index.
1166
- *
1167
- * "ref not present in entries.item_ref" is the authoritative-deletion
1168
- * predicate (see the pass doc comment below), so this is a thin wrapper
1169
- * around the same single-ref probe the rest of improve uses
1170
- * (`getEntryByRef`, index-entries-repository.ts) — which already resolves
1171
- * both storage spellings a write can produce (`salienceWriteKey`/
1172
- * `outcomeWriteKey` = `itemRef ?? ref`): an exact bundle-qualified item_ref,
1173
- * or a bare conceptId matched by suffix across all bundles.
1174
- *
1175
- * On top of that, falls back to the BARE conceptId form (`bareImproveRef` —
1176
- * the same primitive `preparation.ts`'s `normalizeStoredKey` map is built
1177
- * from via `improveStateReadRefs`) when the stored ref carries a bundle
1178
- * prefix that no longer matches exactly. This is the legacy-spelling
1179
- * normalization trap: a naive `asset_ref NOT IN (SELECT item_ref FROM
1180
- * entries)` would treat a live asset whose row predates bundle-qualification
1181
- * (or whose bundle prefix is stale) as an orphan and delete it. Preferring
1182
- * "never delete a live row" over "never miss a genuinely dead one" mirrors
1183
- * `getEntryByRef`'s own bare-conceptId suffix-match trade-off.
572
+ * A stored state ref is live when it, or its bare conceptId, resolves in the
573
+ * index. The bare fallback keeps a live asset whose row predates
574
+ * bundle-qualification from being collected.
1184
575
  */
1185
- function isStateRefLive(indexDb, storedRef) {
1186
- if (getEntryByRef(indexDb, storedRef) !== null)
576
+ function isStateRefLive(snapshot, storedRef) {
577
+ if (isRefLiveInSnapshot(snapshot, storedRef))
1187
578
  return true;
1188
- const bare = bareImproveRef(storedRef);
1189
- return bare !== storedRef && getEntryByRef(indexDb, bare) !== null;
1190
- }
1191
- /**
1192
- * Sweep ONE state table: stamp refs that just went unresolved, clear refs
1193
- * that resolved again, and — only when `collect` is true — delete rows whose
1194
- * `missing_since` is older than {@link STATE_GC_GRACE_MS}. `pending` is a
1195
- * point-in-time snapshot taken AFTER stamp/clear/delete (re-queried, not
1196
- * accumulated), so it reflects the current backlog rather than this run's
1197
- * delta — "every run emits the counts either way, so live data accumulates
1198
- * proof" (close-out plan, Workstream C).
1199
- */
1200
- function gcOneStateTable(args) {
1201
- const { refRows, indexDb, now, collect, stamp, clear, deleteOlderThan, countPending } = args;
1202
- const toStamp = [];
1203
- const toClear = [];
1204
- for (const row of refRows) {
1205
- const live = isStateRefLive(indexDb, row.asset_ref);
1206
- if (!live && row.missing_since == null)
1207
- toStamp.push(row.asset_ref);
1208
- else if (live && row.missing_since != null)
1209
- toClear.push(row.asset_ref);
1210
- }
1211
- if (toStamp.length > 0)
1212
- stamp(toStamp, now);
1213
- if (toClear.length > 0)
1214
- clear(toClear);
1215
- const collected = collect ? deleteOlderThan(now - STATE_GC_GRACE_MS) : 0;
1216
- const pending = countPending();
1217
- return { pending, collected };
579
+ const bare = stripBundle(storedRef);
580
+ return bare !== storedRef && isRefLiveInSnapshot(snapshot, bare);
1218
581
  }
1219
582
  /**
1220
- * Orphan-state GC — #733 (Workstream C). For each of the two per-asset state
1221
- * tables (`asset_salience`, `asset_outcome`), stamps `missing_since` on refs
1222
- * that no longer resolve against `entries.item_ref` in index.db, clears the
1223
- * stamp on refs that resolve again, and — only when `improve.stateGc.collect`
1224
- * is true — deletes rows whose stamp is older than {@link STATE_GC_GRACE_MS}.
1225
- *
1226
- * "ref not present in entries.item_ref" IS the authoritative-deletion
1227
- * predicate: "absent ≠ deleted" is inherited from the indexer, not
1228
- * re-implemented here — an incomplete or failed source scan preserves that
1229
- * source's last-known-good `entries` rows (indexer.ts ~1195-1199), and
1230
- * mass-wipe is already gated upstream (`preserveExistingIndex` +
1231
- * `fullDelete && scanComplete`). A temporarily unreachable source therefore
1232
- * never surfaces candidates; no separate scan-status tracking is needed.
1233
- *
1234
- * Runs under the SAME index-writer lease / borrowed-state.db-connection
1235
- * discipline as the neighboring maintenance passes: `dbCell.current` supplies
1236
- * the already-open index.db handle (#584), and state.db access goes through
1237
- * `withStateDb(..., { borrowed: eventsCtx?.db })` so this never opens a
1238
- * second live writer alongside a long-lived `eventsCtx.db` connection — see
1239
- * the #585 comment on {@link runRetentionPurgePass}'s events-purge call for
1240
- * why that matters ("database is locked").
1241
- *
1242
- * Exported for direct test coverage (tests/integration/commands/improve/
1243
- * state-gc.test.ts), mirroring the `runMemoryInferenceMaintenancePass` /
1244
- * `runGraphExtractionMaintenancePass` / `runRetentionPurgePass` precedent;
1245
- * production callers reach it only through `runMaintenancePassesUnderLease`.
583
+ * Orphan-state GC (#733) over `asset_salience` and `asset_outcome`: stamp
584
+ * `missing_since` on refs that no longer resolve in index.db, clear it on refs
585
+ * that resolve again, and — only with `improve.stateGc.collect` — delete rows
586
+ * stamped longer ago than {@link STATE_GC_GRACE_MS}. An unreachable source keeps
587
+ * its last-known index rows, so it never surfaces candidates. `pending` is the
588
+ * backlog after the sweep; the event is emitted only when there is something to
589
+ * report.
1246
590
  */
1247
591
  export function runOrphanStateGcPass(ctx, dbCell) {
1248
- const { eventsCtx, config } = ctx;
1249
- const warnings = [];
592
+ const { eventsCtx } = ctx;
1250
593
  const indexDb = dbCell.current;
1251
- if (!indexDb) {
1252
- warnings.push("orphan state GC skipped: no index.db handle available");
1253
- return { pending: 0, collected: 0, warnings };
1254
- }
1255
- const collect = config.improve?.stateGc?.collect === true;
594
+ if (!indexDb)
595
+ return { pending: 0, collected: 0, warnings: ["orphan state GC skipped: no index.db handle available"] };
596
+ const collect = ctx.config.improve?.stateGc?.collect === true;
1256
597
  const now = Date.now();
1257
598
  let pending = 0;
1258
599
  let collected = 0;
1259
600
  try {
601
+ const liveRefs = getLiveRefSnapshot(indexDb);
602
+ const sweep = (table) => {
603
+ const toStamp = [];
604
+ const toClear = [];
605
+ for (const row of table.rows) {
606
+ const live = isStateRefLive(liveRefs, row.asset_ref);
607
+ if (!live && row.missing_since == null)
608
+ toStamp.push(row.asset_ref);
609
+ else if (live && row.missing_since != null)
610
+ toClear.push(row.asset_ref);
611
+ }
612
+ if (toStamp.length > 0)
613
+ table.stamp(toStamp, now);
614
+ if (toClear.length > 0)
615
+ table.clear(toClear);
616
+ const removed = collect ? table.deleteOlderThan(now - STATE_GC_GRACE_MS) : 0;
617
+ return { pending: table.countPending(), collected: removed };
618
+ };
1260
619
  withStateDb((stateDb) => {
1261
- const salienceResult = gcOneStateTable({
1262
- refRows: listAssetSalienceMissingState(stateDb),
1263
- indexDb,
1264
- now,
1265
- collect,
1266
- stamp: (refs, ts) => stampAssetSalienceMissing(stateDb, refs, ts),
1267
- clear: (refs) => clearAssetSalienceMissing(stateDb, refs),
1268
- deleteOlderThan: (cutoffMs) => deleteAssetSalienceMissingBefore(stateDb, cutoffMs),
1269
- countPending: () => countAssetSalienceMissing(stateDb),
1270
- });
1271
- const outcomeResult = gcOneStateTable({
1272
- refRows: listAssetOutcomeMissingState(stateDb),
1273
- indexDb,
1274
- now,
1275
- collect,
1276
- stamp: (refs, ts) => stampAssetOutcomeMissing(stateDb, refs, ts),
1277
- clear: (refs) => clearAssetOutcomeMissing(stateDb, refs),
1278
- deleteOlderThan: (cutoffMs) => deleteAssetOutcomeMissingBefore(stateDb, cutoffMs),
1279
- countPending: () => countAssetOutcomeMissing(stateDb),
1280
- });
1281
- // Table-name-shaped keys ("salience"/"outcome", not the guarded
1282
- // "asset_salience"/"asset_outcome" table names) — this file sits
1283
- // outside src/storage/repositories/**, where the state-table-sql
1284
- // lint rule (#672) forbids naming those tables even in a log string
1285
- // or an object key, not just in raw SQL.
1286
- const byTable = { salience: salienceResult, outcome: outcomeResult };
1287
- pending = salienceResult.pending + outcomeResult.pending;
1288
- collected = salienceResult.collected + outcomeResult.collected;
620
+ // Keys avoid the state table names: the state-table-sql lint rule (#672)
621
+ // forbids them outside the repositories.
622
+ const byTable = {
623
+ salience: sweep({
624
+ rows: listAssetSalienceMissingState(stateDb),
625
+ stamp: (refs, ts) => stampAssetSalienceMissing(stateDb, refs, ts),
626
+ clear: (refs) => clearAssetSalienceMissing(stateDb, refs),
627
+ deleteOlderThan: (cutoff) => deleteAssetSalienceMissingBefore(stateDb, cutoff),
628
+ countPending: () => countAssetSalienceMissing(stateDb),
629
+ }),
630
+ outcome: sweep({
631
+ rows: listAssetOutcomeMissingState(stateDb),
632
+ stamp: (refs, ts) => stampAssetOutcomeMissing(stateDb, refs, ts),
633
+ clear: (refs) => clearAssetOutcomeMissing(stateDb, refs),
634
+ deleteOlderThan: (cutoff) => deleteAssetOutcomeMissingBefore(stateDb, cutoff),
635
+ countPending: () => countAssetOutcomeMissing(stateDb),
636
+ }),
637
+ };
638
+ pending = byTable.salience.pending + byTable.outcome.pending;
639
+ collected = byTable.salience.collected + byTable.outcome.collected;
1289
640
  if (pending > 0 || collected > 0) {
1290
641
  info(`[improve] orphan state GC: ${pending} pending, ${collected} collected ` +
1291
- `(salience ${salienceResult.pending}/${salienceResult.collected}, ` +
1292
- `outcome ${outcomeResult.pending}/${outcomeResult.collected})`);
1293
- // #733 — asset_state_gc reports the current per-table backlog
1294
- // snapshot (`pending`) plus this run's deletions (`collected`);
1295
- // emitted only when there is something to report (mirrors the
1296
- // rekey script's no-op-stays-silent precedent) so a perpetually
1297
- // clean stash never accumulates events.
1298
- appendEvent({
1299
- eventType: "asset_state_gc",
1300
- ref: "asset_state/_gc",
1301
- metadata: { pending, collected, byTable },
1302
- }, eventsCtx);
642
+ `(salience ${byTable.salience.pending}/${byTable.salience.collected}, ` +
643
+ `outcome ${byTable.outcome.pending}/${byTable.outcome.collected})`);
644
+ appendEvent({ eventType: "asset_state_gc", ref: "asset_state/_gc", metadata: { pending, collected, byTable } }, eventsCtx);
1303
645
  }
1304
646
  }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1305
647
  }
1306
648
  catch (err) {
1307
- warnings.push(`orphan state GC failed: ${errMessage(err)}`);
649
+ return { pending, collected, warnings: [`orphan state GC failed: ${errMessage(err)}`] };
1308
650
  }
1309
- return { pending, collected, warnings };
651
+ return { pending, collected, warnings: [] };
1310
652
  }