akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (325) hide show
  1. package/CHANGELOG.md +592 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/help/help-improve.md +9 -6
  6. package/dist/assets/hints/cli-hints-full.md +6 -5
  7. package/dist/assets/profiles/default.json +9 -4
  8. package/dist/assets/profiles/frequent.json +1 -1
  9. package/dist/assets/profiles/memory-focus.json +1 -1
  10. package/dist/assets/profiles/proactive-maintenance.json +25 -0
  11. package/dist/assets/profiles/quick.json +1 -1
  12. package/dist/assets/profiles/recombine-only.json +21 -0
  13. package/dist/assets/profiles/reflect-distill.json +30 -0
  14. package/dist/assets/profiles/synthesize.json +15 -0
  15. package/dist/assets/profiles/thorough.json +1 -1
  16. package/dist/assets/prompts/consolidate-system.md +23 -0
  17. package/dist/assets/prompts/contradiction-judge.md +33 -0
  18. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  19. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  20. package/dist/assets/prompts/extract-session.md +11 -3
  21. package/dist/assets/prompts/graph-extract-system.md +1 -0
  22. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  23. package/dist/assets/prompts/memory-infer-system.md +1 -0
  24. package/dist/assets/prompts/memory-infer-user.md +5 -0
  25. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  26. package/dist/assets/prompts/procedural-system.md +44 -0
  27. package/dist/assets/prompts/recombine-system.md +40 -0
  28. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  29. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  30. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  34. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  35. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  36. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  37. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  38. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  39. package/dist/assets/templates/html/health.html +281 -111
  40. package/dist/assets/wiki/ingest-workflow-template.md +45 -16
  41. package/dist/assets/wiki/schema-template.md +4 -4
  42. package/dist/cli/clack.js +56 -0
  43. package/dist/cli/config-migrate.js +7 -1
  44. package/dist/cli/confirm.js +1 -1
  45. package/dist/cli/parse-args.js +46 -1
  46. package/dist/cli/shared.js +28 -0
  47. package/dist/cli.js +25 -21
  48. package/dist/commands/agent/agent-dispatch.js +3 -2
  49. package/dist/commands/agent/agent-support.js +0 -7
  50. package/dist/commands/agent/contribute-cli.js +26 -7
  51. package/dist/commands/config-cli.js +26 -13
  52. package/dist/commands/env/child-env.js +47 -0
  53. package/dist/commands/env/env-cli.js +220 -227
  54. package/dist/commands/env/env.js +14 -67
  55. package/dist/commands/env/secret-cli.js +140 -138
  56. package/dist/commands/feedback-cli.js +156 -155
  57. package/dist/commands/graph/graph-cli.js +5 -13
  58. package/dist/commands/graph/graph.js +3 -3
  59. package/dist/commands/health/advisories.js +151 -0
  60. package/dist/commands/health/checks.js +103 -16
  61. package/dist/commands/health/html-report.js +447 -81
  62. package/dist/commands/health/improve-metrics.js +771 -0
  63. package/dist/commands/health/llm-usage.js +65 -0
  64. package/dist/commands/health/md-report.js +103 -0
  65. package/dist/commands/health/metrics.js +278 -0
  66. package/dist/commands/health/stash-exposure.js +46 -0
  67. package/dist/commands/health/surfaces.js +216 -0
  68. package/dist/commands/health/task-runs.js +135 -0
  69. package/dist/commands/health/types.js +26 -0
  70. package/dist/commands/health/windows.js +195 -0
  71. package/dist/commands/health.js +91 -1091
  72. package/dist/commands/improve/anti-collapse.js +170 -0
  73. package/dist/commands/improve/calibration.js +161 -0
  74. package/dist/commands/improve/collapse-detector.js +421 -0
  75. package/dist/commands/improve/consolidate/chunking.js +141 -0
  76. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  77. package/dist/commands/improve/consolidate/merge.js +145 -0
  78. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  79. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  80. package/dist/commands/improve/consolidate.js +1295 -1277
  81. package/dist/commands/improve/dedup.js +482 -0
  82. package/dist/commands/improve/distill/content-repair.js +202 -0
  83. package/dist/commands/improve/distill/promote-memory.js +229 -0
  84. package/dist/commands/improve/distill/quality-gate.js +236 -0
  85. package/dist/commands/improve/distill-guards.js +127 -0
  86. package/dist/commands/improve/distill-promotion-policy.js +826 -167
  87. package/dist/commands/improve/distill.js +228 -605
  88. package/dist/commands/improve/eligibility.js +434 -0
  89. package/dist/commands/improve/encoding-salience.js +205 -0
  90. package/dist/commands/improve/extract-cli.js +179 -59
  91. package/dist/commands/improve/extract-prompt.js +54 -3
  92. package/dist/commands/improve/extract-watch.js +140 -0
  93. package/dist/commands/improve/extract.js +409 -43
  94. package/dist/commands/improve/feedback-valence.js +54 -0
  95. package/dist/commands/improve/hot-probation.js +45 -0
  96. package/dist/commands/improve/improve-auto-accept.js +157 -10
  97. package/dist/commands/improve/improve-cli.js +115 -73
  98. package/dist/commands/improve/improve-profiles.js +28 -8
  99. package/dist/commands/improve/improve-result-file.js +15 -25
  100. package/dist/commands/improve/improve-session.js +58 -0
  101. package/dist/commands/improve/improve.js +485 -2764
  102. package/dist/commands/improve/locks.js +154 -0
  103. package/dist/commands/improve/loop-stages.js +1100 -0
  104. package/dist/commands/improve/memory/memory-belief.js +14 -15
  105. package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
  106. package/dist/commands/improve/memory/memory-improve.js +27 -27
  107. package/dist/commands/improve/outcome-loop.js +270 -0
  108. package/dist/commands/improve/preparation.js +2002 -0
  109. package/dist/commands/improve/proactive-maintenance.js +37 -35
  110. package/dist/commands/improve/procedural.js +398 -0
  111. package/dist/commands/improve/recombine.js +818 -0
  112. package/dist/commands/improve/reflect-noise.js +0 -0
  113. package/dist/commands/improve/reflect.js +206 -45
  114. package/dist/commands/improve/salience.js +455 -0
  115. package/dist/commands/improve/schema-similarity-gate.js +168 -0
  116. package/dist/commands/improve/shared.js +51 -0
  117. package/dist/commands/improve/triage.js +93 -0
  118. package/dist/commands/lint/agent-linter.js +19 -24
  119. package/dist/commands/lint/base-linter.js +173 -60
  120. package/dist/commands/lint/command-linter.js +19 -24
  121. package/dist/commands/lint/env-key-rules.js +38 -1
  122. package/dist/commands/lint/fact-linter.js +39 -0
  123. package/dist/commands/lint/index.js +31 -13
  124. package/dist/commands/lint/memory-linter.js +1 -1
  125. package/dist/commands/lint/registry.js +7 -2
  126. package/dist/commands/lint/task-linter.js +3 -3
  127. package/dist/commands/lint/workflow-linter.js +26 -1
  128. package/dist/commands/observability-cli.js +4 -4
  129. package/dist/commands/proposal/drain-policies.js +13 -4
  130. package/dist/commands/proposal/drain.js +45 -51
  131. package/dist/commands/proposal/legacy-import.js +115 -0
  132. package/dist/commands/proposal/proposal-cli.js +24 -34
  133. package/dist/commands/proposal/proposal.js +2 -1
  134. package/dist/commands/proposal/propose.js +8 -3
  135. package/dist/commands/proposal/repository.js +829 -0
  136. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  137. package/dist/commands/proposal/validators/proposals.js +93 -895
  138. package/dist/commands/read/curate.js +410 -111
  139. package/dist/commands/read/knowledge.js +10 -3
  140. package/dist/commands/read/remember-cli.js +133 -138
  141. package/dist/commands/read/search-cli.js +15 -8
  142. package/dist/commands/read/search.js +22 -11
  143. package/dist/commands/read/show.js +106 -14
  144. package/dist/commands/registry-cli.js +76 -87
  145. package/dist/commands/remember.js +11 -12
  146. package/dist/commands/sources/add-cli.js +91 -95
  147. package/dist/commands/sources/history.js +1 -1
  148. package/dist/commands/sources/init.js +66 -18
  149. package/dist/commands/sources/installed-stashes.js +11 -3
  150. package/dist/commands/sources/schema-repair.js +44 -46
  151. package/dist/commands/sources/self-update.js +2 -2
  152. package/dist/commands/sources/source-add.js +7 -3
  153. package/dist/commands/sources/sources-cli.js +3 -3
  154. package/dist/commands/sources/stash-cli.js +19 -39
  155. package/dist/commands/sources/stash-skeleton.js +57 -8
  156. package/dist/commands/tasks/default-tasks.js +15 -2
  157. package/dist/commands/tasks/tasks-cli.js +20 -29
  158. package/dist/commands/tasks/tasks.js +39 -11
  159. package/dist/commands/wiki-cli.js +23 -38
  160. package/dist/commands/workflow-cli.js +15 -1
  161. package/dist/core/asset/asset-registry.js +3 -1
  162. package/dist/core/asset/asset-spec.js +21 -4
  163. package/dist/core/asset/frontmatter.js +188 -167
  164. package/dist/core/asset/markdown.js +8 -0
  165. package/dist/core/authoring-rules.js +92 -0
  166. package/dist/core/common.js +4 -23
  167. package/dist/core/concurrent.js +10 -1
  168. package/dist/core/config/config-io.js +10 -1
  169. package/dist/core/config/config-migration.js +18 -40
  170. package/dist/core/config/config-schema.js +382 -62
  171. package/dist/core/config/config-types.js +3 -3
  172. package/dist/core/config/config.js +67 -22
  173. package/dist/core/deep-merge.js +38 -0
  174. package/dist/core/errors.js +1 -0
  175. package/dist/core/eval/rank-metrics.js +113 -0
  176. package/dist/core/events.js +4 -7
  177. package/dist/core/improve-types.js +47 -8
  178. package/dist/core/logs-db.js +14 -75
  179. package/dist/core/parse.js +36 -16
  180. package/dist/core/paths.js +18 -18
  181. package/dist/core/standards/resolve-standards-context.js +87 -0
  182. package/dist/core/standards/resolve-stash-standards.js +99 -0
  183. package/dist/core/standards/resolve-type-conventions.js +66 -0
  184. package/dist/core/state/migrations.js +770 -0
  185. package/dist/core/state-db.js +132 -1126
  186. package/dist/core/structured.js +69 -0
  187. package/dist/core/time.js +53 -0
  188. package/dist/core/warn.js +21 -0
  189. package/dist/core/write-source.js +37 -0
  190. package/dist/indexer/db/db.js +259 -769
  191. package/dist/indexer/db/entry-mapper.js +41 -0
  192. package/dist/indexer/db/graph-db.js +129 -86
  193. package/dist/indexer/db/llm-cache.js +2 -2
  194. package/dist/indexer/db/schema.js +516 -0
  195. package/dist/indexer/ensure-index.js +36 -92
  196. package/dist/indexer/feedback/utility-policy.js +75 -0
  197. package/dist/indexer/graph/graph-boost.js +51 -41
  198. package/dist/indexer/graph/graph-extraction.js +207 -4
  199. package/dist/indexer/index-writer-lock.js +18 -11
  200. package/dist/indexer/index-written-assets.js +105 -0
  201. package/dist/indexer/indexer.js +182 -204
  202. package/dist/indexer/passes/dir-staleness.js +114 -0
  203. package/dist/indexer/passes/memory-inference.js +13 -5
  204. package/dist/indexer/passes/metadata.js +20 -0
  205. package/dist/indexer/read-preflight.js +23 -0
  206. package/dist/indexer/search/db-search.js +89 -13
  207. package/dist/indexer/search/fts-query.js +51 -0
  208. package/dist/indexer/search/ranking-contributors.js +95 -9
  209. package/dist/indexer/search/ranking.js +79 -3
  210. package/dist/indexer/search/search-fields.js +6 -0
  211. package/dist/indexer/search/search-source.js +32 -21
  212. package/dist/indexer/search/semantic-status.js +4 -0
  213. package/dist/indexer/walk/matchers.js +9 -0
  214. package/dist/indexer/walk/walker.js +21 -13
  215. package/dist/integrations/agent/builders.js +39 -13
  216. package/dist/integrations/agent/config.js +20 -59
  217. package/dist/integrations/agent/detect.js +9 -0
  218. package/dist/integrations/agent/index.js +3 -19
  219. package/dist/integrations/agent/model-aliases.js +7 -2
  220. package/dist/integrations/agent/profiles.js +7 -1
  221. package/dist/integrations/agent/prompts.js +75 -9
  222. package/dist/integrations/agent/runner-dispatch.js +59 -0
  223. package/dist/integrations/agent/runner.js +13 -9
  224. package/dist/integrations/agent/spawn.js +69 -67
  225. package/dist/integrations/harnesses/claude/agent-builder.js +1 -1
  226. package/dist/integrations/harnesses/claude/index.js +2 -0
  227. package/dist/integrations/harnesses/claude/session-log.js +10 -0
  228. package/dist/integrations/harnesses/index.js +2 -3
  229. package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
  230. package/dist/integrations/harnesses/opencode/index.js +2 -0
  231. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  232. package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
  233. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +98 -17
  234. package/dist/integrations/harnesses/types.js +1 -0
  235. package/dist/integrations/session-logs/index.js +16 -0
  236. package/dist/llm/call-ai.js +2 -2
  237. package/dist/llm/client.js +34 -11
  238. package/dist/llm/embedder.js +67 -4
  239. package/dist/llm/embedders/cache.js +3 -1
  240. package/dist/llm/embedders/deterministic.js +66 -0
  241. package/dist/llm/embedders/local.js +73 -3
  242. package/dist/llm/feature-gate.js +16 -15
  243. package/dist/llm/graph-extract.js +67 -44
  244. package/dist/llm/memory-infer-impl.js +138 -0
  245. package/dist/llm/memory-infer.js +1 -127
  246. package/dist/llm/metadata-enhance.js +44 -31
  247. package/dist/llm/structured-call.js +49 -0
  248. package/dist/migrate-storage-node.mjs +8 -0
  249. package/dist/output/context.js +5 -5
  250. package/dist/output/renderers.js +85 -14
  251. package/dist/output/shapes/curate.js +14 -2
  252. package/dist/output/shapes/helpers.js +0 -3
  253. package/dist/output/shapes/passthrough.js +2 -1
  254. package/dist/output/text/helpers.js +29 -1
  255. package/dist/output/text/workflow.js +1 -0
  256. package/dist/registry/providers/skills-sh.js +21 -147
  257. package/dist/registry/providers/static-index.js +15 -157
  258. package/dist/registry/resolve.js +27 -9
  259. package/dist/runtime.js +25 -1
  260. package/dist/scripts/migrate-storage.js +2661 -2369
  261. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +883 -596
  262. package/dist/setup/detect.js +9 -0
  263. package/dist/setup/legacy-config.js +106 -0
  264. package/dist/setup/prompt.js +57 -0
  265. package/dist/setup/providers.js +14 -0
  266. package/dist/setup/registry-stash-loader.js +12 -0
  267. package/dist/setup/semantic-assets.js +124 -0
  268. package/dist/setup/setup.js +52 -1614
  269. package/dist/setup/steps/connection.js +734 -0
  270. package/dist/setup/steps/output.js +31 -0
  271. package/dist/setup/steps/platforms.js +124 -0
  272. package/dist/setup/steps/semantic.js +27 -0
  273. package/dist/setup/steps/sources.js +222 -0
  274. package/dist/setup/steps/stashdir.js +42 -0
  275. package/dist/setup/steps/tasks.js +152 -0
  276. package/dist/sources/include.js +6 -2
  277. package/dist/sources/providers/filesystem.js +0 -1
  278. package/dist/sources/providers/git-install.js +210 -0
  279. package/dist/sources/providers/git-provider.js +234 -0
  280. package/dist/sources/providers/git-stash.js +248 -0
  281. package/dist/sources/providers/git.js +10 -661
  282. package/dist/sources/providers/npm.js +2 -6
  283. package/dist/sources/providers/provider-utils.js +13 -7
  284. package/dist/sources/providers/sync-from-ref.js +9 -1
  285. package/dist/sources/providers/website.js +9 -5
  286. package/dist/sources/website-ingest.js +187 -29
  287. package/dist/sources/wiki-fetchers/registry.js +53 -0
  288. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  289. package/dist/storage/database.js +45 -10
  290. package/dist/storage/managed-db.js +82 -0
  291. package/dist/storage/repositories/canaries-repository.js +107 -0
  292. package/dist/storage/repositories/consolidation-repository.js +38 -0
  293. package/dist/storage/repositories/embeddings-repository.js +72 -0
  294. package/dist/storage/repositories/events-repository.js +187 -0
  295. package/dist/storage/repositories/extract-sessions-repository.js +96 -0
  296. package/dist/storage/repositories/improve-runs-repository.js +146 -0
  297. package/dist/storage/repositories/index-db.js +14 -8
  298. package/dist/storage/repositories/proposals-repository.js +220 -0
  299. package/dist/storage/repositories/recombine-repository.js +213 -0
  300. package/dist/storage/repositories/registry-cache.js +93 -0
  301. package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
  302. package/dist/storage/repositories/task-history-repository.js +93 -0
  303. package/dist/storage/sqlite-pragmas.js +146 -0
  304. package/dist/tasks/backends/cron.js +1 -1
  305. package/dist/tasks/backends/index.js +9 -0
  306. package/dist/tasks/backends/launchd.js +1 -1
  307. package/dist/tasks/backends/schtasks.js +1 -1
  308. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  309. package/dist/tasks/runner.js +15 -13
  310. package/dist/text-import-hook.mjs +0 -0
  311. package/dist/wiki/wiki.js +52 -11
  312. package/dist/workflows/cli.js +1 -0
  313. package/dist/workflows/db.js +3 -4
  314. package/dist/workflows/runtime/runs.js +43 -118
  315. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  316. package/dist/workflows/validate-summary.js +2 -7
  317. package/docs/README.md +69 -18
  318. package/docs/data-and-telemetry.md +5 -4
  319. package/docs/migration/release-notes/0.7.0.md +1 -1
  320. package/docs/migration/release-notes/0.9.0.md +39 -0
  321. package/package.json +10 -10
  322. package/dist/assets/tasks/core/update-stashes.yml +0 -4
  323. package/dist/commands/db-cli.js +0 -23
  324. package/dist/indexer/db/db-backup.js +0 -376
  325. package/dist/indexer/passes/staleness-detect.js +0 -488
@@ -0,0 +1,2002 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { parseAssetRef } from "../../core/asset/asset-ref.js";
7
+ import { parseFrontmatter } from "../../core/asset/frontmatter.js";
8
+ import { daysToMs } from "../../core/common.js";
9
+ import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
10
+ import { rethrowIfTestIsolationError } from "../../core/errors.js";
11
+ import { appendEvent, readEvents } from "../../core/events.js";
12
+ import { openStateDatabase, withStateDb } from "../../core/state-db.js";
13
+ import { info, warn } from "../../core/warn.js";
14
+ import { closeDatabase, getRetrievalCounts, getZeroResultSearches, openExistingDatabase } from "../../indexer/db/db.js";
15
+ import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
16
+ import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
17
+ import { withLlmStage } from "../../llm/usage-telemetry.js";
18
+ import { persistPhaseThreshold } from "../../storage/repositories/improve-runs-repository.js";
19
+ import { listProposalGateDecisions, listStateProposals } from "../../storage/repositories/proposals-repository.js";
20
+ import { akmLint } from "../lint/index.js";
21
+ import { getProposal, listProposals } from "../proposal/repository.js";
22
+ import { runSchemaRepairPass } from "../sources/schema-repair.js";
23
+ import { computeThresholdAutoTune, gateDecisionsToSamples, summarizeCalibration, } from "./calibration.js";
24
+ import { akmConsolidate } from "./consolidate.js";
25
+ // Eligibility / candidate-selection predicates live in ./eligibility.
26
+ import { buildLatestFeedbackTsMap, buildLatestProposalTsMap, buildUtilityMap, dedupeRefs, findAssetFilePath, isDistillCandidateRef, isLessonCandidate, isSignalDeltaEligible, } from "./eligibility.js";
27
+ import { akmExtract, countNewExtractCandidates } from "./extract.js";
28
+ import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
29
+ import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
30
+ import { resolveProcessEnabled } from "./improve-profiles.js";
31
+ import { applyMemoryCleanup } from "./memory/memory-improve.js";
32
+ import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, OUTCOME_SCORE_MAX, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
33
+ import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
34
+ import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getConsecutiveNoOps, getLastUseMsByRef, isContentEncodingRow, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
35
+ // ── improve preparation stage ───────────────────────
36
+ // The pre-loop preparation pipeline (consolidation, session-extract, validation/
37
+ // repair, eligibility partitioning, selectors) extracted from improve.ts.
38
+ /**
39
+ * #612 / WS-4 — bounded, opt-in per-phase auto-accept threshold auto-tune.
40
+ *
41
+ * Reads `improve.calibration` from config. When `autoTune` is enabled, computes
42
+ * the calibration of recent gate decisions, derives a bounded threshold
43
+ * adjustment (clamped into the configured band, capped per step), logs it, and
44
+ * records a `calibration_autotune` event. Returns the new threshold (integer
45
+ * 0-100) when an adjustment was made, or `undefined` to leave the caller's
46
+ * threshold unchanged.
47
+ *
48
+ * WS-4 change: accepts an optional `phase` parameter. When provided, the tuned
49
+ * threshold is persisted to `improve_gate_thresholds` (state.db Migration 012)
50
+ * keyed by phase so `makeGateConfig` can read it back on the next run and each
51
+ * phase maintains its own calibrated threshold rather than a shared global.
52
+ *
53
+ * WS-4 ceiling: `maxThreshold` defaults to 85 (not 100) to prevent the gate
54
+ * converging to pure exploitation and shutting down Gap-3/4 novelty throughput.
55
+ *
56
+ * DEFAULT OFF: with no `improve.calibration` block (or `autoTune: false`) this
57
+ * returns `undefined` immediately, so the gate threshold is unchanged and
58
+ * behaviour is byte-identical to today.
59
+ */
60
+ export function maybeAutoTuneThreshold(currentThreshold, config, stateDbPath, ctx, phase) {
61
+ const cal = config.improve?.calibration;
62
+ if (!cal?.autoTune)
63
+ return undefined;
64
+ // WS-4: default maxThreshold is now 85 (ceiling to prevent pure exploitation).
65
+ // Callers that explicitly set maxThreshold in config override this default.
66
+ const tuneConfig = {
67
+ autoTune: true,
68
+ minThreshold: cal.minThreshold ?? 0,
69
+ maxThreshold: cal.maxThreshold ?? 85,
70
+ maxStep: cal.maxStep ?? 5,
71
+ minSamples: cal.minSamples ?? 20,
72
+ targetAcceptRate: cal.targetAcceptRate ?? 0.9,
73
+ };
74
+ // Defensive: an inverted band disables tuning rather than clamping to nonsense.
75
+ if (tuneConfig.minThreshold > tuneConfig.maxThreshold)
76
+ return undefined;
77
+ const summary = withStateDb((db) => {
78
+ const allDecisions = listProposalGateDecisions(db);
79
+ // WS-4 fix: when called with a phase label, restrict calibration to that
80
+ // phase's decision pool so a reflect-dominated run cannot tighten the
81
+ // consolidate gate (or vice-versa). The gate field is `improve:<phase>`,
82
+ // matching what improve-auto-accept.ts stamps at line ~163.
83
+ const gateLabel = phase ? `improve:${phase}` : undefined;
84
+ const decisions = gateLabel ? allDecisions.filter((d) => d.gate === gateLabel) : allDecisions;
85
+ return summarizeCalibration(gateDecisionsToSamples(decisions));
86
+ }, { path: stateDbPath });
87
+ const result = computeThresholdAutoTune(currentThreshold, summary, tuneConfig);
88
+ if (!result.adjusted)
89
+ return undefined;
90
+ const appendEventFn = ctx?.appendEventFn ?? appendEvent;
91
+ const phaseLabel = phase ?? "global";
92
+ info(`[improve] calibration auto-tune (${phaseLabel}): threshold ${result.previousThreshold} -> ${result.newThreshold} ` +
93
+ `(${result.reason}; samples=${summary.samples}, acceptRate=${summary.overallAcceptRate}, ` +
94
+ `gap=${summary.calibrationGap}, band=[${tuneConfig.minThreshold},${tuneConfig.maxThreshold}])`);
95
+ try {
96
+ appendEventFn({
97
+ eventType: "calibration_autotune",
98
+ ref: "improve:calibration",
99
+ metadata: {
100
+ phase: phaseLabel,
101
+ previousThreshold: result.previousThreshold,
102
+ newThreshold: result.newThreshold,
103
+ delta: result.delta,
104
+ reason: result.reason,
105
+ samples: summary.samples,
106
+ overallAcceptRate: summary.overallAcceptRate,
107
+ calibrationGap: summary.calibrationGap,
108
+ minThreshold: tuneConfig.minThreshold,
109
+ maxThreshold: tuneConfig.maxThreshold,
110
+ },
111
+ });
112
+ }
113
+ catch (err) {
114
+ warn(`[improve] calibration auto-tune event not recorded: ${err instanceof Error ? err.message : String(err)}`);
115
+ }
116
+ // WS-4: Persist the per-phase threshold so makeGateConfig reads it on the
117
+ // next run. Best-effort — a write failure must not abort the improve run.
118
+ if (phase) {
119
+ try {
120
+ withStateDb((persistDb) => persistPhaseThreshold(persistDb, phase, result.newThreshold), {
121
+ path: stateDbPath,
122
+ });
123
+ }
124
+ catch (err) {
125
+ warn(`[improve] calibration auto-tune: failed to persist phase threshold for ${phase}: ${err instanceof Error ? err.message : String(err)}`);
126
+ }
127
+ }
128
+ return result.newThreshold;
129
+ }
130
+ /**
131
+ * Run (or gate-skip) the memory consolidation pass.
132
+ *
133
+ * #551 — two coordinated changes live here:
134
+ *
135
+ * 1. STRUCTURAL: this runs before extract in the improve pipeline (see
136
+ * `runImprovePreparationStage`). Consolidation therefore only ever judges
137
+ * PRIOR-run memories; current-run extract promotions are invisible to it.
138
+ *
139
+ * 2. SMARTER POOL-DELTA GATE: even among on-disk files, a memory whose only
140
+ * post-`lastConsolidateTs` mtime bump came from its OWN auto-accept
141
+ * promotion (i.e. it was just promoted by extract in the immediately
142
+ * preceding run and has not had a full improve cycle to settle) does NOT
143
+ * count as "work to do". We exclude those paths from the pool-delta check
144
+ * using the `promoted` events already emitted with each promotion's
145
+ * `assetPath`. A genuinely-settled prior memory — one edited by feedback,
146
+ * reflect, manual edit, or simply older than the last consolidate — still
147
+ * triggers the run. This is gate-option (a) from the issue (same-run /
148
+ * adjacent-run promotion exclusion), chosen over option (b) because there
149
+ * is no `extract_completed` event in the data model to gate against;
150
+ * `promoted` events with `assetPath` already carry exactly the signal we
151
+ * need, so the fix is non-invasive and provably correct.
152
+ */
153
+ export async function runConsolidationPass(args) {
154
+ const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx, budgetSignal, runBudgetMs } = args;
155
+ const baseConfig = options.config ?? loadConfig();
156
+ const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
157
+ const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
158
+ const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
159
+ // When volume triggers a consolidation pass, force-enable the consolidate
160
+ // process on the default improve profile so the gate accepts the run even
161
+ // if the user's config disabled it. We synthesise a new profile override
162
+ // rather than mutating connection settings.
163
+ const consolidationConfig = volumeTriggered
164
+ ? {
165
+ ...baseConfig,
166
+ profiles: {
167
+ ...(baseConfig.profiles ?? {}),
168
+ improve: {
169
+ ...(baseConfig.profiles?.improve ?? {}),
170
+ default: {
171
+ ...(baseConfig.profiles?.improve?.default ?? {}),
172
+ processes: {
173
+ ...(baseConfig.profiles?.improve?.default?.processes ?? {}),
174
+ consolidate: {
175
+ ...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
176
+ enabled: true,
177
+ },
178
+ },
179
+ },
180
+ },
181
+ },
182
+ }
183
+ : baseConfig;
184
+ // 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
185
+ // memory file has been updated since the most recent successful
186
+ // consolidate_completed event. Time-based cooldowns produced the same
187
+ // synchronised-wave failure mode the reflect/distill cooldowns did; the
188
+ // pool-delta gate ties consolidation to actual work-to-do.
189
+ const recentConsolidations = readEvents({ type: "consolidate_completed" });
190
+ const lastConsolidation = recentConsolidations.events
191
+ .filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
192
+ .sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
193
+ const lastConsolidateTs = lastConsolidation?.ts;
194
+ // #551 smarter gate: build the set of memory asset paths whose only delta
195
+ // since the last consolidate is their OWN auto-accept promotion. Those files
196
+ // have not had a full improve cycle to settle, so they offer no merge /
197
+ // contradiction candidates yet — excluding them stops the gate firing on
198
+ // freshly-promoted single-source memories. We read `promoted` events emitted
199
+ // after the last consolidate; each carries the written `assetPath`.
200
+ const promotedSinceConsolidate = (() => {
201
+ const paths = new Set();
202
+ try {
203
+ const promoted = readEvents({
204
+ type: "promoted",
205
+ ...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
206
+ }).events;
207
+ for (const e of promoted) {
208
+ const ap = e.metadata?.assetPath;
209
+ if (typeof ap === "string" && ap.length > 0)
210
+ paths.add(path.resolve(ap));
211
+ }
212
+ }
213
+ catch {
214
+ // best-effort: if the events query fails, fall back to no exclusions
215
+ // (preserves pre-#551 behaviour rather than over-skipping).
216
+ }
217
+ return paths;
218
+ })();
219
+ // Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do,
220
+ // EXCEPT files whose only post-consolidate change was their own promotion.
221
+ // Using file mtime keeps this query DB-free and matches what the indexer
222
+ // already uses as the canonical `memory.updated_at` proxy.
223
+ //
224
+ // Bootstrap: when no successful consolidate_completed event has ever been
225
+ // recorded, we cannot evaluate the pool-delta — treat as eligible so a
226
+ // fresh stash runs consolidate once before the steady-state gate kicks in.
227
+ const memoryUpdatedAfterLastConsolidate = (() => {
228
+ if (volumeTriggered)
229
+ return true; // volume override forces the run regardless.
230
+ if (!lastConsolidateTs)
231
+ return true; // bootstrap path: never consolidated.
232
+ if (!primaryStashDir)
233
+ return false;
234
+ const memoriesDir = path.join(primaryStashDir, "memories");
235
+ if (!fs.existsSync(memoriesDir))
236
+ return false;
237
+ try {
238
+ return fs.readdirSync(memoriesDir).some((f) => {
239
+ if (!f.endsWith(".md"))
240
+ return false;
241
+ const filePath = path.join(memoriesDir, f);
242
+ // #551: skip files that were only touched by their own promotion this
243
+ // cohort — they have no settled merge/contradiction candidates yet.
244
+ if (promotedSinceConsolidate.has(path.resolve(filePath)))
245
+ return false;
246
+ try {
247
+ return fs.statSync(filePath).mtime.toISOString() > lastConsolidateTs;
248
+ }
249
+ catch {
250
+ return false;
251
+ }
252
+ });
253
+ }
254
+ catch {
255
+ return false;
256
+ }
257
+ })();
258
+ const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
259
+ // Profile gate: if profile explicitly disables consolidate, skip the entire pass.
260
+ const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
261
+ // #553 minPoolSize guard: skip consolidation when the eligible memory pool is
262
+ // below a minimum size, rather than spending an LLM pass on a handful of
263
+ // memories. This is an INDEPENDENT skip condition from #551's mtime pool-delta
264
+ // gate — either can skip. Default 500; `minPoolSize: 0` disables the guard.
265
+ // Evaluated against the eligible-pool count BEFORE entering the LLM loop so a
266
+ // skip costs ZERO LLM calls.
267
+ const CONSOLIDATE_DEFAULT_MIN_POOL_SIZE = 500;
268
+ const configuredMinPoolSize = improveProfile?.processes?.consolidate?.minPoolSize;
269
+ const minPoolSize = typeof configuredMinPoolSize === "number" ? configuredMinPoolSize : CONSOLIDATE_DEFAULT_MIN_POOL_SIZE;
270
+ const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
271
+ // volumeTriggered means the pool already exceeds the volume threshold (100),
272
+ // so a force-triggered run never trips the pool-size guard. The guard only
273
+ // engages when minPoolSize > 0 and the eligible pool is strictly below it.
274
+ const poolBelowMinSize = !volumeTriggered && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
275
+ let consolidation = {
276
+ schemaVersion: 1,
277
+ ok: true,
278
+ shape: "consolidate-result",
279
+ dryRun: false,
280
+ previewOnly: false,
281
+ target: "",
282
+ processed: 0,
283
+ merged: 0,
284
+ deleted: 0,
285
+ promoted: [],
286
+ contradicted: 0,
287
+ warnings: [],
288
+ durationMs: 0,
289
+ };
290
+ let gateAutoAcceptedCount = 0;
291
+ let gateAutoAcceptFailedCount = 0;
292
+ const consolidateGateCfg = makeGateConfig("consolidate", {
293
+ globalThreshold: options.autoAccept,
294
+ dryRun: options.dryRun ?? false,
295
+ stashDir: primaryStashDir,
296
+ config: consolidationConfig,
297
+ eventsCtx,
298
+ stateDbPath: eventsCtx?.dbPath,
299
+ }, { minimumThreshold: 95 });
300
+ if (consolidateDisabledByProfile) {
301
+ info("[improve] consolidation skipped (disabled by improve profile)");
302
+ }
303
+ else if (poolBelowMinSize) {
304
+ // #553: eligible pool below the configured minimum — skip with zero LLM
305
+ // calls. Reuse the #551 `improve_skipped` emission path so health surfaces
306
+ // it via the dynamic skipReasons aggregation under `pool_below_min_size`.
307
+ appendEvent({
308
+ eventType: "improve_skipped",
309
+ ref: "memory:_consolidation",
310
+ metadata: {
311
+ reason: "pool_below_min_size",
312
+ poolSize: eligiblePoolSize,
313
+ minPoolSize,
314
+ },
315
+ }, eventsCtx);
316
+ info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
317
+ }
318
+ else if (!consolidationOnCooldown) {
319
+ consolidation = await withLlmStage("consolidate", () => akmConsolidate({
320
+ ...options.consolidateOptions,
321
+ config: consolidationConfig,
322
+ stashDir: options.stashDir,
323
+ // Active profile for this improve run — lets consolidate's secondary
324
+ // process-config reads honor `--profile <name>` instead of `default`.
325
+ improveProfile,
326
+ autoTriggered: volumeTriggered,
327
+ // Tie consolidate proposals back to this improve invocation so
328
+ // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
329
+ sourceRun: `consolidate-${Date.now()}`,
330
+ // Pass profile-configured options. incrementalSince narrows the pool to
331
+ // recently-changed memories + graph neighbours — use this for frequent
332
+ // passes (quick-shredder). Leave absent in the nightly default profile for
333
+ // a full-pool sweep that catches stale-but-unmerged duplicates.
334
+ incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
335
+ limit: improveProfile?.processes?.consolidate?.limit,
336
+ neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
337
+ maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
338
+ // #617 — deterministic near-duplicate dedup pre-pass. DEFAULT OFF; only
339
+ // runs when the profile explicitly sets `consolidate.dedup.enabled`.
340
+ dedup: improveProfile?.processes?.consolidate?.dedup,
341
+ // #581 — judged-state cache. DEFAULT OFF; only engages when the profile
342
+ // explicitly sets `consolidate.judgedCache.enabled`. Skips memories
343
+ // judged-unchanged since their last judge so one run sweeps the full
344
+ // corpus instead of narrowing to a time-window slice.
345
+ judgedCache: improveProfile?.processes?.consolidate?.judgedCache,
346
+ // Honor profile.autoAccept (already merged into options.autoAccept at the
347
+ // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
348
+ // is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
349
+ // (which maps to undefined) from disabling consolidation auto-accept.
350
+ // options.consolidateOptions.autoAccept (if explicitly provided by caller)
351
+ // still wins because the spread above runs first.
352
+ autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
353
+ // WS-3a: forward budget signal for graceful abort on timeout, and pass
354
+ // the profile's p90 estimate for cold-start budget reduction.
355
+ signal: budgetSignal,
356
+ p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
357
+ // WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
358
+ // can flag when consolidation alone exceeded the budget.
359
+ runBudgetMs,
360
+ }));
361
+ {
362
+ const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
363
+ try {
364
+ if (!primaryStashDir)
365
+ return { proposalId, confidence: undefined };
366
+ const proposal = getProposal(primaryStashDir, proposalId);
367
+ return { proposalId, confidence: proposal.confidence };
368
+ }
369
+ catch {
370
+ return { proposalId, confidence: undefined };
371
+ }
372
+ }), consolidateGateCfg);
373
+ gateAutoAcceptedCount += consolidateGr.promoted.length;
374
+ gateAutoAcceptFailedCount += consolidateGr.failed.length;
375
+ }
376
+ if (consolidation.processed > 0) {
377
+ appendEvent({
378
+ eventType: "consolidate_completed",
379
+ ref: "memory:_consolidation",
380
+ metadata: {
381
+ processed: consolidation.processed,
382
+ merged: consolidation.merged,
383
+ deleted: consolidation.deleted,
384
+ contradicted: consolidation.contradicted,
385
+ failedChunks: consolidation.failedChunks ?? 0,
386
+ durationMs: consolidation.durationMs,
387
+ },
388
+ }, eventsCtx);
389
+ }
390
+ }
391
+ else {
392
+ appendEvent({
393
+ eventType: "improve_skipped",
394
+ ref: "memory:_consolidation",
395
+ metadata: {
396
+ reason: "consolidation_no_memory_updates",
397
+ lastEventTs: lastConsolidation?.ts ?? null,
398
+ },
399
+ }, eventsCtx);
400
+ info("[improve] consolidation skipped (no memory updates since last run)");
401
+ }
402
+ // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
403
+ const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
404
+ // WS-4: Per-phase threshold auto-tune for the consolidate phase.
405
+ // Persists result for the NEXT run's makeGateConfig to read.
406
+ const consolidateTuneDbPath = eventsCtx?.dbPath;
407
+ if (options.autoAccept !== undefined && consolidateTuneDbPath) {
408
+ try {
409
+ maybeAutoTuneThreshold(consolidateGateCfg.phaseThreshold ?? options.autoAccept, consolidationConfig, consolidateTuneDbPath, undefined, "consolidate");
410
+ }
411
+ catch (err) {
412
+ warn(`[improve] calibration auto-tune (consolidate) skipped: ${err instanceof Error ? err.message : String(err)}`);
413
+ }
414
+ }
415
+ return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
416
+ }
417
+ /**
418
+ * Phase 0.4 — session-extract pass. Reads native session files through the
419
+ * SessionLogHarness registry, asks a bounded LLM for candidate proposals, gates
420
+ * them, and drains the extract backlog. Failures are non-fatal (collected into
421
+ * `warnings`). Returns the extract results + the gate counters seeded from the
422
+ * consolidation pass and accumulated here.
423
+ */
424
+ async function runSessionExtractPass(args) {
425
+ const { options, primaryStashDir, improveProfile, eventsCtx, seedGateAccepted, seedGateFailed } = args;
426
+ const warnings = [];
427
+ // Phase 0.4 — session-extract pass.
428
+ //
429
+ // Reads native session files (claude-code JSONL, opencode storage tree)
430
+ // through the SessionLogHarness registry, pre-filters noise, and asks a
431
+ // bounded in-tree LLM to produce candidate memory/lesson/knowledge
432
+ // proposals for content the agent did NOT preserve via inline `akm remember`
433
+ // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
434
+ // hook with an on-demand pull pipeline.
435
+ //
436
+ // Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
437
+ // (#593: the gate respects the resolved improve profile, not just the
438
+ // hardcoded `default` profile path the legacy feature flag reads).
439
+ // Each available harness gets one call with the default --since window;
440
+ // already-seen sessions (tracked in state.db.extract_sessions_seen) are
441
+ // skipped automatically so re-runs don't burn LLM calls on unchanged data.
442
+ //
443
+ // Failures are non-fatal — one harness throwing doesn't abort improve.
444
+ // The extract envelope's own `warnings` field surfaces what went wrong.
445
+ let extractResults;
446
+ // Seed the preparation-stage gate counters with consolidation's auto-accept
447
+ // gate results (#551: consolidation now runs in this stage), then accumulate
448
+ // extract's gate results on top.
449
+ let gateAutoAcceptedCount = seedGateAccepted;
450
+ let gateAutoAcceptFailedCount = seedGateFailed;
451
+ const extractConfig = options.config ?? loadConfig();
452
+ const extractGateCfg = makeGateConfig("extract", {
453
+ globalThreshold: options.autoAccept,
454
+ dryRun: options.dryRun ?? false,
455
+ stashDir: primaryStashDir,
456
+ config: extractConfig,
457
+ eventsCtx,
458
+ stateDbPath: eventsCtx?.dbPath,
459
+ });
460
+ // #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
461
+ // already done upstream; here we elide every akmExtract/processSession call)
462
+ // when the NEW (unseen, in-window) candidate-session pool is below a minimum.
463
+ // 22% of improve runs produce zero memory-inference writes because extract
464
+ // finds no new sessions, yet still burns the full extract pipeline. Default 0
465
+ // (disabled) preserves existing always-run behaviour; only opted-in profiles
466
+ // (e.g. `frequent`) set it. Evaluated BEFORE any LLM call so a skip costs zero
467
+ // LLM work AND writes nothing — which also means no extract auto-accept bumps
468
+ // memory mtimes, so a skipped extract never flags work for the NEXT run's
469
+ // consolidation mtime-gate (the downstream trigger #554 asks us to suppress).
470
+ const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
471
+ // Read from the ACTIVE resolved profile (not always `default`), matching how
472
+ // `extract.enabled` resolves — otherwise a non-default profile (e.g.
473
+ // `frequent`) setting `minNewSessions` was silently ignored.
474
+ const configuredMinNewSessions = improveProfile.processes?.extract?.minNewSessions;
475
+ const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
476
+ // #593/#594: the ACTIVE resolved improve profile is the single source of
477
+ // truth for whether extract runs. (Previously this also ANDed in the legacy
478
+ // `session_extraction` feature flag, which only reads
479
+ // `profiles.improve.default.processes.extract.enabled`; that made the default
480
+ // profile a global kill switch, so a non-default profile enabling extract was
481
+ // silently overridden. The default profile is now just another profile.)
482
+ // `akmExtract` re-checks the same active profile internally via `improveProfile`.
483
+ if (resolveProcessEnabled("extract", improveProfile)) {
484
+ const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
485
+ // The guard engages only when minNewSessions > 0; 0 disables it entirely.
486
+ let belowMinNewSessions = false;
487
+ if (minNewSessions > 0 && availableHarnesses.length > 0) {
488
+ const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
489
+ const newCandidateCount = countFn(extractConfig, {
490
+ ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
491
+ improveProfile,
492
+ // Use the ACTIVE profile's discovery window so the gate counts over the
493
+ // same window akmExtract will scan (not always `default`).
494
+ ...(improveProfile.processes?.extract?.defaultSince
495
+ ? { since: improveProfile.processes.extract.defaultSince }
496
+ : {}),
497
+ // C2: pin the candidate-count state.db open to the boundary-resolved path.
498
+ ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
499
+ });
500
+ if (newCandidateCount < minNewSessions) {
501
+ belowMinNewSessions = true;
502
+ // Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
503
+ // skipReasons aggregation surfaces this under `below_min_new_sessions`.
504
+ appendEvent({
505
+ eventType: "improve_skipped",
506
+ ref: "memory:_extract",
507
+ metadata: {
508
+ reason: "below_min_new_sessions",
509
+ newSessions: newCandidateCount,
510
+ minNewSessions,
511
+ },
512
+ }, eventsCtx);
513
+ info(`[improve] extract skipped (new sessions ${newCandidateCount} < minNewSessions ${minNewSessions})`);
514
+ }
515
+ }
516
+ if (!belowMinNewSessions && availableHarnesses.length > 0) {
517
+ extractResults = [];
518
+ for (const h of availableHarnesses) {
519
+ try {
520
+ const result = await withLlmStage("session-extraction", () => akmExtract({
521
+ type: h.name,
522
+ ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
523
+ config: extractConfig,
524
+ // Thread the ACTIVE profile so extract's internal gate + per-process
525
+ // config read the running profile, not always `default`.
526
+ improveProfile,
527
+ dryRun: options.dryRun ?? false,
528
+ ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
529
+ // C2: pin extract's skip-tracking state.db open to the boundary path.
530
+ ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
531
+ }));
532
+ extractResults.push(result);
533
+ {
534
+ const gr = await runAutoAcceptGate(primaryStashDir
535
+ ? result.proposals.map((proposalId) => {
536
+ const proposal = getProposal(primaryStashDir, proposalId);
537
+ return { proposalId, confidence: resolveExtractConfidence(proposal) };
538
+ })
539
+ : [], extractGateCfg);
540
+ gateAutoAcceptedCount += gr.promoted.length;
541
+ gateAutoAcceptFailedCount += gr.failed.length;
542
+ }
543
+ }
544
+ catch (err) {
545
+ const msg = err instanceof Error ? err.message : String(err);
546
+ warnings.push(`extract(${h.name}) failed: ${msg}`);
547
+ }
548
+ }
549
+ if (extractResults.length === 0) {
550
+ // All harnesses threw — clear so the envelope's `extract` field is
551
+ // absent rather than misleadingly empty.
552
+ extractResults = undefined;
553
+ }
554
+ }
555
+ }
556
+ // Backlog drain: gate any pending extract proposals that weren't created in
557
+ // this run (i.e. pre-date the gate or were produced by a run that timed out
558
+ // before the gate fired). Without this, eligible proposals accumulate
559
+ // indefinitely — the fresh-gate only covers the current run's output.
560
+ if (primaryStashDir && !options.dryRun && options.autoAccept !== undefined) {
561
+ const freshIds = new Set((extractResults ?? []).flatMap((r) => r.proposals));
562
+ const backlog = listProposals(primaryStashDir, { status: "pending" }).filter((p) => p.source === "extract" && !freshIds.has(p.id));
563
+ if (backlog.length > 0) {
564
+ const backlogCandidates = backlog.map((p) => ({
565
+ proposalId: p.id,
566
+ confidence: resolveExtractConfidence(p),
567
+ }));
568
+ const backlogGr = await runAutoAcceptGate(backlogCandidates, extractGateCfg);
569
+ gateAutoAcceptedCount += backlogGr.promoted.length;
570
+ gateAutoAcceptFailedCount += backlogGr.failed.length;
571
+ }
572
+ }
573
+ return { extractResults, gateAutoAcceptedCount, gateAutoAcceptFailedCount, warnings, extractGateCfg };
574
+ }
575
+ /**
576
+ * Phase 1 — validation + schema-repair pass. Scans postCleanupRefs for assets
577
+ * with structural problems (missing file, missing lesson description), attempts
578
+ * LLM schema repair, and returns the still-failing ref set + the repair records.
579
+ */
580
+ async function runValidationAndRepairPass(args) {
581
+ const { postCleanupRefs, options, startMs, budgetMs, primaryStashDir } = args;
582
+ const validationFailures = [];
583
+ for (const candidate of postCleanupRefs) {
584
+ try {
585
+ // #591: use the path pre-resolved at planning time when it is still on
586
+ // disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
587
+ // stash. Fall back to findAssetFilePath only for refs that bypassed
588
+ // collectEligibleRefs' index scan or whose file moved since planning.
589
+ const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
590
+ ? candidate.filePath
591
+ : await findAssetFilePath(candidate.ref, options.stashDir);
592
+ if (!filePath) {
593
+ validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
594
+ continue;
595
+ }
596
+ if (path.extname(filePath).toLowerCase() !== ".md") {
597
+ continue;
598
+ }
599
+ if (isLessonCandidate(candidate.ref)) {
600
+ const raw = fs.readFileSync(filePath, "utf8");
601
+ const fm = parseFrontmatter(raw).data;
602
+ if (!fm.description)
603
+ validationFailures.push({ ref: candidate.ref, reason: "missing description" });
604
+ }
605
+ }
606
+ catch (e) {
607
+ validationFailures.push({ ref: candidate.ref, reason: String(e) });
608
+ }
609
+ }
610
+ if (validationFailures.length > 0) {
611
+ info(`[improve] ${validationFailures.length} assets have validation issues (will attempt schema repair):`);
612
+ for (const f of validationFailures)
613
+ info(` ${f.ref}: ${f.reason}`);
614
+ }
615
+ let schemaRepairs = [];
616
+ let repairedRefs = new Set();
617
+ // Schema repair pass: attempt to fix validation failures via LLM before skipping.
618
+ if (validationFailures.length > 0 && options.repairValidationFailures !== false) {
619
+ const baseConfigForRepair = options.config ?? loadConfig();
620
+ const llmCfg = getDefaultLlmConfig(baseConfigForRepair);
621
+ if (llmCfg) {
622
+ const result = await runSchemaRepairPass(validationFailures, {
623
+ startMs,
624
+ budgetMs,
625
+ llmConfig: llmCfg,
626
+ // #591/#379 regression: options.stashDir is the raw, unresolved CLI
627
+ // flag (only set when --stash-dir is passed explicitly — never true
628
+ // for the scheduled tasks). primaryStashDir is the already-resolved
629
+ // source path and is what runSchemaRepairPass's `stashDir` param
630
+ // documents itself as needing ("proposal-queue writes"). Passing
631
+ // options.stashDir here made every schema-repair attempt throw
632
+ // `runSchemaRepairPass requires stashDir` on every cron invocation.
633
+ stashDir: primaryStashDir,
634
+ findFilePath: findAssetFilePath,
635
+ isLessonCandidateFn: isLessonCandidate,
636
+ });
637
+ schemaRepairs = result.repairs;
638
+ repairedRefs = result.repairedRefs;
639
+ }
640
+ }
641
+ const validationFailureRefs = new Set(validationFailures.filter((f) => !repairedRefs.has(f.ref)).map((f) => f.ref));
642
+ if (repairedRefs.size > 0) {
643
+ info(`[improve] schema repair fixed ${repairedRefs.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
644
+ }
645
+ return { validationFailures, validationFailureRefs, schemaRepairs };
646
+ }
647
+ export async function runImprovePreparationStage(args) {
648
+ const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, budgetSignal, } = args;
649
+ const actions = [];
650
+ const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
651
+ // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
652
+ let memoryIndexHealth;
653
+ if (primaryStashDir) {
654
+ const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
655
+ if (fs.existsSync(memoryMdPath)) {
656
+ try {
657
+ const lines = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
658
+ const overBudget = lines >= 180;
659
+ memoryIndexHealth = { lineCount: lines, overBudget };
660
+ if (overBudget) {
661
+ cleanupWarnings.push(`MEMORY.md has ${lines} lines (budget: 200). Consolidation strongly recommended.`);
662
+ }
663
+ }
664
+ catch {
665
+ // best-effort
666
+ }
667
+ }
668
+ }
669
+ // Phase 0.3 — memory consolidation pass (#551).
670
+ //
671
+ // Consolidation runs BEFORE the session-extract pass. This is the structural
672
+ // half of the #551 fix: extract auto-accept writes brand-new memory .md files
673
+ // on every run, which previously made the consolidation pool-delta gate fire
674
+ // unconditionally (any new file => "memory updated since last consolidate").
675
+ // By running consolidation first, the gate and akmConsolidate only ever see
676
+ // memories that existed at the start of the run — current-run extract
677
+ // promotions are not on disk yet. The complementary smarter-gate logic
678
+ // (excluding adjacent-run promotions) lives in `runConsolidationPass`.
679
+ const consolidationPass = await runConsolidationPass({
680
+ options,
681
+ primaryStashDir,
682
+ memorySummary,
683
+ improveProfile,
684
+ eventsCtx,
685
+ budgetSignal,
686
+ runBudgetMs: budgetMs,
687
+ });
688
+ // Phase 0.4 — session-extract pass (see runSessionExtractPass).
689
+ const extractPass = await runSessionExtractPass({
690
+ options,
691
+ primaryStashDir,
692
+ improveProfile,
693
+ eventsCtx,
694
+ seedGateAccepted: consolidationPass.gateAutoAcceptedCount,
695
+ seedGateFailed: consolidationPass.gateAutoAcceptFailedCount,
696
+ });
697
+ const extractResults = extractPass.extractResults;
698
+ const gateAutoAcceptedCount = extractPass.gateAutoAcceptedCount;
699
+ const gateAutoAcceptFailedCount = extractPass.gateAutoAcceptFailedCount;
700
+ if (extractPass.warnings.length > 0)
701
+ cleanupWarnings.push(...extractPass.warnings);
702
+ // eligibleCount = raw pre-filter count (before cooldown/signal/cleanup filters).
703
+ // improve_completed.plannedRefs = post-filter count of refs that actually entered the loop.
704
+ appendEvent({
705
+ eventType: "improve_invoked",
706
+ ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
707
+ metadata: { scope, dryRun: options.dryRun ?? false, eligibleCount: plannedRefs.length },
708
+ }, eventsCtx);
709
+ // ensureIndex now runs in akmImprove() BEFORE collectEligibleRefs so the
710
+ // eligible-ref query sees a populated `entries` table on the very first
711
+ // pass after a DB version upgrade (#339). Any failure messages from that
712
+ // earlier call were threaded in via args.initialCleanupWarnings.
713
+ let appliedCleanup;
714
+ try {
715
+ appliedCleanup =
716
+ primaryStashDir && memoryCleanupPlan ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan) : undefined;
717
+ }
718
+ catch (err) {
719
+ cleanupWarnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
720
+ }
721
+ const archivedRefs = appliedCleanup?.archived.map((record) => record.ref) ?? [];
722
+ const removed = new Set(archivedRefs);
723
+ const postCleanupRefs = archivedRefs.length === 0 ? plannedRefs : plannedRefs.filter((r) => !removed.has(r.ref));
724
+ // ── Phase 1: validation pass + schema repair (run on full postCleanupRefs) ──
725
+ // Identifies refs whose on-disk asset has structural problems. Validation
726
+ // failures are excluded from every downstream bucket. Run early so the
727
+ // cooldown partition operates on a clean set.
728
+ if (appliedCleanup) {
729
+ for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
730
+ const archived = appliedCleanup.archived.find((record) => record.ref === candidate.ref);
731
+ if (!archived)
732
+ continue;
733
+ actions.push({
734
+ ref: candidate.ref,
735
+ mode: "memory-prune",
736
+ result: { ok: true, pruned: true, reason: candidate.reason },
737
+ });
738
+ }
739
+ if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
740
+ try {
741
+ await reindexFn({ stashDir: primaryStashDir });
742
+ }
743
+ catch (err) {
744
+ cleanupWarnings.push(`reindex after cleanup failed: ${err instanceof Error ? err.message : String(err)}`);
745
+ }
746
+ }
747
+ }
748
+ const { validationFailures, validationFailureRefs, schemaRepairs } = await runValidationAndRepairPass({
749
+ postCleanupRefs,
750
+ options,
751
+ startMs,
752
+ budgetMs,
753
+ primaryStashDir,
754
+ });
755
+ // Phase 0.5 — structural hygiene pass
756
+ let lintSummary;
757
+ if (primaryStashDir) {
758
+ try {
759
+ const lintResult = akmLint({ fix: true, dir: primaryStashDir });
760
+ lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
761
+ }
762
+ catch {
763
+ // lint is best-effort; never block improve
764
+ }
765
+ }
766
+ // O-5 / #378: Per-originator rolling error windows.
767
+ // Reflexion (arXiv:2303.11366) warns that cross-task verbal critique
768
+ // contamination degrades below single-shot baseline. Each originator key
769
+ // ("schema-repair", "reflect") maintains its own rolling window so that
770
+ // schema-repair failures are not injected as avoidPatterns into reflect calls.
771
+ const recentErrors = {};
772
+ const RECENT_ERRORS_CAP = 3;
773
+ // Helper: push an error onto an originator's rolling window.
774
+ function pushRecentError(originator, msg) {
775
+ if (!recentErrors[originator])
776
+ recentErrors[originator] = [];
777
+ recentErrors[originator].push(msg);
778
+ if (recentErrors[originator].length > RECENT_ERRORS_CAP)
779
+ recentErrors[originator].shift();
780
+ }
781
+ // Seed schema-repair originator window from any schema-repair errors.
782
+ for (const repair of schemaRepairs) {
783
+ if (repair.outcome === "error") {
784
+ const errMsg = repair.error ?? `schema repair error: ${repair.reason}`;
785
+ pushRecentError("schema-repair", errMsg);
786
+ }
787
+ }
788
+ // ── Phase 2: signal-delta eligibility sets built EARLY ────────────────────
789
+ // 0.8.0 replaces the flat time-based cooldowns (which produced synchronised
790
+ // waves whenever many refs cooled at the same instant — see the 2026-05-26
791
+ // 54-ref simultaneous-reflect incident) with a *signal-delta* gate:
792
+ //
793
+ // reflectEligible(ref) ≡ latestFeedbackTs(ref) > lastReflectProposalTs(ref)
794
+ // distillEligible(ref) ≡ latestFeedbackTs(ref) > lastDistillProposalTs(ref)
795
+ //
796
+ // i.e. a ref is re-eligible iff new feedback has landed since the last
797
+ // proposal was generated for it. Stable content with no new signal stays
798
+ // out of the queue regardless of clock time; a sudden burst of feedback
799
+ // surfaces only the refs that the burst actually touches.
800
+ //
801
+ // The 30-day FEEDBACK_SIGNAL_WINDOW_DAYS bound still applies — only feedback
802
+ // events newer than that count as "current signal". Ancient one-off
803
+ // negatives don't permanently lock a ref into every run.
804
+ //
805
+ // High-retrieval refs (P0-A path) use a simpler "eligible once" rule: a
806
+ // ref with no feedback signal but retrievalCount ≥ threshold is eligible
807
+ // exactly once (no prior reflect proposal). Subsequent re-eligibility for
808
+ // those refs requires either a new feedback event (then the normal
809
+ // signal-delta gate applies) or human action. Documented limitation: this
810
+ // path does not re-fire on retrieval-count growth alone in 0.8.0; storing
811
+ // the retrieval count in proposal metadata for proper delta-tracking is
812
+ // captured as future work.
813
+ const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
814
+ const feedbackSinceCutoff = new Date(Date.now() - daysToMs(FEEDBACK_SIGNAL_WINDOW_DAYS)).toISOString();
815
+ // Build the three timestamp maps once across the entire postCleanupRefs set.
816
+ // Per-ref queries would be N+1 and the planner is already the hottest path
817
+ // in `akm improve`.
818
+ const candidateRefs = postCleanupRefs.filter((r) => !validationFailureRefs.has(r.ref)).map((r) => r.ref);
819
+ const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff);
820
+ const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect");
821
+ const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill");
822
+ // Refs the distill signal-delta gate rejected at planning time. The main
823
+ // loop reads this to skip distill for these refs without re-checking
824
+ // eligibility per iteration.
825
+ const distillCooledRefs = new Set();
826
+ const preCooldownCount = postCleanupRefs.length;
827
+ // ── Phase 3: partition postCleanupRefs by signal-delta eligibility ────────
828
+ // Three buckets (validation failures are excluded entirely):
829
+ // eligibleRefs — reflect signal-delta passes (full reflect+distill
830
+ // loop path; distill guard remains in the loop for
831
+ // refs that fail the distill signal-delta gate).
832
+ // distillOnlyRefs — reflect blocked but distill signal-delta passes
833
+ // AND ref is a distill candidate.
834
+ // noFeedbackPool — neither signal-delta gate passes *and* the ref has
835
+ // no recent feedback signal at all. These are NOT
836
+ // skipped here: they are handed to the high-retrieval
837
+ // fallback (P0-A) below so frequently-retrieved but
838
+ // never-rated assets can still be improved. Only refs
839
+ // that P0-A declines are ultimately fully skipped.
840
+ // fullySkippedCount — has stale feedback but no signal delta → genuine
841
+ // skip (counted, aggregated event emitted post-loop),
842
+ // excluded from sort.
843
+ const eligibleRefs = [];
844
+ const distillOnlyRefs = [];
845
+ // Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
846
+ const noFeedbackPool = [];
847
+ let fullySkippedCount = 0;
848
+ // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
849
+ const scopeRefBypass = scope.mode === "ref";
850
+ for (const r of postCleanupRefs) {
851
+ if (validationFailureRefs.has(r.ref))
852
+ continue;
853
+ if (scopeRefBypass) {
854
+ eligibleRefs.push(r);
855
+ continue;
856
+ }
857
+ const reflectOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastReflectProposalTs);
858
+ const distillOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastDistillProposalTs);
859
+ const isDistillCandidate = isDistillCandidateRef(r.ref, options.stashDir);
860
+ if (reflectOk) {
861
+ if (!distillOk && isDistillCandidate) {
862
+ // Reflect passes the gate, distill does not — emit the synthetic
863
+ // distill-skipped action and event up-front so the in-loop guard
864
+ // does not have to re-derive eligibility.
865
+ distillCooledRefs.add(r.ref);
866
+ actions.push({ ref: r.ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
867
+ appendEvent({
868
+ eventType: "improve_skipped",
869
+ ref: r.ref,
870
+ metadata: { reason: "distill_no_new_signal" },
871
+ }, eventsCtx);
872
+ }
873
+ else if (!distillOk) {
874
+ // Not a distill candidate AND distill gate doesn't pass — just mark
875
+ // distillCooled so the loop's distill section is a no-op.
876
+ distillCooledRefs.add(r.ref);
877
+ }
878
+ eligibleRefs.push(r);
879
+ }
880
+ else if (distillOk && isDistillCandidate) {
881
+ // Reflect blocked but distill passes → distill-only bucket.
882
+ distillOnlyRefs.push(r);
883
+ }
884
+ else if (!latestFeedbackTs.has(r.ref)) {
885
+ // Neither signal-delta gate passes AND there is no recent feedback signal
886
+ // at all. Rather than skip outright, defer to the high-retrieval fallback
887
+ // (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
888
+ // what that path is meant to rescue. Refs P0-A declines are skipped there.
889
+ noFeedbackPool.push(r);
890
+ }
891
+ else {
892
+ // Has feedback on record but no signal delta since the last proposal —
893
+ // genuinely fully skipped. Counted here; a single aggregated
894
+ // improve_skipped event is emitted after the loop (mirrors
895
+ // profile_filtered_all_passes) instead of one event per ref.
896
+ fullySkippedCount++;
897
+ actions.push({
898
+ ref: r.ref,
899
+ mode: "distill-skipped",
900
+ result: { ok: true, reason: "no new signal since last proposal" },
901
+ });
902
+ }
903
+ }
904
+ // Emit ONE aggregated skip event for the fully-skipped bucket rather than one
905
+ // improve_skipped event per ref (#592 pattern, mirrors
906
+ // profile_filtered_all_passes above). The per-ref loop previously produced
907
+ // ~11K state.db writes per run on a large stash, the dominant contributor to
908
+ // 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
909
+ // run summary; no downstream consumer needs a per-ref DB audit trail (health's
910
+ // skip histogram reads the `no_new_signal` counter from the count field).
911
+ if (fullySkippedCount > 0) {
912
+ appendEvent({
913
+ eventType: "improve_skipped",
914
+ ref: undefined,
915
+ metadata: {
916
+ reason: "no_new_signal",
917
+ count: fullySkippedCount,
918
+ },
919
+ }, eventsCtx);
920
+ }
921
+ // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
922
+ // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
923
+ // deferred noFeedbackPool that may be rescued by the high-retrieval fallback
924
+ // (P0-A). The fully-skipped bucket has already been routed and its aggregated
925
+ // event emitted; we deliberately avoid spending DB/CPU on refs that the
926
+ // signal-delta gate rejected with feedback already on record.
927
+ const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
928
+ // Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
929
+ // partition above could not place these in a reflect/distill bucket, but they
930
+ // may still qualify if they have been retrieved often enough. Two disjoint
931
+ // sources feed this set:
932
+ // 1. noFeedbackPool — refs with no recent feedback that the partition loop
933
+ // deliberately deferred here (otherwise they would never reach P0-A).
934
+ // 2. processableRefs entries that turn out to carry no recent feedback
935
+ // *signal* once feedbackSummary is computed below.
936
+ // (1) is added here; (2) is folded in after feedbackSummary is built.
937
+ // Gap 6: only surface feedback signals from the last 30 days so that
938
+ // ancient one-off feedback events don't permanently lock an asset into
939
+ // every improve run. Assets with only stale signals fall through to the
940
+ // high-retrieval path (P0-A) or are skipped until new signals arrive.
941
+ // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
942
+ // Phase 2 above for the signal-delta gate; we reuse them here.)
943
+ // Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
944
+ // open state.db once per asset (which caused 5000+ accumulated FDs and a
945
+ // 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
946
+ // above: one readEvents() call fetches ALL feedback events, then we aggregate
947
+ // in-memory by ref — O(1) DB opens regardless of candidate set size.
948
+ // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
949
+ // ratios are available for any noFeedbackPool ref that P0-A rescues below.
950
+ //
951
+ // Behavioral note: positive/negative COUNTS are all-time (same as the old
952
+ // per-ref readEvents call which had no `since` filter); hasSignal is bounded
953
+ // to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
954
+ const feedbackSummary = new Map();
955
+ {
956
+ const feedbackCandidateSet = new Set([...processableRefs, ...noFeedbackPool].map((r) => r.ref));
957
+ if (feedbackCandidateSet.size > 0) {
958
+ // Fetch ALL feedback events in one query (no ref filter, no since filter =
959
+ // single full table scan). Filtering per-ref in memory avoids N sequential
960
+ // state.db opens — the dominant FD-leak path on large stashes.
961
+ const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
962
+ for (const e of allFeedbackEvents) {
963
+ const ref = e.ref;
964
+ if (!ref || !feedbackCandidateSet.has(ref))
965
+ continue;
966
+ const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
967
+ const meta = e.metadata;
968
+ // hasSignal: only count feedback events within the 30-day window.
969
+ if (!entry.hasSignal &&
970
+ (e.ts ?? "") >= feedbackSinceCutoff &&
971
+ meta !== undefined &&
972
+ (typeof meta.signal === "string" || typeof meta.note === "string")) {
973
+ entry.hasSignal = true;
974
+ }
975
+ // positive/negative: all-time counts (no since filter, matching prior behaviour).
976
+ if (meta?.signal === "positive")
977
+ entry.positive++;
978
+ else if (meta?.signal === "negative")
979
+ entry.negative++;
980
+ feedbackSummary.set(ref, entry);
981
+ }
982
+ // Ensure every candidate has an entry (even refs with zero feedback events).
983
+ for (const ref of feedbackCandidateSet) {
984
+ if (!feedbackSummary.has(ref)) {
985
+ feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
986
+ }
987
+ }
988
+ }
989
+ }
990
+ const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
991
+ // P0-A: also surface zero-feedback assets that have been retrieved many times.
992
+ const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
993
+ const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
994
+ // Zero-feedback candidates for P0-A: processableRefs without a recent signal,
995
+ // plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
996
+ // disjoint by construction, but guard against overlap defensively).
997
+ const noFeedbackSeen = new Set();
998
+ const noFeedbackCandidates = [];
999
+ for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
1000
+ if (noFeedbackSeen.has(r.ref))
1001
+ continue;
1002
+ noFeedbackSeen.add(r.ref);
1003
+ noFeedbackCandidates.push(r);
1004
+ }
1005
+ let highRetrievalRefs = [];
1006
+ // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
1007
+ // proactive-maintenance selector below can reuse them without a second DB pass.
1008
+ // Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
1009
+ // step 2: recency is MANDATORY — never pinned to floor).
1010
+ let retrievalCounts = new Map();
1011
+ let lastUseMsForProactive = new Map();
1012
+ let dbForRetrieval;
1013
+ try {
1014
+ dbForRetrieval = openExistingDatabase();
1015
+ const showEventCount = countUsageEventsByType(dbForRetrieval, "show");
1016
+ if (showEventCount === 0) {
1017
+ warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
1018
+ }
1019
+ // Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
1020
+ // Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
1021
+ // had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
1022
+ // to 0 regardless of actual use. Two assets of the same type — one
1023
+ // heavily-retrieved, one never-touched — would receive identical rankScores.
1024
+ // Fix (WS-1 blocker 3): union the feedback pool into the lookup.
1025
+ const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
1026
+ retrievalCounts = getRetrievalCounts(dbForRetrieval, allCandidateRefs);
1027
+ lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
1028
+ // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
1029
+ // ref qualifies exactly once — when it has actually been retrieved
1030
+ // (retrievalCount ≥ 1) AND retrievalCount ≥ threshold AND no prior reflect
1031
+ // proposal exists for it. Once a reflect proposal is on record, subsequent
1032
+ // re-eligibility requires explicit feedback (which flows through the normal
1033
+ // signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
1034
+ // from rescuing genuinely never-retrieved assets — the fallback is for
1035
+ // *retrieved* assets, not silent ones. Tracking growth in retrieval count
1036
+ // would require persisting the count in proposal metadata; deferred to a
1037
+ // follow-up.
1038
+ highRetrievalRefs = noFeedbackCandidates.filter((r) => {
1039
+ const count = retrievalCounts.get(r.ref) ?? 0;
1040
+ return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
1041
+ });
1042
+ }
1043
+ catch (err) {
1044
+ rethrowIfTestIsolationError(err);
1045
+ // best-effort: if DB unavailable, highRetrievalRefs stays empty
1046
+ }
1047
+ finally {
1048
+ if (dbForRetrieval)
1049
+ closeDatabase(dbForRetrieval);
1050
+ }
1051
+ // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
1052
+ // The signal-delta gate and P0-A only surface assets with fresh feedback or a
1053
+ // raw-retrieval spike. Neither revisits a stable, high-value asset on a
1054
+ // schedule, so on a quiet stash useful assets drift stale and are never
1055
+ // refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
1056
+ // and the run is whole-stash / type scope, this selector ranks the eligible
1057
+ // population by a composite maintenance priority, gates on staleness ("due"),
1058
+ // bounds to top-N, and folds the winners into the SAME candidate set the other
1059
+ // two sources feed — so they flow through the existing #580 empty-diff /
1060
+ // cosmetic suppression and additive-distill gates. It adds no new mutation
1061
+ // logic of its own. The due gate doubles as the rotation cooldown: a freshly
1062
+ // reflected asset is excluded until it ages back past `dueDays`, so successive
1063
+ // runs rotate through the due pool rather than re-selecting the same heads.
1064
+ let proactiveRefs = [];
1065
+ let proactiveMaintenanceSummary;
1066
+ const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
1067
+ if (proactiveEnabled) {
1068
+ const pmCfg = improveProfile.processes?.proactiveMaintenance;
1069
+ const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
1070
+ const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
1071
+ // Candidate population: the zero-feedback / non-signal pool — exactly the
1072
+ // assets the other two sources would NOT pick this run. Exclude any P0-A
1073
+ // rescued this run so we never double-select the same ref.
1074
+ const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
1075
+ const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
1076
+ const selection = selectProactiveMaintenanceRefs({
1077
+ candidates: pmCandidates,
1078
+ lastReflectTs: lastReflectProposalTs,
1079
+ lastDistillTs: lastDistillProposalTs,
1080
+ retrievalCounts,
1081
+ // WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
1082
+ lastUseMs: lastUseMsForProactive,
1083
+ sizeBytesOf: (r) => {
1084
+ const fp = r.filePath;
1085
+ if (!fp)
1086
+ return undefined;
1087
+ try {
1088
+ return fs.statSync(fp).size;
1089
+ }
1090
+ catch {
1091
+ return undefined;
1092
+ }
1093
+ },
1094
+ dueDays,
1095
+ maxPerRun,
1096
+ });
1097
+ proactiveRefs = selection.selected;
1098
+ proactiveMaintenanceSummary = {
1099
+ selected: selection.selected.length,
1100
+ dueTotal: selection.dueTotal,
1101
+ neverReflected: selection.neverReflected,
1102
+ };
1103
+ // Aggregated observability event (never per-ref — avoids the event flood the
1104
+ // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
1105
+ appendEvent({
1106
+ eventType: "proactive_selected",
1107
+ ref: undefined,
1108
+ metadata: {
1109
+ count: selection.selected.length,
1110
+ dueTotal: selection.dueTotal,
1111
+ neverReflected: selection.neverReflected,
1112
+ },
1113
+ }, eventsCtx);
1114
+ if (selection.selected.length > 0) {
1115
+ info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
1116
+ `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
1117
+ }
1118
+ }
1119
+ // ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
1120
+ // Zero-feedback refs whose encoding_salience (set at distill time by
1121
+ // scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
1122
+ // into the improve run even without retrieval or feedback signal. This rescues
1123
+ // newly distilled assets that the stash has not yet surfaced to users.
1124
+ //
1125
+ // Cap: at most 10% of the effective run limit so the lane cannot crowd out
1126
+ // reactive feedback. Requires state.db to have an asset_salience row — refs
1127
+ // without a row (pre-#608 assets still on the type-weight stub) are skipped.
1128
+ //
1129
+ // Cooldown: a ref qualifies at most once — when no prior reflect proposal
1130
+ // exists for it (`!lastReflectProposalTs.has`). Without this guard the lane
1131
+ // re-selects the same high-salience refs on EVERY run (auto-accept emits a
1132
+ // `promoted` event, not `feedback`, so the ref never leaves
1133
+ // noFeedbackCandidates), burning LLM calls and churning the asset. This
1134
+ // mirrors the P0-A high-retrieval gate's `!lastReflectProposalTs.has(r.ref)`
1135
+ // guard above so all "rescue" lanes share the same once-per-asset semantics.
1136
+ //
1137
+ // Content-provenance gate (#644 follow-up): the row must ALSO carry a genuine
1138
+ // content-derived encoding score (`isContentEncodingRow`). Otherwise the lane
1139
+ // admits the per-type WEIGHT STUB (skill/agent 0.9, command/workflow 0.8,
1140
+ // lesson 0.75 from DEFAULT_TYPE_ENCODING_WEIGHTS) for every distill-unscored
1141
+ // asset — i.e. "high-salience" degenerates into "is a skill/agent/command/
1142
+ // lesson", which selected the lore-writer type-stub agent on every run. Only
1143
+ // content-scored assets earn the high-salience rescue; type-stub rows must
1144
+ // earn retrieval/feedback signal via the other lanes. This PRESERVES #608's
1145
+ // intent — distilled assets (the lane's real targets) keep their real content
1146
+ // score and still qualify — while cutting the type-stub waste. See §5 F1 of
1147
+ // docs/design/improve-salience-working-reference.md and #608/#644.
1148
+ const highSalienceRefs = [];
1149
+ const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
1150
+ const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
1151
+ const proactiveAndRetrievalSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
1152
+ try {
1153
+ withStateDb((dbForHighSalience) => {
1154
+ // Derive the cap from the resolved reflect limit (mirrors improve.ts's
1155
+ // options.limit resolution) so an unbounded whole-stash run does not
1156
+ // collapse the lane to exactly 1 ref via the bare `?? 10` fallback.
1157
+ const effectiveLimit = options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit ?? 10;
1158
+ const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
1159
+ const candidates = noFeedbackCandidates.filter((r) => !proactiveAndRetrievalSet.has(r.ref));
1160
+ // Collect ALL qualifying candidates, then take the top-N BY SCORE — the
1161
+ // previous first-N-in-scan-order break meant a higher-salience candidate
1162
+ // found later in the scan lost its slot to an earlier lower-scoring one.
1163
+ const qualifying = [];
1164
+ for (const r of candidates) {
1165
+ const row = getAssetSalience(dbForHighSalience, r.ref);
1166
+ if (row &&
1167
+ isContentEncodingRow(row, parseAssetRef(r.ref).type) &&
1168
+ row.encoding_salience >= salienceThreshold &&
1169
+ !lastReflectProposalTs.has(r.ref)) {
1170
+ qualifying.push({ ref: r, score: row.encoding_salience });
1171
+ }
1172
+ }
1173
+ qualifying.sort((a, b) => b.score - a.score);
1174
+ for (const q of qualifying.slice(0, highSalienceCap)) {
1175
+ highSalienceRefs.push(q.ref);
1176
+ }
1177
+ }, { path: eventsCtx?.dbPath });
1178
+ }
1179
+ catch (err) {
1180
+ rethrowIfTestIsolationError(err);
1181
+ // best-effort: if DB unavailable, highSalienceRefs stays empty
1182
+ }
1183
+ if (highSalienceRefs.length > 0) {
1184
+ info(`[improve] high-salience lane admitted ${highSalienceRefs.length} content-scored ref(s) ` +
1185
+ `(threshold=${salienceThreshold}, requires content-derived encoding_source)`);
1186
+ }
1187
+ // Record an in-memory skip action for every zero-feedback ref that the
1188
+ // partition loop deferred to P0-A but P0-A then declined (retrievalCount below
1189
+ // threshold, or a prior reflect proposal already on record). These never make
1190
+ // it into mergedRefs, so without this they would silently vanish from the run
1191
+ // summary. No DB event is written here — these refs carry no signal at all, so
1192
+ // there is nothing for the skip histogram to aggregate; the action log alone
1193
+ // preserves the per-ref audit trail (mirrors the fully-skipped action above).
1194
+ const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
1195
+ for (const r of noFeedbackPool) {
1196
+ if (rescuedSet.has(r.ref))
1197
+ continue;
1198
+ actions.push({
1199
+ ref: r.ref,
1200
+ mode: "distill-skipped",
1201
+ result: { ok: true, reason: "no new signal since last proposal" },
1202
+ });
1203
+ }
1204
+ // If the user explicitly scoped to a single ref, always act on it —
1205
+ // skip the signal/retrieval filter entirely. The filter exists to avoid
1206
+ // noisy "improve everything" runs; it should not gate an intentional
1207
+ // per-ref invocation where the user's explicit choice is the signal.
1208
+ //
1209
+ // For type/all scope: only process refs with usage signals (recent feedback
1210
+ // or sufficient retrievals). A stash with no signals has 0 eligible refs —
1211
+ // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
1212
+ // to bring them into the eligible pool.
1213
+ // Layer-2 proactive refs join the eligible set alongside feedback-signal and
1214
+ // high-retrieval (P0-A) refs. The four sources are disjoint by construction
1215
+ // (proactive draws from noFeedbackCandidates with the P0-A picks removed, and
1216
+ // high-salience draws from the remainder), but dedupe defensively so a ref can
1217
+ // never enter the loop twice. `requireFeedbackSignal` still suppresses all
1218
+ // fallback sources for callers that want feedback-only runs.
1219
+ const signalAndRetrievalRefs = dedupeRefs([
1220
+ ...signalFiltered,
1221
+ ...highRetrievalRefs,
1222
+ ...proactiveRefs,
1223
+ ...highSalienceRefs,
1224
+ ]);
1225
+ let mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
1226
+ // ── Attribution tagging: stamp each ref with the eligibility lane that
1227
+ // selected it ──────────────────────────────────────────────────────────────
1228
+ // Every reflect/distill proposal must record WHICH lane chose its source asset
1229
+ // so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
1230
+ // (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
1231
+ // lane map here — the one place all four lanes are known — and stamp it onto
1232
+ // each ImproveEligibleRef object. Because the ref objects are shared by
1233
+ // reference across buckets, the stamp travels with the ref through the sort,
1234
+ // disk-check, and loop stages down to the reflect/distill event emit sites and
1235
+ // createProposal calls. See EligibilitySource for the lane vocabulary.
1236
+ //
1237
+ // Precedence (prefer the most specific reactive signal):
1238
+ // scope > signal-delta > high-retrieval > proactive > high-salience
1239
+ // A ref with real feedback is attributed to feedback even if it was also due
1240
+ // for proactive maintenance or had high encoding salience. We apply lanes
1241
+ // weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
1242
+ // wins outright (user intent).
1243
+ const eligibilitySourceByRef = new Map();
1244
+ for (const r of highSalienceRefs)
1245
+ eligibilitySourceByRef.set(r.ref, "high-salience");
1246
+ for (const r of proactiveRefs)
1247
+ eligibilitySourceByRef.set(r.ref, "proactive");
1248
+ for (const r of highRetrievalRefs)
1249
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
1250
+ for (const r of signalFiltered)
1251
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
1252
+ if (scope.mode === "ref") {
1253
+ // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
1254
+ // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
1255
+ for (const r of processableRefs)
1256
+ eligibilitySourceByRef.set(r.ref, "scope");
1257
+ }
1258
+ for (const r of mergedRefs) {
1259
+ // "unknown" is a genuine fallback, never a silent alias for signal-delta:
1260
+ // only refs we truly cannot attribute land here (none in practice, since
1261
+ // mergedRefs is always a subset of the four lanes above).
1262
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
1263
+ }
1264
+ // WS-1 — Unified salience vector (S1 seam).
1265
+ //
1266
+ // The legacy sort combined three independent formulas (utility EMA, negative-only
1267
+ // ratio / symmetric-valence magnitude, and the proactive-maintenance priority
1268
+ // formula). WS-1 converges them into one `computeSalience()` call per ref, with
1269
+ // three independently-stored sub-scores and one documented rankScore projection.
1270
+ //
1271
+ // Migration note: if a profile still has `symmetricValence` set, emit a one-time
1272
+ // warning — its behaviour (symmetric |valence| attention) is now always-on as
1273
+ // part of the salience vector, so the knob is a no-op and will be removed in 0.10.
1274
+ if (improveProfile.symmetricValence === true) {
1275
+ warn("[improve] Profile option 'symmetricValence' is deprecated (WS-1 salience vector). " +
1276
+ "Symmetric valence is now always active; remove the option from your improve profile.");
1277
+ }
1278
+ // Fetch last-use timestamps from the index DB for the full merged set so the
1279
+ // recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
1280
+ // This reuses the index DB opened earlier for retrieval counts; a separate
1281
+ // lightweight open is used here to avoid holding the connection longer than needed.
1282
+ let lastUseMsByRef = new Map();
1283
+ // utilityMap is kept for backward-compatible observability (health report reads it).
1284
+ const utilityMap = buildUtilityMap(mergedRefs);
1285
+ let dbForSalience;
1286
+ try {
1287
+ dbForSalience = openExistingDatabase();
1288
+ lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs.map((r) => r.ref));
1289
+ }
1290
+ catch (err) {
1291
+ rethrowIfTestIsolationError(err);
1292
+ // best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
1293
+ }
1294
+ finally {
1295
+ if (dbForSalience)
1296
+ closeDatabase(dbForSalience);
1297
+ }
1298
+ // ── WS-2 Outcome loop ─────────────────────────────────────────────────────
1299
+ //
1300
+ // Update asset_outcome for every ref in the merged set BEFORE computing the
1301
+ // salience vector so the updated outcome_score feeds outcomeSalience this run.
1302
+ //
1303
+ // Inputs per ref:
1304
+ // - currentRetrievalCount: from retrievalCounts (index DB)
1305
+ // - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
1306
+ // - negativeFeedbackCount: cumulative negatives from feedbackSummary
1307
+ // - acceptedChangeCount: accepted proposals for this ref (state.db)
1308
+ // - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
1309
+ // - utilityScore: from utilityMap (for warm-start seed on new rows)
1310
+ //
1311
+ // Best-effort: outcome failures never block the salience or ranking pass.
1312
+ const outcomeSalienceByRef = new Map();
1313
+ try {
1314
+ withStateDb((outcomeDb) => {
1315
+ // Count accepted proposals per ref in one pass (avoid N separate queries).
1316
+ // Scoped to primaryStashDir when available so multi-stash installs don't
1317
+ // inflate counts with proposals from other stashes.
1318
+ const acceptedCountByRef = new Map();
1319
+ try {
1320
+ const acceptedProposals = listStateProposals(outcomeDb, {
1321
+ status: "accepted",
1322
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
1323
+ });
1324
+ for (const p of acceptedProposals) {
1325
+ acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
1326
+ }
1327
+ }
1328
+ catch {
1329
+ // best-effort: if proposals query fails, accepted counts stay at 0
1330
+ }
1331
+ // Update each ref's outcome row and collect the resulting outcome scores.
1332
+ const rawOutcomeScores = new Map();
1333
+ const nowForOutcome = Date.now();
1334
+ for (const r of mergedRefs) {
1335
+ const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
1336
+ const valenceResult = computeValenceScore(fb);
1337
+ try {
1338
+ const result = updateAssetOutcome(outcomeDb, {
1339
+ ref: r.ref,
1340
+ currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
1341
+ lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
1342
+ acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
1343
+ negativeFeedbackCount: fb.negative,
1344
+ valence: valenceResult.valence,
1345
+ utilityScore: utilityMap.get(r.ref),
1346
+ now: nowForOutcome,
1347
+ });
1348
+ rawOutcomeScores.set(r.ref, result.outcomeScore);
1349
+ }
1350
+ catch {
1351
+ // best-effort per-ref: skip this ref's outcome update on failure
1352
+ }
1353
+ }
1354
+ // Compute stash-wide max outcome_score for normalisation (diversity floor).
1355
+ // Read ALL rows (not just this run's batch) so the normalisation is
1356
+ // stash-relative, not pool-relative.
1357
+ let maxOutcomeScore = 0;
1358
+ try {
1359
+ const allOutcomes = getAllAssetOutcomes(outcomeDb);
1360
+ for (const row of allOutcomes) {
1361
+ if (row.outcome_score > maxOutcomeScore)
1362
+ maxOutcomeScore = row.outcome_score;
1363
+ }
1364
+ // Read-clip: legacy rows written before the OUTCOME_SCORE_MAX write-clip
1365
+ // existed can sit above the ceiling (live max was 3.13). Without this
1366
+ // clip they inflate the normalisation denominator and floor everyone
1367
+ // else's outcomeSalience (#691 follow-up).
1368
+ maxOutcomeScore = Math.min(maxOutcomeScore, OUTCOME_SCORE_MAX);
1369
+ // Proxy-adequacy tripwire (two-tailed): inverted (corr < −0.3) and
1370
+ // dead (|corr| < 0.1 at n ≥ 500) both emit health events.
1371
+ const adequacy = computeProxyAdequacy(allOutcomes);
1372
+ if (adequacy.isInverted) {
1373
+ appendEvent({
1374
+ eventType: "outcome_proxy_inverted",
1375
+ ref: undefined,
1376
+ metadata: {
1377
+ correlation: adequacy.correlation,
1378
+ n: adequacy.n,
1379
+ note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
1380
+ },
1381
+ }, eventsCtx);
1382
+ }
1383
+ if (adequacy.isDead) {
1384
+ appendEvent({
1385
+ eventType: "outcome_proxy_dead",
1386
+ ref: undefined,
1387
+ metadata: {
1388
+ correlation: adequacy.correlation,
1389
+ n: adequacy.n,
1390
+ note: "|corr(outcome_score, accepted_change_rate)| < 0.1 at n ≥ 500: outcome_score is statistically unrelated to improvement outcomes — the proxy is noise, not signal. Rank contributions derived from it are not currently informative.",
1391
+ },
1392
+ }, eventsCtx);
1393
+ }
1394
+ }
1395
+ catch {
1396
+ // best-effort: tripwire failure never blocks ranking
1397
+ }
1398
+ // Convert raw outcome scores → normalised outcomeSalience values in [0,1].
1399
+ for (const [ref, score] of rawOutcomeScores) {
1400
+ const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
1401
+ outcomeSalienceByRef.set(ref, normalised);
1402
+ }
1403
+ // Also fetch outcome scores for refs NOT updated this run (stale or absent)
1404
+ // so the outcomeSalience read path works for all refs in the batch.
1405
+ const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
1406
+ if (missingRefs.length > 0) {
1407
+ const storedScores = getOutcomeScoresByRef(outcomeDb, missingRefs);
1408
+ for (const [ref, score] of storedScores) {
1409
+ outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
1410
+ }
1411
+ }
1412
+ }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1413
+ }
1414
+ catch (err) {
1415
+ rethrowIfTestIsolationError(err);
1416
+ // best-effort: outcome failures never block salience computation
1417
+ }
1418
+ // Compute the salience vector for every ref in the merged set.
1419
+ // retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
1420
+ // so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
1421
+ // outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
1422
+ //
1423
+ // R1 loop closure: the outcome weight is ON by default (the G2 saturation
1424
+ // cap makes it safe). Operators opt out with
1425
+ // improve.salience.outcomeWeightEnabled: false in the config.
1426
+ const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
1427
+ const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled !== false;
1428
+ const salienceMap = new Map();
1429
+ const nowForSalience = Date.now();
1430
+ // #644 — preserve content-derived encoding scores across runs.
1431
+ //
1432
+ // Before computing the salience vector, load each ref's stored encoding score
1433
+ // and its provenance. When the stored row carries a genuine content-derived
1434
+ // score (written by the distill path via `scoreEncodingSalience`), pass that
1435
+ // value back in as `inputs.encodingSalience` so `computeSalience` does NOT fall
1436
+ // back to the type-weight stub — keeping both the persisted `encoding_salience`
1437
+ // AND the derived `rank_score` keyed on real novelty/magnitude/prediction-error.
1438
+ // Refs that have never been content-scored keep the type-weight stub fallback.
1439
+ const storedEncodingByRef = new Map();
1440
+ try {
1441
+ withStateDb((dbForStoredEncoding) => {
1442
+ for (const r of mergedRefs) {
1443
+ const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
1444
+ const row = getAssetSalience(dbForStoredEncoding, r.ref);
1445
+ if (row && isContentEncodingRow(row, type)) {
1446
+ storedEncodingByRef.set(r.ref, row.encoding_salience);
1447
+ }
1448
+ }
1449
+ }, { path: eventsCtx?.dbPath });
1450
+ }
1451
+ catch (err) {
1452
+ rethrowIfTestIsolationError(err);
1453
+ // best-effort: if DB unavailable, fall back to type-weight stub (prior behaviour)
1454
+ }
1455
+ for (const r of mergedRefs) {
1456
+ const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
1457
+ const sizeBytes = (() => {
1458
+ const fp = r.filePath;
1459
+ if (!fp)
1460
+ return undefined;
1461
+ try {
1462
+ return fs.statSync(fp).size;
1463
+ }
1464
+ catch {
1465
+ return undefined;
1466
+ }
1467
+ })();
1468
+ const storedEncoding = storedEncodingByRef.get(r.ref);
1469
+ const vector = computeSalience({
1470
+ ref: r.ref,
1471
+ type,
1472
+ // #644: pass the stored content-derived score (if any) so the type-weight
1473
+ // stub is NOT re-asserted over a real distill-written encoding score.
1474
+ ...(storedEncoding !== undefined ? { encodingSalience: storedEncoding } : {}),
1475
+ retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
1476
+ lastUseMs: lastUseMsByRef.get(r.ref),
1477
+ utilityScore: utilityMap.get(r.ref),
1478
+ outcomeSalience: outcomeSalienceByRef.get(r.ref),
1479
+ sizeBytes,
1480
+ now: nowForSalience,
1481
+ outcomeWeightEnabled,
1482
+ });
1483
+ salienceMap.set(r.ref, vector);
1484
+ }
1485
+ // Persist salience vectors to state.db (best-effort, non-blocking).
1486
+ // The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
1487
+ //
1488
+ // Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
1489
+ //
1490
+ // BEFORE persisting the new rankScores, read ALL existing rows from state.db
1491
+ // (not just the per-run candidate pool). This gives stash-wide rank positions so
1492
+ // the top-200/below-500 thresholds are meaningful.
1493
+ //
1494
+ // Two distinct scenarios:
1495
+ //
1496
+ // A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
1497
+ // ordering was never persisted in state.db (asset_salience is a new WS-1 table).
1498
+ // However, the old formula's inputs are available in-scope for every candidate
1499
+ // in the current pool: utility comes from utilityMap and the attention term
1500
+ // from feedbackSummary (positive/negative counts). We reconstruct the old
1501
+ // combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
1502
+ // for every ref in salienceMap and rank them, giving a candidate-pool-scoped
1503
+ // old ordering. This is a partial reconstruction (only current-pool refs, not
1504
+ // stash-wide), but it is the most faithful comparison possible at cutover and
1505
+ // allows the top-200→below-500 forgetting guard to fire if the formula change
1506
+ // dramatically reorders the candidate pool.
1507
+ // See docs/archive/improve-reconciliation-plan.md §WS-1 step 7 — the stash-wide
1508
+ // ordering was unreconstructable (no prior state.db snapshot), so this candidate-
1509
+ // pool partial reconstruction is the documented resolution for the first-run case.
1510
+ // Emit `improve_salience_first_run` to mark the cutover moment and include the
1511
+ // reconstructed comparison result in the metadata.
1512
+ //
1513
+ // B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
1514
+ // them with the current run's salienceMap updates for new ranks, and call
1515
+ // buildRankChangeReport with stash-wide positions. This detects real rank drift
1516
+ // — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
1517
+ // below position 500.
1518
+ //
1519
+ // Measurement-protocol deferral (plan §269, Part-V):
1520
+ // The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
1521
+ // quality gate are deferred pending owner sign-off. Full measurement requires a
1522
+ // before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
1523
+ // will re-introduce outcome salience and trigger the full re-tuning pass at that
1524
+ // time. salience.ts already accepts outcomeSalience directly as an input
1525
+ // (see SalienceInputs.outcomeSalience); no separate hook is needed.
1526
+ //
1527
+ // Forgetting-safety collection: populated inside scenario B below, consumed
1528
+ // after the try/catch to union candidates into mergedRefs before the sort.
1529
+ // Only refs from a real pre-existing ordering (scenario B) are collected;
1530
+ // empty on scenario A or when no candidates dropped below the threshold.
1531
+ let pendingForgettingRefs = [];
1532
+ try {
1533
+ withStateDb((stateDb) => {
1534
+ // Step 7: stash-wide rank-change report BEFORE overwriting the table.
1535
+ //
1536
+ // Load ALL existing rows so rank positions are stash-relative, not pool-relative.
1537
+ const existingAllScores = getAllRankScores(stateDb);
1538
+ if (existingAllScores.size === 0) {
1539
+ // Scenario A: first WS-1 run — table empty.
1540
+ //
1541
+ // Reconstruct the old combinedEligibilityScore ordering for the current
1542
+ // candidate pool using inputs that are already in-scope: utility from
1543
+ // utilityMap and the attention term from feedbackSummary (positive/negative
1544
+ // counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
1545
+ //
1546
+ // Limitation: this covers only the current-run candidate pool, not the full
1547
+ // stash. The stash-wide ordering was never persisted (asset_salience is a new
1548
+ // WS-1 table), so this is the most faithful comparison possible at cutover.
1549
+ // See docs/archive/improve-reconciliation-plan.md §WS-1 step 7.
1550
+ const reconstructedOldScores = new Map();
1551
+ for (const ref of salienceMap.keys()) {
1552
+ const utility = utilityMap.get(ref) ?? 0;
1553
+ const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
1554
+ const attention = computeValenceScore(fb).attention;
1555
+ reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
1556
+ }
1557
+ // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
1558
+ const toRanks = (scores) => {
1559
+ const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
1560
+ return new Map(sorted.map(([ref], i) => [ref, i + 1]));
1561
+ };
1562
+ const oldRanks = toRanks(reconstructedOldScores);
1563
+ const newRanks = toRanks(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
1564
+ const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
1565
+ if (firstRunReport.forgettingCandidates.length > 0) {
1566
+ warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
1567
+ `Top drops: ${firstRunReport.forgettingCandidates
1568
+ .slice(0, 5)
1569
+ .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
1570
+ .join(", ")}`);
1571
+ pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
1572
+ }
1573
+ appendEvent({
1574
+ eventType: "improve_salience_first_run",
1575
+ ref: undefined,
1576
+ metadata: {
1577
+ candidateCount: salienceMap.size,
1578
+ note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); see improve-reconciliation-plan.md §WS-1 step 7",
1579
+ forgettingCandidates: firstRunReport.forgettingCandidates.length,
1580
+ topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
1581
+ ref: e.ref,
1582
+ oldRank: e.oldRank,
1583
+ newRank: e.newRank,
1584
+ })),
1585
+ },
1586
+ }, eventsCtx);
1587
+ }
1588
+ else {
1589
+ // Scenario B: subsequent run — compare stash-wide old vs. new ranks.
1590
+ //
1591
+ // Build new scores by merging the full table with this run's updates.
1592
+ // Refs in salienceMap override their stored value; refs not in this run
1593
+ // retain their stored value unchanged. This gives a complete stash-wide
1594
+ // picture of what the new ordering looks like after this run.
1595
+ const mergedNewScores = new Map(existingAllScores);
1596
+ for (const [ref, vector] of salienceMap) {
1597
+ mergedNewScores.set(ref, vector.rankScore);
1598
+ }
1599
+ // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
1600
+ const toRanks = (scores) => {
1601
+ const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
1602
+ return new Map(sorted.map(([ref], i) => [ref, i + 1]));
1603
+ };
1604
+ const oldRanks = toRanks(existingAllScores);
1605
+ const newRanks = toRanks(mergedNewScores);
1606
+ const report = buildRankChangeReport(oldRanks, newRanks);
1607
+ if (report.forgettingCandidates.length > 0) {
1608
+ warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
1609
+ `Top drops: ${report.forgettingCandidates
1610
+ .slice(0, 5)
1611
+ .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
1612
+ .join(", ")}`);
1613
+ // Collect refs for protective consolidation pass (plan §WS-1 step 7).
1614
+ // These are force-included in the candidate pool (mergedRefs) after
1615
+ // this try block, bypassing cooldown/signal-delta gating.
1616
+ pendingForgettingRefs = report.forgettingCandidates.map((e) => e.ref);
1617
+ }
1618
+ appendEvent({
1619
+ eventType: "improve_salience_rank_change",
1620
+ ref: undefined,
1621
+ metadata: {
1622
+ stashSize: existingAllScores.size,
1623
+ totalChanged: report.allChanges.length,
1624
+ forgettingCandidates: report.forgettingCandidates.length,
1625
+ topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
1626
+ ref: e.ref,
1627
+ oldRank: e.oldRank,
1628
+ newRank: e.newRank,
1629
+ })),
1630
+ },
1631
+ }, eventsCtx);
1632
+ }
1633
+ for (const [ref, vector] of salienceMap) {
1634
+ upsertAssetSalience(stateDb, ref, vector, nowForSalience);
1635
+ }
1636
+ }, { path: eventsCtx?.dbPath });
1637
+ }
1638
+ catch (err) {
1639
+ rethrowIfTestIsolationError(err);
1640
+ // best-effort: salience persistence failure never blocks ranking
1641
+ }
1642
+ // ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
1643
+ // Forgetting candidates detected in scenario B are force-injected into
1644
+ // mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
1645
+ // signal-delta gating. Any ref already present in mergedRefs keeps its
1646
+ // existing eligibilitySource (stronger reactive signals win); refs not yet in
1647
+ // the pool are synthesised as minimal ImproveEligibleRef stubs and labelled
1648
+ // 'forgetting-safety' so S5/WS-5 can slice by lane. The dedupeRefs call
1649
+ // ensures no ref can enter the loop twice.
1650
+ if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
1651
+ const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
1652
+ const newForgettingRefs = [];
1653
+ for (const ref of pendingForgettingRefs) {
1654
+ if (!existingRefSet.has(ref)) {
1655
+ // Ref not already in the candidate pool — synthesise a stub so it
1656
+ // participates in the reflect/distill loop with proper attribution.
1657
+ newForgettingRefs.push({ ref, reason: "scope-type", eligibilitySource: "forgetting-safety" });
1658
+ }
1659
+ // Always stamp the lane in the attribution map (overwrites weaker lanes;
1660
+ // stronger reactive signals — scope/signal-delta/high-retrieval/proactive
1661
+ // — are written after this block so they take precedence).
1662
+ eligibilitySourceByRef.set(ref, "forgetting-safety");
1663
+ }
1664
+ if (newForgettingRefs.length > 0) {
1665
+ mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
1666
+ }
1667
+ // Re-stamp attribution for any refs whose lane needs updating.
1668
+ // Precedence (weakest → strongest, each overwrites the previous):
1669
+ // proactive < high-retrieval < forgetting-safety < signal-delta
1670
+ // Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
1671
+ // forgetting-safety sits above proactive and high-retrieval so that a ref
1672
+ // flagged as a forgetting candidate is always visible to S5/WS-5 as such,
1673
+ // even when it was also due for a proactive maintenance run. signal-delta
1674
+ // overrides forgetting-safety because a ref with fresh feedback is reactive
1675
+ // and doesn't need the protective pass label for measurement purposes.
1676
+ for (const r of highSalienceRefs)
1677
+ eligibilitySourceByRef.set(r.ref, "high-salience");
1678
+ for (const r of proactiveRefs)
1679
+ eligibilitySourceByRef.set(r.ref, "proactive");
1680
+ for (const r of highRetrievalRefs)
1681
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
1682
+ // Apply forgetting-safety OVER proactive, high-retrieval, and high-salience
1683
+ // (already stamped in the loop above via
1684
+ // `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
1685
+ // set() calls above for proactive/high-retrieval/high-salience overwrite the
1686
+ // earlier forgetting-safety stamp — so we re-apply forgetting-safety now for
1687
+ // those refs that are both forgetting candidates AND in another fallback lane.
1688
+ for (const ref of pendingForgettingRefs) {
1689
+ eligibilitySourceByRef.set(ref, "forgetting-safety");
1690
+ }
1691
+ // signal-delta is the strongest reactive signal and overrides forgetting-safety.
1692
+ for (const r of signalFiltered)
1693
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
1694
+ // Update eligibilitySource on the ref objects themselves for any refs whose
1695
+ // lane changed (covers both new stubs and pre-existing refs).
1696
+ for (const r of mergedRefs) {
1697
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
1698
+ }
1699
+ }
1700
+ // ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
1701
+ // Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
1702
+ // revisited even with zero reactive signal (no feedback, no retrieval) and
1703
+ // regardless of cooldown — exactly like the forgetting-safety lane, replay is
1704
+ // injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
1705
+ //
1706
+ // Strictly additive: the replay slice is appended AFTER the --limit fresh slice
1707
+ // (see the loopRefs partition below), so it can never shrink the fresh-ref set.
1708
+ // Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
1709
+ // and budget is spent only on refs not already in mergedRefs (so a stronger lane
1710
+ // never has its budget wasted or its label overwritten).
1711
+ //
1712
+ // Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
1713
+ // no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
1714
+ const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
1715
+ const replayRefSet = new Set();
1716
+ if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
1717
+ try {
1718
+ withStateDb((replayDb) => {
1719
+ const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
1720
+ const allRankScores = getAllRankScores(replayDb);
1721
+ // Candidate universe = every salience row NOT already in the pool, ordered by
1722
+ // rank_score desc with a deterministic ref-string tie-break (mirrors the main
1723
+ // sort). Converged refs (consecutive_no_ops >= dampener threshold) are fully
1724
+ // EXCLUDED — a stronger skip than the dampener (which only halves order).
1725
+ let convergedSkipped = 0;
1726
+ const candidates = [];
1727
+ for (const [ref, rankScore] of allRankScores) {
1728
+ if (alreadyInPool.has(ref))
1729
+ continue;
1730
+ const noOps = getConsecutiveNoOps(replayDb, ref);
1731
+ if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
1732
+ convergedSkipped++;
1733
+ continue;
1734
+ }
1735
+ candidates.push({ ref, rankScore });
1736
+ }
1737
+ candidates.sort((a, b) => b.rankScore !== a.rankScore ? b.rankScore - a.rankScore : a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0);
1738
+ const candidatePool = candidates.length;
1739
+ const selected = candidates.slice(0, replayBudget);
1740
+ const newReplayRefs = [];
1741
+ for (const { ref } of selected) {
1742
+ replayRefSet.add(ref);
1743
+ // Synthesise a stub (mirror the forgetting-safety stub). Resolve the
1744
+ // backing file from the planned-ref pool so the downstream existsSync
1745
+ // guard keeps the ref (a replay candidate from asset_salience whose file
1746
+ // is gone correctly drops out). Only refs present in the indexed pool can
1747
+ // be revisited — refs without a planned entry get no filePath and are
1748
+ // dropped by the disk check, which is the desired behavior.
1749
+ const planned = plannedRefs.find((p) => p.ref === ref);
1750
+ newReplayRefs.push({
1751
+ ref,
1752
+ reason: "scope-type",
1753
+ eligibilitySource: "replay",
1754
+ ...(planned?.filePath ? { filePath: planned.filePath } : {}),
1755
+ });
1756
+ // Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
1757
+ if (!salienceMap.has(ref)) {
1758
+ salienceMap.set(ref, {
1759
+ encoding: 0,
1760
+ outcome: 0,
1761
+ retrieval: 0,
1762
+ rankScore: allRankScores.get(ref) ?? 0,
1763
+ });
1764
+ }
1765
+ }
1766
+ if (newReplayRefs.length > 0) {
1767
+ mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
1768
+ // Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
1769
+ // keyed by a stronger lane.
1770
+ for (const ref of replayRefSet) {
1771
+ if (!eligibilitySourceByRef.has(ref))
1772
+ eligibilitySourceByRef.set(ref, "replay");
1773
+ }
1774
+ for (const r of mergedRefs) {
1775
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
1776
+ }
1777
+ }
1778
+ // Aggregated observability event (never per-ref).
1779
+ appendEvent({
1780
+ eventType: "improve_replay_selected",
1781
+ ref: undefined,
1782
+ metadata: {
1783
+ count: newReplayRefs.length,
1784
+ budget: replayBudget,
1785
+ convergedSkipped,
1786
+ candidatePool,
1787
+ },
1788
+ }, eventsCtx);
1789
+ }, { path: eventsCtx?.dbPath });
1790
+ }
1791
+ catch (err) {
1792
+ rethrowIfTestIsolationError(err);
1793
+ // best-effort: if DB unavailable, replayRefSet stays empty
1794
+ }
1795
+ }
1796
+ // Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
1797
+ // Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
1798
+ // this function. The effective score is used ONLY for processing/selection
1799
+ // order — the persisted rank_score in asset_salience is never mutated here.
1800
+ const noOpMap = new Map();
1801
+ try {
1802
+ const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
1803
+ if (noOpDb) {
1804
+ const ownsNoOpDb = !eventsCtx?.db;
1805
+ try {
1806
+ for (const r of mergedRefs) {
1807
+ noOpMap.set(r.ref, getConsecutiveNoOps(noOpDb, r.ref));
1808
+ }
1809
+ }
1810
+ finally {
1811
+ if (ownsNoOpDb)
1812
+ noOpDb.close();
1813
+ }
1814
+ }
1815
+ }
1816
+ catch {
1817
+ // best-effort: dampener failure never blocks selection
1818
+ }
1819
+ // Sort by effective selection score (desc), with explicit ref-string tie-break
1820
+ // for determinism. The effective score applies the consolidation-selection
1821
+ // dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
1822
+ // THRESHOLD) are penalised by FACTOR so they sort after peers with similar
1823
+ // rankScore. The persisted rank_score is left unchanged — this is the whole
1824
+ // point of the dampener (stable assets stay fully retrievable).
1825
+ //
1826
+ // WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
1827
+ // "consolidation-selection" in the plan refers to THIS reflect/distill
1828
+ // eligibility ordering — i.e. which assets are chosen for the reflect/distill
1829
+ // LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
1830
+ // which runs earlier and never reads noOpMap). The no-op counter originates
1831
+ // from no-change reflect / quality-rejected distill outcomes; the dampener
1832
+ // suppresses repeated LLM attempts on those same assets without touching their
1833
+ // persisted rank_score (so they remain fully retrievable).
1834
+ //
1835
+ // This is the ONLY ranking path — negativeOnlyRatio and the legacy
1836
+ // symmetricValence branch are replaced. The three eligibilitySource lanes
1837
+ // (signal-delta / high-retrieval / proactive) survive as labels (set above).
1838
+ const effectiveScore = (ref) => {
1839
+ const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
1840
+ const noOps = noOpMap.get(ref) ?? 0;
1841
+ return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
1842
+ };
1843
+ const sorted = [...mergedRefs].sort((a, b) => {
1844
+ const scoreA = effectiveScore(a.ref);
1845
+ const scoreB = effectiveScore(b.ref);
1846
+ if (scoreB !== scoreA)
1847
+ return scoreB - scoreA;
1848
+ // Stable tie-break: deterministic regardless of input ordering.
1849
+ return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
1850
+ });
1851
+ // Phase 0: surface coverage gaps from zero-result search queries
1852
+ let coverageGaps = [];
1853
+ try {
1854
+ const dbForGaps = openExistingDatabase();
1855
+ try {
1856
+ coverageGaps = getZeroResultSearches(dbForGaps);
1857
+ }
1858
+ finally {
1859
+ closeDatabase(dbForGaps);
1860
+ }
1861
+ }
1862
+ catch (err) {
1863
+ rethrowIfTestIsolationError(err);
1864
+ // best-effort
1865
+ }
1866
+ // actionableRefs is the post-cooldown, post-validation, post-signal, post-sort
1867
+ // set — i.e. the genuinely processable refs in priority order. Note: this is
1868
+ // a semantic shift from earlier code where actionableRefs was the pre-cooldown
1869
+ // sorted set; the new meaning matches reality and is documented on
1870
+ // ImprovePreparationResult.actionableRefs.
1871
+ //
1872
+ // Final guard: drop any candidate whose backing file is no longer on disk.
1873
+ // Phase 1 validation captures missing files at the start of preparation, but
1874
+ // the gap between that check and dispatch can be minutes on large stashes —
1875
+ // long enough for a checkpoint / git checkout / external cleanup to delete
1876
+ // the asset. Empirically (improve-critical-review 2026-05-20) the single
1877
+ // biggest reject category was "Asset no longer exists on disk" (604/1407 =
1878
+ // 43%), meaning reflect/distill was producing proposals against deleted refs.
1879
+ // A cheap existsSync per surviving candidate eliminates that wasted work.
1880
+ const assetMissingOnDisk = [];
1881
+ const existsCheckedActionable = [];
1882
+ for (const candidate of sorted) {
1883
+ // #591: prefer the path pre-resolved at planning time (synchronous
1884
+ // existsSync) over a serial async DB lookup per ref.
1885
+ const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
1886
+ ? candidate.filePath
1887
+ : await findAssetFilePath(candidate.ref, options.stashDir);
1888
+ if (filePath && fs.existsSync(filePath)) {
1889
+ existsCheckedActionable.push(candidate);
1890
+ }
1891
+ else {
1892
+ assetMissingOnDisk.push(candidate.ref);
1893
+ }
1894
+ }
1895
+ // #592 audit: one summary event instead of one per missing ref. Normally
1896
+ // tiny, but a stash deletion racing the run could make this O(n) sequential
1897
+ // state.db writes. `refs` is capped so the metadata row stays bounded.
1898
+ if (assetMissingOnDisk.length > 0) {
1899
+ appendEvent({
1900
+ eventType: "improve_skipped",
1901
+ ref: undefined,
1902
+ metadata: {
1903
+ reason: "asset_missing_on_disk",
1904
+ count: assetMissingOnDisk.length,
1905
+ refs: assetMissingOnDisk.slice(0, 50),
1906
+ },
1907
+ }, eventsCtx);
1908
+ }
1909
+ const actionableRefs = existsCheckedActionable;
1910
+ // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
1911
+ // preserving sort order. distillOnlyRefs participate in the sort so --limit
1912
+ // picks them by score, not by arbitrary position.
1913
+ const distillOnlyRefSetForSort = new Set(distillOnlyRefs.map((r) => r.ref));
1914
+ const reflectAndDistillRefsAfterSort = [];
1915
+ const distillOnlyRefsAfterSort = [];
1916
+ for (const r of actionableRefs) {
1917
+ if (distillOnlyRefSetForSort.has(r.ref)) {
1918
+ distillOnlyRefsAfterSort.push(r);
1919
+ }
1920
+ else {
1921
+ reflectAndDistillRefsAfterSort.push(r);
1922
+ }
1923
+ }
1924
+ // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
1925
+ //
1926
+ // #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
1927
+ // fresh slice. Without this split, a high-rankScore replay ref could sort above
1928
+ // a fresh ref in the single combined slice and STEAL its slot (violating AC2).
1929
+ // We partition into the replay lane vs the rest, apply --limit to the
1930
+ // non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
1931
+ // after the fresh slice. Sort order within each partition is preserved.
1932
+ //
1933
+ // Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
1934
+ // replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
1935
+ // `replayLoop.slice(0, 0) === []` — byte-identical.
1936
+ const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
1937
+ const replayLoop = allLoopRefs.filter((r) => r.eligibilitySource === "replay");
1938
+ const nonReplayLoop = allLoopRefs.filter((r) => r.eligibilitySource !== "replay");
1939
+ const baseLoop = options.limit ? nonReplayLoop.slice(0, options.limit) : nonReplayLoop;
1940
+ const loopRefs = [...baseLoop, ...replayLoop.slice(0, replayBudget)];
1941
+ // Update the returned distillOnlyRefs to the sorted order so callers see the
1942
+ // ranked view (loop stage uses it as a Set so order is irrelevant, but the
1943
+ // shape change keeps downstream consumers consistent).
1944
+ const distillOnlyRefsResult = distillOnlyRefsAfterSort;
1945
+ const totalReflectBlocked = fullySkippedCount + distillOnlyRefs.length;
1946
+ if (totalReflectBlocked > 0) {
1947
+ info(`[improve] ${totalReflectBlocked} of ${preCooldownCount} indexed refs blocked by reflect signal-delta ` +
1948
+ `(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
1949
+ }
1950
+ if (signalAndRetrievalRefs.length > 0) {
1951
+ info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
1952
+ }
1953
+ if (validationFailureRefs.size > 0) {
1954
+ info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
1955
+ }
1956
+ if (assetMissingOnDisk.length > 0) {
1957
+ info(`[improve] ${assetMissingOnDisk.length} candidates dropped — file not on disk`);
1958
+ }
1959
+ const deferredCount = actionableRefs.length - loopRefs.length;
1960
+ info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
1961
+ (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
1962
+ // WS-4: Per-phase threshold auto-tune for the extract phase.
1963
+ // Persists result for the NEXT run's makeGateConfig to read.
1964
+ const extractTuneDbPath = eventsCtx?.dbPath;
1965
+ if (options.autoAccept !== undefined && extractTuneDbPath) {
1966
+ try {
1967
+ maybeAutoTuneThreshold(extractPass.extractGateCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), extractTuneDbPath, undefined, "extract");
1968
+ }
1969
+ catch (err) {
1970
+ warn(`[improve] calibration auto-tune (extract) skipped: ${err instanceof Error ? err.message : String(err)}`);
1971
+ }
1972
+ }
1973
+ return {
1974
+ actions,
1975
+ cleanupWarnings,
1976
+ appliedCleanup,
1977
+ memoryIndexHealth,
1978
+ extract: extractResults,
1979
+ actionableRefs,
1980
+ signalBearingSet,
1981
+ validationFailures,
1982
+ schemaRepairs,
1983
+ lintSummary,
1984
+ loopRefs,
1985
+ distillCooledRefs,
1986
+ distillOnlyRefs: distillOnlyRefsResult,
1987
+ coverageGaps,
1988
+ recentErrors,
1989
+ utilityMap,
1990
+ gateAutoAcceptedCount,
1991
+ gateAutoAcceptFailedCount,
1992
+ consolidation: consolidationPass.consolidation,
1993
+ consolidationRan: consolidationPass.consolidationRan,
1994
+ ...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
1995
+ };
1996
+ }
1997
+ // TODO(refactor): 13 args including `actions`/`recentErrors` mutation channels. Restructure into immutable plan + mutable context objects — deferred to dedicated refactor with isolated testing.
1998
+ /**
1999
+ * Parameter object for {@link runImproveLoopStage} (WS10). Pure type reshape of
2000
+ * the former inline arg struct — every field, name, and type is preserved so the
2001
+ * function body and all runtime values are byte-identical. No control-flow change.
2002
+ */