akm-cli 0.9.0-beta.9 → 0.9.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (381) hide show
  1. package/CHANGELOG.md +715 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/help/help-improve.md +9 -6
  6. package/dist/assets/hints/cli-hints-full.md +6 -5
  7. package/dist/assets/profiles/default.json +9 -4
  8. package/dist/assets/profiles/frequent.json +1 -1
  9. package/dist/assets/profiles/memory-focus.json +1 -1
  10. package/dist/assets/profiles/proactive-maintenance.json +25 -0
  11. package/dist/assets/profiles/quick.json +1 -1
  12. package/dist/assets/profiles/recombine-only.json +21 -0
  13. package/dist/assets/profiles/reflect-distill.json +30 -0
  14. package/dist/assets/profiles/synthesize.json +15 -0
  15. package/dist/assets/profiles/thorough.json +1 -1
  16. package/dist/assets/prompts/consolidate-system.md +23 -0
  17. package/dist/assets/prompts/contradiction-judge.md +33 -0
  18. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  19. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  20. package/dist/assets/prompts/extract-session.md +11 -3
  21. package/dist/assets/prompts/graph-extract-system.md +1 -0
  22. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  23. package/dist/assets/prompts/memory-infer-system.md +1 -0
  24. package/dist/assets/prompts/memory-infer-user.md +5 -0
  25. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  26. package/dist/assets/prompts/procedural-system.md +44 -0
  27. package/dist/assets/prompts/recombine-system.md +40 -0
  28. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  29. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  30. package/dist/assets/prompts/workflow-unit-preamble.md +26 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  34. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  35. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  36. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  37. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  38. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  39. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  40. package/dist/assets/templates/html/health.html +281 -111
  41. package/dist/assets/wiki/ingest-workflow-template.md +45 -16
  42. package/dist/assets/wiki/schema-template.md +4 -4
  43. package/dist/cli/clack.js +56 -0
  44. package/dist/cli/config-migrate.js +7 -1
  45. package/dist/cli/confirm.js +1 -1
  46. package/dist/cli/parse-args.js +46 -1
  47. package/dist/cli/shared.js +28 -0
  48. package/dist/cli.js +25 -21
  49. package/dist/commands/agent/agent-dispatch.js +3 -2
  50. package/dist/commands/agent/agent-support.js +0 -7
  51. package/dist/commands/agent/contribute-cli.js +26 -7
  52. package/dist/commands/config-cli.js +26 -13
  53. package/dist/commands/env/child-env.js +47 -0
  54. package/dist/commands/env/env-binding.js +95 -0
  55. package/dist/commands/env/env-cli.js +228 -292
  56. package/dist/commands/env/env.js +14 -67
  57. package/dist/commands/env/secret-cli.js +140 -138
  58. package/dist/commands/feedback-cli.js +156 -155
  59. package/dist/commands/graph/graph-cli.js +5 -13
  60. package/dist/commands/graph/graph.js +3 -3
  61. package/dist/commands/health/advisories.js +151 -0
  62. package/dist/commands/health/checks.js +103 -16
  63. package/dist/commands/health/html-report.js +447 -81
  64. package/dist/commands/health/improve-metrics.js +771 -0
  65. package/dist/commands/health/llm-usage.js +65 -0
  66. package/dist/commands/health/md-report.js +103 -0
  67. package/dist/commands/health/metrics.js +278 -0
  68. package/dist/commands/health/stash-exposure.js +46 -0
  69. package/dist/commands/health/surfaces.js +216 -0
  70. package/dist/commands/health/task-runs.js +135 -0
  71. package/dist/commands/health/types.js +26 -0
  72. package/dist/commands/health/windows.js +195 -0
  73. package/dist/commands/health.js +91 -1091
  74. package/dist/commands/improve/anti-collapse.js +170 -0
  75. package/dist/commands/improve/calibration.js +161 -0
  76. package/dist/commands/improve/collapse-detector.js +421 -0
  77. package/dist/commands/improve/consolidate/chunking.js +141 -0
  78. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  79. package/dist/commands/improve/consolidate/merge.js +145 -0
  80. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  81. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  82. package/dist/commands/improve/consolidate.js +1295 -1277
  83. package/dist/commands/improve/dedup.js +482 -0
  84. package/dist/commands/improve/distill/content-repair.js +202 -0
  85. package/dist/commands/improve/distill/promote-memory.js +229 -0
  86. package/dist/commands/improve/distill/quality-gate.js +236 -0
  87. package/dist/commands/improve/distill-guards.js +127 -0
  88. package/dist/commands/improve/distill-promotion-policy.js +826 -167
  89. package/dist/commands/improve/distill.js +228 -605
  90. package/dist/commands/improve/eligibility.js +434 -0
  91. package/dist/commands/improve/encoding-salience.js +205 -0
  92. package/dist/commands/improve/extract-cli.js +179 -59
  93. package/dist/commands/improve/extract-prompt.js +54 -3
  94. package/dist/commands/improve/extract-watch.js +140 -0
  95. package/dist/commands/improve/extract.js +409 -43
  96. package/dist/commands/improve/feedback-valence.js +54 -0
  97. package/dist/commands/improve/hot-probation.js +45 -0
  98. package/dist/commands/improve/improve-auto-accept.js +157 -10
  99. package/dist/commands/improve/improve-cli.js +115 -73
  100. package/dist/commands/improve/improve-profiles.js +28 -8
  101. package/dist/commands/improve/improve-result-file.js +15 -25
  102. package/dist/commands/improve/improve-session.js +58 -0
  103. package/dist/commands/improve/improve.js +485 -2764
  104. package/dist/commands/improve/locks.js +154 -0
  105. package/dist/commands/improve/loop-stages.js +1100 -0
  106. package/dist/commands/improve/memory/memory-belief.js +14 -15
  107. package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
  108. package/dist/commands/improve/memory/memory-improve.js +27 -27
  109. package/dist/commands/improve/outcome-loop.js +270 -0
  110. package/dist/commands/improve/preparation.js +2002 -0
  111. package/dist/commands/improve/proactive-maintenance.js +37 -35
  112. package/dist/commands/improve/procedural.js +398 -0
  113. package/dist/commands/improve/recombine.js +818 -0
  114. package/dist/commands/improve/reflect-noise.js +0 -0
  115. package/dist/commands/improve/reflect.js +206 -45
  116. package/dist/commands/improve/salience.js +455 -0
  117. package/dist/commands/improve/schema-similarity-gate.js +168 -0
  118. package/dist/commands/improve/shared.js +51 -0
  119. package/dist/commands/improve/triage.js +93 -0
  120. package/dist/commands/lint/agent-linter.js +19 -24
  121. package/dist/commands/lint/base-linter.js +173 -60
  122. package/dist/commands/lint/command-linter.js +19 -24
  123. package/dist/commands/lint/env-key-rules.js +38 -1
  124. package/dist/commands/lint/fact-linter.js +39 -0
  125. package/dist/commands/lint/index.js +31 -13
  126. package/dist/commands/lint/memory-linter.js +1 -1
  127. package/dist/commands/lint/registry.js +7 -2
  128. package/dist/commands/lint/task-linter.js +3 -3
  129. package/dist/commands/lint/workflow-linter.js +26 -1
  130. package/dist/commands/observability-cli.js +4 -4
  131. package/dist/commands/proposal/drain-policies.js +13 -4
  132. package/dist/commands/proposal/drain.js +45 -51
  133. package/dist/commands/proposal/legacy-import.js +115 -0
  134. package/dist/commands/proposal/proposal-cli.js +24 -34
  135. package/dist/commands/proposal/proposal.js +2 -1
  136. package/dist/commands/proposal/propose.js +8 -3
  137. package/dist/commands/proposal/repository.js +829 -0
  138. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  139. package/dist/commands/proposal/validators/proposals.js +93 -895
  140. package/dist/commands/read/curate.js +410 -111
  141. package/dist/commands/read/knowledge.js +10 -3
  142. package/dist/commands/read/remember-cli.js +133 -138
  143. package/dist/commands/read/search-cli.js +15 -8
  144. package/dist/commands/read/search.js +22 -11
  145. package/dist/commands/read/show.js +106 -14
  146. package/dist/commands/registry-cli.js +76 -87
  147. package/dist/commands/remember.js +11 -12
  148. package/dist/commands/sources/add-cli.js +91 -95
  149. package/dist/commands/sources/history.js +1 -1
  150. package/dist/commands/sources/init.js +66 -18
  151. package/dist/commands/sources/installed-stashes.js +11 -3
  152. package/dist/commands/sources/migration-help.js +7 -4
  153. package/dist/commands/sources/schema-repair.js +44 -46
  154. package/dist/commands/sources/self-update.js +2 -2
  155. package/dist/commands/sources/source-add.js +7 -3
  156. package/dist/commands/sources/sources-cli.js +3 -3
  157. package/dist/commands/sources/stash-cli.js +19 -39
  158. package/dist/commands/sources/stash-skeleton.js +57 -8
  159. package/dist/commands/tasks/default-tasks.js +15 -2
  160. package/dist/commands/tasks/tasks-cli.js +20 -29
  161. package/dist/commands/tasks/tasks.js +39 -11
  162. package/dist/commands/wiki-cli.js +23 -38
  163. package/dist/commands/workflow-cli.js +291 -13
  164. package/dist/core/asset/asset-registry.js +3 -1
  165. package/dist/core/asset/asset-spec.js +79 -5
  166. package/dist/core/asset/frontmatter.js +188 -167
  167. package/dist/core/asset/markdown.js +8 -0
  168. package/dist/core/authoring-rules.js +92 -0
  169. package/dist/core/common.js +4 -23
  170. package/dist/core/concurrent.js +10 -1
  171. package/dist/core/config/config-io.js +10 -1
  172. package/dist/core/config/config-migration.js +18 -40
  173. package/dist/core/config/config-schema.js +403 -62
  174. package/dist/core/config/config-types.js +3 -3
  175. package/dist/core/config/config.js +67 -22
  176. package/dist/core/deep-merge.js +38 -0
  177. package/dist/core/errors.js +1 -0
  178. package/dist/core/eval/rank-metrics.js +113 -0
  179. package/dist/core/events.js +4 -7
  180. package/dist/core/improve-types.js +47 -8
  181. package/dist/core/json-schema.js +142 -0
  182. package/dist/core/logs-db.js +14 -75
  183. package/dist/core/parse.js +36 -16
  184. package/dist/core/paths.js +18 -18
  185. package/dist/core/standards/resolve-standards-context.js +87 -0
  186. package/dist/core/standards/resolve-stash-standards.js +99 -0
  187. package/dist/core/standards/resolve-type-conventions.js +66 -0
  188. package/dist/core/state/migrations.js +770 -0
  189. package/dist/core/state-db.js +132 -1126
  190. package/dist/core/structured.js +69 -0
  191. package/dist/core/time.js +53 -0
  192. package/dist/core/warn.js +21 -0
  193. package/dist/core/write-source.js +37 -0
  194. package/dist/indexer/db/db.js +261 -770
  195. package/dist/indexer/db/entry-mapper.js +41 -0
  196. package/dist/indexer/db/graph-db.js +129 -86
  197. package/dist/indexer/db/llm-cache.js +2 -2
  198. package/dist/indexer/db/schema.js +516 -0
  199. package/dist/indexer/ensure-index.js +36 -92
  200. package/dist/indexer/feedback/utility-policy.js +75 -0
  201. package/dist/indexer/graph/graph-boost.js +51 -41
  202. package/dist/indexer/graph/graph-extraction.js +207 -4
  203. package/dist/indexer/index-writer-lock.js +18 -11
  204. package/dist/indexer/index-written-assets.js +105 -0
  205. package/dist/indexer/indexer.js +182 -204
  206. package/dist/indexer/passes/dir-staleness.js +114 -0
  207. package/dist/indexer/passes/memory-inference.js +13 -5
  208. package/dist/indexer/passes/metadata.js +20 -0
  209. package/dist/indexer/read-preflight.js +23 -0
  210. package/dist/indexer/search/db-search.js +89 -13
  211. package/dist/indexer/search/fts-query.js +51 -0
  212. package/dist/indexer/search/ranking-contributors.js +95 -9
  213. package/dist/indexer/search/ranking.js +79 -3
  214. package/dist/indexer/search/search-fields.js +6 -0
  215. package/dist/indexer/search/search-source.js +32 -21
  216. package/dist/indexer/search/semantic-status.js +4 -0
  217. package/dist/indexer/walk/matchers.js +48 -0
  218. package/dist/indexer/walk/walker.js +21 -13
  219. package/dist/integrations/agent/builders.js +41 -13
  220. package/dist/integrations/agent/config.js +20 -59
  221. package/dist/integrations/agent/detect.js +9 -0
  222. package/dist/integrations/agent/index.js +3 -19
  223. package/dist/integrations/agent/model-aliases.js +16 -2
  224. package/dist/integrations/agent/profiles.js +79 -6
  225. package/dist/integrations/agent/prompts.js +75 -9
  226. package/dist/integrations/agent/runner-dispatch.js +83 -0
  227. package/dist/integrations/agent/runner.js +13 -9
  228. package/dist/integrations/agent/spawn.js +206 -81
  229. package/dist/integrations/harnesses/aider/agent-builder.js +113 -0
  230. package/dist/integrations/harnesses/aider/index.js +58 -0
  231. package/dist/integrations/harnesses/aider/result-extractor.js +53 -0
  232. package/dist/integrations/harnesses/amazonq/agent-builder.js +153 -0
  233. package/dist/integrations/harnesses/amazonq/index.js +59 -0
  234. package/dist/integrations/harnesses/amazonq/result-extractor.js +48 -0
  235. package/dist/integrations/harnesses/claude/agent-builder.js +46 -7
  236. package/dist/integrations/harnesses/claude/index.js +27 -23
  237. package/dist/integrations/harnesses/claude/result-extractor.js +52 -0
  238. package/dist/integrations/harnesses/claude/session-log.js +10 -0
  239. package/dist/integrations/harnesses/codex/agent-builder.js +137 -0
  240. package/dist/integrations/harnesses/codex/index.js +63 -0
  241. package/dist/integrations/harnesses/codex/result-extractor.js +73 -0
  242. package/dist/integrations/harnesses/copilot/agent-builder.js +122 -0
  243. package/dist/integrations/harnesses/copilot/index.js +60 -0
  244. package/dist/integrations/harnesses/copilot/result-extractor.js +151 -0
  245. package/dist/integrations/harnesses/gemini/agent-builder.js +121 -0
  246. package/dist/integrations/harnesses/gemini/index.js +60 -0
  247. package/dist/integrations/harnesses/gemini/result-extractor.js +121 -0
  248. package/dist/integrations/harnesses/index.js +28 -7
  249. package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
  250. package/dist/integrations/harnesses/opencode/index.js +17 -16
  251. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  252. package/dist/integrations/harnesses/opencode-sdk/harness.js +65 -0
  253. package/dist/integrations/harnesses/opencode-sdk/index.js +10 -34
  254. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +642 -71
  255. package/dist/integrations/harnesses/openhands/agent-builder.js +126 -0
  256. package/dist/integrations/harnesses/openhands/index.js +58 -0
  257. package/dist/integrations/harnesses/openhands/result-extractor.js +103 -0
  258. package/dist/integrations/harnesses/pi/agent-builder.js +104 -0
  259. package/dist/integrations/harnesses/pi/index.js +58 -0
  260. package/dist/integrations/harnesses/pi/result-extractor.js +135 -0
  261. package/dist/integrations/harnesses/types.js +8 -0
  262. package/dist/integrations/session-logs/index.js +40 -11
  263. package/dist/llm/call-ai.js +2 -2
  264. package/dist/llm/client.js +34 -11
  265. package/dist/llm/embedder.js +67 -4
  266. package/dist/llm/embedders/cache.js +3 -1
  267. package/dist/llm/embedders/deterministic.js +66 -0
  268. package/dist/llm/embedders/local.js +73 -3
  269. package/dist/llm/feature-gate.js +16 -15
  270. package/dist/llm/graph-extract.js +67 -44
  271. package/dist/llm/memory-infer-impl.js +138 -0
  272. package/dist/llm/memory-infer.js +1 -127
  273. package/dist/llm/metadata-enhance.js +44 -31
  274. package/dist/llm/structured-call.js +49 -0
  275. package/dist/migrate-storage-node.mjs +8 -0
  276. package/dist/output/context.js +5 -5
  277. package/dist/output/renderers.js +87 -15
  278. package/dist/output/shapes/curate.js +14 -2
  279. package/dist/output/shapes/helpers.js +0 -3
  280. package/dist/output/shapes/passthrough.js +6 -1
  281. package/dist/output/text/helpers.js +241 -2
  282. package/dist/output/text/workflow.js +4 -1
  283. package/dist/registry/providers/skills-sh.js +21 -147
  284. package/dist/registry/providers/static-index.js +15 -157
  285. package/dist/registry/resolve.js +27 -9
  286. package/dist/runtime.js +25 -1
  287. package/dist/schemas/akm-config.json +14225 -0
  288. package/dist/schemas/akm-workflow.json +328 -0
  289. package/dist/scripts/migrate-storage.js +2743 -8390
  290. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +1652 -607
  291. package/dist/setup/detect.js +9 -0
  292. package/dist/setup/legacy-config.js +106 -0
  293. package/dist/setup/prompt.js +57 -0
  294. package/dist/setup/providers.js +14 -0
  295. package/dist/setup/registry-stash-loader.js +12 -0
  296. package/dist/setup/semantic-assets.js +124 -0
  297. package/dist/setup/setup.js +52 -1614
  298. package/dist/setup/steps/connection.js +734 -0
  299. package/dist/setup/steps/output.js +31 -0
  300. package/dist/setup/steps/platforms.js +124 -0
  301. package/dist/setup/steps/semantic.js +27 -0
  302. package/dist/setup/steps/sources.js +222 -0
  303. package/dist/setup/steps/stashdir.js +42 -0
  304. package/dist/setup/steps/tasks.js +152 -0
  305. package/dist/sources/include.js +6 -2
  306. package/dist/sources/providers/filesystem.js +0 -1
  307. package/dist/sources/providers/git-install.js +210 -0
  308. package/dist/sources/providers/git-provider.js +234 -0
  309. package/dist/sources/providers/git-stash.js +248 -0
  310. package/dist/sources/providers/git.js +10 -661
  311. package/dist/sources/providers/npm.js +2 -6
  312. package/dist/sources/providers/provider-utils.js +13 -7
  313. package/dist/sources/providers/sync-from-ref.js +9 -1
  314. package/dist/sources/providers/website.js +9 -5
  315. package/dist/sources/website-ingest.js +187 -29
  316. package/dist/sources/wiki-fetchers/registry.js +53 -0
  317. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  318. package/dist/storage/database.js +45 -10
  319. package/dist/storage/managed-db.js +82 -0
  320. package/dist/storage/repositories/canaries-repository.js +107 -0
  321. package/dist/storage/repositories/consolidation-repository.js +38 -0
  322. package/dist/storage/repositories/embeddings-repository.js +72 -0
  323. package/dist/storage/repositories/events-repository.js +187 -0
  324. package/dist/storage/repositories/extract-sessions-repository.js +96 -0
  325. package/dist/storage/repositories/improve-runs-repository.js +146 -0
  326. package/dist/storage/repositories/index-db.js +14 -8
  327. package/dist/storage/repositories/proposals-repository.js +220 -0
  328. package/dist/storage/repositories/recombine-repository.js +213 -0
  329. package/dist/storage/repositories/registry-cache.js +93 -0
  330. package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
  331. package/dist/storage/repositories/task-history-repository.js +93 -0
  332. package/dist/storage/repositories/workflow-runs-repository.js +189 -1
  333. package/dist/storage/sqlite-pragmas.js +146 -0
  334. package/dist/tasks/backends/cron.js +1 -1
  335. package/dist/tasks/backends/index.js +9 -0
  336. package/dist/tasks/backends/launchd.js +1 -1
  337. package/dist/tasks/backends/schtasks.js +1 -1
  338. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  339. package/dist/tasks/runner.js +15 -13
  340. package/dist/text-import-hook.mjs +1 -1
  341. package/dist/wiki/wiki.js +52 -11
  342. package/dist/workflows/authoring/authoring.js +123 -10
  343. package/dist/workflows/authoring/workflow-program-template.yaml +31 -0
  344. package/dist/workflows/cli.js +5 -0
  345. package/dist/workflows/db.js +138 -4
  346. package/dist/workflows/exec/brief.js +484 -0
  347. package/dist/workflows/exec/native-executor.js +975 -0
  348. package/dist/workflows/exec/param-secrets.js +115 -0
  349. package/dist/workflows/exec/report.js +1295 -0
  350. package/dist/workflows/exec/run-workflow.js +596 -0
  351. package/dist/workflows/exec/scheduler.js +100 -0
  352. package/dist/workflows/exec/step-work.js +1156 -0
  353. package/dist/workflows/exec/unit-writer.js +23 -0
  354. package/dist/workflows/exec/watch.js +116 -0
  355. package/dist/workflows/exec/worktree.js +171 -0
  356. package/dist/workflows/ir/compile.js +388 -0
  357. package/dist/workflows/ir/params.js +54 -0
  358. package/dist/workflows/ir/plan-hash.js +33 -0
  359. package/dist/workflows/ir/schema.js +4 -0
  360. package/dist/workflows/parser.js +3 -1
  361. package/dist/workflows/program/expressions.js +369 -0
  362. package/dist/workflows/program/parser.js +760 -0
  363. package/dist/workflows/program/project.js +105 -0
  364. package/dist/workflows/program/schema.js +54 -0
  365. package/dist/workflows/renderer.js +82 -5
  366. package/dist/workflows/runtime/agent-identity.js +59 -14
  367. package/dist/workflows/runtime/runs.js +248 -153
  368. package/dist/workflows/runtime/unit-checkin.js +45 -0
  369. package/dist/workflows/runtime/workflow-asset-loader.js +188 -0
  370. package/dist/workflows/validate-summary.js +26 -10
  371. package/dist/workflows/validator.js +1 -1
  372. package/docs/README.md +69 -18
  373. package/docs/data-and-telemetry.md +7 -5
  374. package/docs/migration/release-notes/0.7.0.md +1 -1
  375. package/docs/migration/release-notes/0.9.0-beta.60.md +19 -0
  376. package/docs/migration/release-notes/0.9.0.md +39 -0
  377. package/package.json +10 -10
  378. package/dist/assets/tasks/core/update-stashes.yml +0 -4
  379. package/dist/commands/db-cli.js +0 -23
  380. package/dist/indexer/db/db-backup.js +0 -376
  381. package/dist/indexer/passes/staleness-detect.js +0 -488
@@ -0,0 +1,771 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { queryImproveRuns } from "../../storage/repositories/improve-runs-repository.js";
5
+ import { summarizeCalibration } from "../improve/calibration.js";
6
+ export function roundRate(value) {
7
+ return Number(value.toFixed(4));
8
+ }
9
+ export function parseTaskMetadata(row) {
10
+ try {
11
+ return JSON.parse(row.metadata_json);
12
+ }
13
+ catch {
14
+ return {};
15
+ }
16
+ }
17
+ function createUnknownImproveMetrics() {
18
+ return {
19
+ invoked: 0,
20
+ completed: 0,
21
+ skipped: 0,
22
+ skipReasons: {},
23
+ plannedRefs: 0,
24
+ profileFilteredRefs: 0,
25
+ actions: {
26
+ reflect: { ok: 0, failed: 0, cooldown: 0, skipped: 0, guardRejected: 0, skippedByReason: {} },
27
+ distill: {
28
+ queued: 0,
29
+ llmFailed: 0,
30
+ qualityRejected: 0,
31
+ judgeRejected: 0,
32
+ validatorRejected: 0,
33
+ configDisabled: 0,
34
+ skipped: 0,
35
+ skippedByReason: {},
36
+ deferred: 0,
37
+ deferredByReason: {},
38
+ },
39
+ memoryPrune: 0,
40
+ memoryInference: 0,
41
+ graphExtraction: 0,
42
+ error: 0,
43
+ },
44
+ autoAccept: { promoted: 0, validationFailed: 0 },
45
+ calibration: summarizeCalibration([]),
46
+ reflectsWithErrorContext: 0,
47
+ coverageGapCount: 0,
48
+ evalCasesWritten: 0,
49
+ deadUrlCount: 0,
50
+ memorySummary: { eligible: 0, derived: 0 },
51
+ memoryCleanup: {
52
+ pruneCandidates: 0,
53
+ contradictionCandidates: 0,
54
+ beliefStateTransitions: 0,
55
+ consolidationCandidates: 0,
56
+ archived: 0,
57
+ warnings: 0,
58
+ },
59
+ consolidation: {
60
+ ran: false,
61
+ processed: 0,
62
+ promoted: 0,
63
+ merged: 0,
64
+ deleted: 0,
65
+ contradicted: 0,
66
+ judgedNoAction: 0,
67
+ mergedSecondaries: 0,
68
+ failedChunkMemories: 0,
69
+ skipReasons: {},
70
+ failedChunks: 0,
71
+ totalChunks: 0,
72
+ durationMs: 0,
73
+ },
74
+ memoryInference: {
75
+ ran: false,
76
+ considered: 0,
77
+ cacheHits: 0,
78
+ retryAttempts: 0,
79
+ freshAttempts: 0,
80
+ splitParents: 0,
81
+ written: 0,
82
+ skippedNoFacts: 0,
83
+ skippedChildExists: 0,
84
+ skippedAborted: 0,
85
+ unaccounted: 0,
86
+ htmlErrorCount: 0,
87
+ yieldEligibleRuns: 0,
88
+ yieldEligibleConsidered: 0,
89
+ yieldEligibleWritten: 0,
90
+ yieldRate: 0,
91
+ durationMs: 0,
92
+ writes: 0,
93
+ },
94
+ graphExtraction: {
95
+ ran: false,
96
+ extractedFiles: 0,
97
+ entities: 0,
98
+ relations: 0,
99
+ cacheHits: 0,
100
+ cacheMisses: 0,
101
+ cacheHitRate: 0,
102
+ truncations: 0,
103
+ failures: 0,
104
+ htmlErrors: 0,
105
+ retryAttempts: 0,
106
+ nonArrayBatchFailures: 0,
107
+ durationMs: 0,
108
+ },
109
+ sessionExtraction: {
110
+ ran: false,
111
+ sessionsScanned: 0,
112
+ sessionsExtracted: 0,
113
+ sessionsSkipped: 0,
114
+ proposalsCreated: 0,
115
+ warnings: 0,
116
+ durationMs: 0,
117
+ },
118
+ wallTime: {
119
+ count: 0,
120
+ medianMs: 0,
121
+ p95Ms: 0,
122
+ minMs: 0,
123
+ maxMs: 0,
124
+ byPhase: {
125
+ consolidation: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
126
+ memoryInference: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
127
+ graphExtraction: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
128
+ },
129
+ },
130
+ perfTelemetry: {
131
+ dedupPoolSize: 0,
132
+ llmPoolSize: 0,
133
+ judgedCacheSkipped: 0,
134
+ embedMs: 0,
135
+ embedCacheHits: 0,
136
+ embedCacheMisses: 0,
137
+ overBudgetRuns: 0,
138
+ runsWithTelemetry: 0,
139
+ },
140
+ coverage: {
141
+ rate: Number.NaN,
142
+ eligibleFraction: Number.NaN,
143
+ acceptedProposals: 0,
144
+ distinctRefs: 0,
145
+ churnRatio: Number.NaN,
146
+ totalAssets: 0,
147
+ },
148
+ };
149
+ }
150
+ export function toFiniteNumber(value) {
151
+ if (typeof value === "number" && Number.isFinite(value))
152
+ return value;
153
+ if (typeof value === "string" && value.trim()) {
154
+ const parsed = Number(value);
155
+ if (Number.isFinite(parsed))
156
+ return parsed;
157
+ }
158
+ return 0;
159
+ }
160
+ /**
161
+ * Event-derived metrics. Only `completed` and skipReasons/invoked are sourced
162
+ * from events in v2 — the richer fields come from {@link summarizeImproveRuns}.
163
+ * The function still receives `improve_completed` events so that the completed
164
+ * count reflects the canonical event stream (it lines up 1:1 with improve_runs
165
+ * rows in practice, but the events table remains the system-of-record for the
166
+ * existence of a run).
167
+ */
168
+ export function summarizeImproveCompleted(events) {
169
+ const metrics = createUnknownImproveMetrics();
170
+ metrics.completed = events.length;
171
+ return metrics;
172
+ }
173
+ /**
174
+ * Project a single `improve_runs.result_json` envelope into an accumulator-shaped
175
+ * ImproveHealthMetrics. The aggregator merges these per-row metrics into one
176
+ * window-level metric.
177
+ */
178
+ function projectRunMetrics(result) {
179
+ const metrics = createUnknownImproveMetrics();
180
+ // plannedRefs (array of {ref, reason})
181
+ const plannedRefs = result.plannedRefs;
182
+ if (Array.isArray(plannedRefs))
183
+ metrics.plannedRefs += plannedRefs.length;
184
+ // profileFilteredRefs (array of {ref, reason}) — 2026-05-27: pre-filter
185
+ // bucket from `collectEligibleRefs` so the metric reflects work the
186
+ // planner dropped before signal-delta / per-pass dispatch.
187
+ const profileFilteredRefs = result.profileFilteredRefs;
188
+ if (Array.isArray(profileFilteredRefs))
189
+ metrics.profileFilteredRefs += profileFilteredRefs.length;
190
+ // actions: split reflect / distill by outcome, count others.
191
+ const actions = result.actions;
192
+ if (Array.isArray(actions)) {
193
+ for (const action of actions) {
194
+ const mode = typeof action.mode === "string" ? action.mode : "";
195
+ switch (mode) {
196
+ case "reflect":
197
+ metrics.actions.reflect.ok += 1;
198
+ break;
199
+ case "reflect-failed":
200
+ metrics.actions.reflect.failed += 1;
201
+ break;
202
+ case "reflect-cooldown":
203
+ metrics.actions.reflect.cooldown += 1;
204
+ break;
205
+ case "reflect-skipped": {
206
+ metrics.actions.reflect.skipped += 1;
207
+ const r = action.result;
208
+ const reason = typeof r?.reason === "string" && r.reason.trim() ? r.reason : "unknown";
209
+ metrics.actions.reflect.skippedByReason[reason] = (metrics.actions.reflect.skippedByReason[reason] ?? 0) + 1;
210
+ break;
211
+ }
212
+ case "reflect-guard-rejected":
213
+ metrics.actions.reflect.guardRejected += 1;
214
+ break;
215
+ case "distill": {
216
+ const r = action.result;
217
+ const outcome = typeof r?.outcome === "string" ? r.outcome : "";
218
+ switch (outcome) {
219
+ case "queued":
220
+ metrics.actions.distill.queued += 1;
221
+ break;
222
+ case "llm_failed":
223
+ metrics.actions.distill.llmFailed += 1;
224
+ break;
225
+ case "quality_rejected":
226
+ case "review_needed":
227
+ metrics.actions.distill.qualityRejected += 1;
228
+ metrics.actions.distill.judgeRejected += 1;
229
+ break;
230
+ case "validation_failed":
231
+ metrics.actions.distill.qualityRejected += 1;
232
+ metrics.actions.distill.validatorRejected += 1;
233
+ break;
234
+ case "config_disabled":
235
+ metrics.actions.distill.configDisabled += 1;
236
+ break;
237
+ case "skipped": {
238
+ // Previously dropped on the floor. The four sub-paths that emit
239
+ // `outcome: "skipped"` (see distill.ts:893, 1024, 1120, 1576):
240
+ // - recursive_lesson_input (type guard refused a lesson input)
241
+ // - conflict_noop (LLM resolved destination conflict as NOOP)
242
+ // - proposal-skipped cooldown / dedup at persistence
243
+ // 465 events/7d in the user's live stack. The result message
244
+ // typically encodes the reason; we also accept an explicit
245
+ // `skipReason` field when downstream code sets it.
246
+ metrics.actions.distill.deferred += 1;
247
+ const explicitReason = typeof r?.skipReason === "string" ? r.skipReason : undefined;
248
+ const msg = typeof r?.message === "string" ? r.message : "";
249
+ let reason = explicitReason ?? "unknown";
250
+ if (!explicitReason) {
251
+ if (/lesson inputs/i.test(msg))
252
+ reason = "recursive_lesson_input";
253
+ else if (/NOOP/.test(msg))
254
+ reason = "conflict_noop";
255
+ else if (/cooldown/i.test(msg))
256
+ reason = "proposal_cooldown";
257
+ else if (/content[_ ]?hash/i.test(msg))
258
+ reason = "content_hash_match";
259
+ }
260
+ metrics.actions.distill.deferredByReason[reason] =
261
+ (metrics.actions.distill.deferredByReason[reason] ?? 0) + 1;
262
+ break;
263
+ }
264
+ default:
265
+ break;
266
+ }
267
+ break;
268
+ }
269
+ case "distill-skipped": {
270
+ metrics.actions.distill.skipped += 1;
271
+ const r = action.result;
272
+ const reason = typeof r?.reason === "string" && r.reason.trim() ? r.reason : "unknown";
273
+ metrics.actions.distill.skippedByReason[reason] = (metrics.actions.distill.skippedByReason[reason] ?? 0) + 1;
274
+ break;
275
+ }
276
+ case "memory-prune":
277
+ metrics.actions.memoryPrune += 1;
278
+ break;
279
+ case "memory-inference":
280
+ metrics.actions.memoryInference += 1;
281
+ break;
282
+ case "graph-extraction":
283
+ metrics.actions.graphExtraction += 1;
284
+ break;
285
+ case "error":
286
+ metrics.actions.error += 1;
287
+ break;
288
+ }
289
+ }
290
+ }
291
+ // C1 (13-bus-factor): new runs persist the bounded `distillSkipped` aggregate
292
+ // instead of per-ref `distill-skipped` rows. Read the total + per-reason
293
+ // histogram from it into the SAME `distill.skipped` / `skippedByReason`
294
+ // metric the per-ref loop above populated. Legacy rows carry the per-ref rows
295
+ // and NO aggregate (counted above); new rows carry the aggregate and NO
296
+ // per-ref rows (counted here) — so a run is never double-counted.
297
+ const distillSkipped = result.distillSkipped;
298
+ if (distillSkipped && typeof distillSkipped === "object") {
299
+ metrics.actions.distill.skipped += toFiniteNumber(distillSkipped.total);
300
+ const byReason = distillSkipped.byReason;
301
+ if (byReason && typeof byReason === "object") {
302
+ for (const [reason, count] of Object.entries(byReason)) {
303
+ metrics.actions.distill.skippedByReason[reason] =
304
+ (metrics.actions.distill.skippedByReason[reason] ?? 0) + toFiniteNumber(count);
305
+ }
306
+ }
307
+ }
308
+ metrics.autoAccept.promoted += toFiniteNumber(result.gateAutoAcceptedCount);
309
+ metrics.autoAccept.validationFailed += toFiniteNumber(result.gateAutoAcceptFailedCount);
310
+ metrics.reflectsWithErrorContext += toFiniteNumber(result.reflectsWithErrorContext);
311
+ if (Array.isArray(result.coverageGaps))
312
+ metrics.coverageGapCount += result.coverageGaps.length;
313
+ metrics.evalCasesWritten += toFiniteNumber(result.evalCasesWritten);
314
+ if (Array.isArray(result.deadUrls))
315
+ metrics.deadUrlCount += result.deadUrls.length;
316
+ const memorySummary = result.memorySummary;
317
+ if (memorySummary) {
318
+ metrics.memorySummary.eligible += toFiniteNumber(memorySummary.eligible);
319
+ metrics.memorySummary.derived += toFiniteNumber(memorySummary.derived);
320
+ }
321
+ const memoryCleanup = result.memoryCleanup;
322
+ if (memoryCleanup) {
323
+ if (Array.isArray(memoryCleanup.pruneCandidates))
324
+ metrics.memoryCleanup.pruneCandidates += memoryCleanup.pruneCandidates.length;
325
+ if (Array.isArray(memoryCleanup.contradictionCandidates))
326
+ metrics.memoryCleanup.contradictionCandidates += memoryCleanup.contradictionCandidates.length;
327
+ if (Array.isArray(memoryCleanup.beliefStateTransitions))
328
+ metrics.memoryCleanup.beliefStateTransitions += memoryCleanup.beliefStateTransitions.length;
329
+ if (Array.isArray(memoryCleanup.consolidationCandidates))
330
+ metrics.memoryCleanup.consolidationCandidates += memoryCleanup.consolidationCandidates.length;
331
+ if (Array.isArray(memoryCleanup.archived))
332
+ metrics.memoryCleanup.archived += memoryCleanup.archived.length;
333
+ if (Array.isArray(memoryCleanup.warnings))
334
+ metrics.memoryCleanup.warnings += memoryCleanup.warnings.length;
335
+ }
336
+ const consolidation = result.consolidation;
337
+ if (consolidation) {
338
+ metrics.consolidation.processed += toFiniteNumber(consolidation.processed);
339
+ metrics.consolidation.merged += toFiniteNumber(consolidation.merged);
340
+ metrics.consolidation.deleted += toFiniteNumber(consolidation.deleted);
341
+ metrics.consolidation.contradicted += toFiniteNumber(consolidation.contradicted);
342
+ if (Array.isArray(consolidation.promoted))
343
+ metrics.consolidation.promoted += consolidation.promoted.length;
344
+ metrics.consolidation.failedChunks += toFiniteNumber(consolidation.failedChunks);
345
+ metrics.consolidation.totalChunks += toFiniteNumber(consolidation.totalChunks);
346
+ metrics.consolidation.durationMs += toFiniteNumber(consolidation.durationMs);
347
+ metrics.consolidation.judgedNoAction += toFiniteNumber(consolidation.judgedNoAction);
348
+ metrics.consolidation.mergedSecondaries += toFiniteNumber(consolidation.mergedSecondaries);
349
+ metrics.consolidation.failedChunkMemories += toFiniteNumber(consolidation.failedChunkMemories);
350
+ // Structured emitter (new on this branch): consolidate.ts now pushes
351
+ // per-ref grouped `{ref, skips: [{op, reason}]}` entries to `skipReasons`
352
+ // for every deterministic post-LLM rejection. Each ref appears once but
353
+ // may carry multiple skips; aggregate every reason. Pre-fix envelopes have
354
+ // neither field, so be defensive.
355
+ const skipReasons = consolidation.skipReasons;
356
+ if (Array.isArray(skipReasons)) {
357
+ for (const entry of skipReasons) {
358
+ if (!entry || typeof entry !== "object")
359
+ continue;
360
+ const skips = entry.skips;
361
+ if (!Array.isArray(skips))
362
+ continue;
363
+ for (const skip of skips) {
364
+ if (!skip || typeof skip !== "object")
365
+ continue;
366
+ const reason = skip.reason;
367
+ if (typeof reason !== "string" || !reason.trim())
368
+ continue;
369
+ metrics.consolidation.skipReasons[reason] = (metrics.consolidation.skipReasons[reason] ?? 0) + 1;
370
+ }
371
+ }
372
+ }
373
+ // WS-5: extract perf telemetry from the consolidation envelope.
374
+ // Pre-WS-5 envelopes lack `perfTelemetry`; be defensive.
375
+ const perf = consolidation.perfTelemetry;
376
+ if (perf) {
377
+ metrics.perfTelemetry.runsWithTelemetry += 1;
378
+ metrics.perfTelemetry.dedupPoolSize += toFiniteNumber(perf.dedupPoolSize);
379
+ metrics.perfTelemetry.llmPoolSize += toFiniteNumber(perf.llmPoolSize);
380
+ metrics.perfTelemetry.judgedCacheSkipped += toFiniteNumber(perf.judgedCacheSkipped);
381
+ metrics.perfTelemetry.embedMs += toFiniteNumber(perf.embedMs);
382
+ metrics.perfTelemetry.embedCacheHits += toFiniteNumber(perf.embedCacheHits);
383
+ metrics.perfTelemetry.embedCacheMisses += toFiniteNumber(perf.embedCacheMisses);
384
+ const budgetFrac = toFiniteNumber(perf.estimatedBudgetFractionUsed);
385
+ if (budgetFrac > 1.0)
386
+ metrics.perfTelemetry.overBudgetRuns += 1;
387
+ }
388
+ }
389
+ const memoryInference = result.memoryInference;
390
+ if (memoryInference) {
391
+ const considered = toFiniteNumber(memoryInference.considered);
392
+ const writtenFacts = toFiniteNumber(memoryInference.writtenFacts);
393
+ metrics.memoryInference.considered += considered;
394
+ metrics.memoryInference.cacheHits += toFiniteNumber(memoryInference.cacheHits);
395
+ metrics.memoryInference.retryAttempts += toFiniteNumber(memoryInference.retryAttempts);
396
+ metrics.memoryInference.splitParents += toFiniteNumber(memoryInference.splitParents);
397
+ metrics.memoryInference.written += writtenFacts;
398
+ metrics.memoryInference.skippedNoFacts += toFiniteNumber(memoryInference.skippedNoFacts);
399
+ metrics.memoryInference.skippedChildExists += toFiniteNumber(memoryInference.skippedChildExists);
400
+ metrics.memoryInference.skippedAborted += toFiniteNumber(memoryInference.skippedAborted);
401
+ metrics.memoryInference.unaccounted += toFiniteNumber(memoryInference.unaccounted);
402
+ metrics.memoryInference.htmlErrorCount += toFiniteNumber(memoryInference.htmlErrorCount);
403
+ // Yield-rate gating: pre-cache-feature envelopes lack the `cacheHits`
404
+ // field entirely. Treating their `considered` as freshAttempts (since
405
+ // cacheHits=0) is mathematically tempting but operationally wrong —
406
+ // historical runs with the legacy schema have no cache instrumentation
407
+ // and the SUM dragged the reported rate to ~14% in local data. Only
408
+ // contribute to the yield aggregate when the envelope actually carries
409
+ // the field. See investigation 2026-05-26.
410
+ if (Object.hasOwn(memoryInference, "cacheHits")) {
411
+ metrics.memoryInference.yieldEligibleRuns += 1;
412
+ metrics.memoryInference.yieldEligibleConsidered += considered;
413
+ metrics.memoryInference.yieldEligibleWritten += writtenFacts;
414
+ }
415
+ }
416
+ metrics.memoryInference.durationMs += toFiniteNumber(result.memoryInferenceDurationMs);
417
+ const graphExtraction = result.graphExtraction;
418
+ if (graphExtraction) {
419
+ const quality = graphExtraction.quality;
420
+ if (quality)
421
+ metrics.graphExtraction.extractedFiles += toFiniteNumber(quality.extractedFiles);
422
+ metrics.graphExtraction.entities += toFiniteNumber(graphExtraction.totalEntities);
423
+ metrics.graphExtraction.relations += toFiniteNumber(graphExtraction.totalRelations);
424
+ const telemetry = graphExtraction.telemetry;
425
+ if (telemetry) {
426
+ metrics.graphExtraction.cacheHits += toFiniteNumber(telemetry.cacheHits);
427
+ metrics.graphExtraction.cacheMisses += toFiniteNumber(telemetry.cacheMisses);
428
+ metrics.graphExtraction.truncations += toFiniteNumber(telemetry.truncationCount);
429
+ metrics.graphExtraction.failures += toFiniteNumber(telemetry.failureCount);
430
+ metrics.graphExtraction.htmlErrors += toFiniteNumber(telemetry.htmlErrorCount);
431
+ metrics.graphExtraction.retryAttempts += toFiniteNumber(telemetry.retryAttempts);
432
+ metrics.graphExtraction.nonArrayBatchFailures += toFiniteNumber(telemetry.nonArrayBatchFailures);
433
+ }
434
+ }
435
+ metrics.graphExtraction.durationMs += toFiniteNumber(result.graphExtractionDurationMs);
436
+ if (Array.isArray(result.extract)) {
437
+ for (const e of result.extract) {
438
+ metrics.sessionExtraction.sessionsScanned += toFiniteNumber(e.sessionsProcessed);
439
+ metrics.sessionExtraction.sessionsSkipped += toFiniteNumber(e.sessionsSkipped);
440
+ if (Array.isArray(e.sessions)) {
441
+ metrics.sessionExtraction.sessionsExtracted += e.sessions.filter((s) => Array.isArray(s.proposalIds) && s.proposalIds.length > 0).length;
442
+ }
443
+ metrics.sessionExtraction.proposalsCreated += Array.isArray(e.proposals) ? e.proposals.length : 0;
444
+ metrics.sessionExtraction.warnings += Array.isArray(e.warnings) ? e.warnings.length : 0;
445
+ metrics.sessionExtraction.durationMs += toFiniteNumber(e.durationMs);
446
+ }
447
+ }
448
+ return metrics;
449
+ }
450
+ /**
451
+ * Finalize derived flags and rates on an accumulator. Used both for the
452
+ * window-level aggregate and for each per-run row in --detail per-run mode
453
+ * so the single-row metrics still expose `ran` / `yieldRate` / `cacheHitRate`.
454
+ */
455
+ function finalizeImproveMetrics(metrics) {
456
+ metrics.consolidation.ran =
457
+ metrics.consolidation.processed > 0 ||
458
+ metrics.consolidation.durationMs > 0 ||
459
+ metrics.consolidation.promoted > 0 ||
460
+ metrics.consolidation.merged > 0 ||
461
+ metrics.consolidation.deleted > 0 ||
462
+ metrics.consolidation.contradicted > 0 ||
463
+ metrics.consolidation.totalChunks > 0;
464
+ metrics.memoryInference.ran =
465
+ metrics.memoryInference.considered > 0 ||
466
+ metrics.memoryInference.written > 0 ||
467
+ metrics.memoryInference.durationMs > 0;
468
+ metrics.memoryInference.writes = metrics.memoryInference.written;
469
+ // Yield denominator excludes cache hits AND legacy (pre-cacheHits-field)
470
+ // envelopes. Only runs whose envelope carries a `cacheHits` field
471
+ // contribute to freshAttempts/yieldRate; legacy rows remain in
472
+ // `considered`/`written` for totals but are excluded from the rate so
473
+ // they cannot drag it down. See ImproveHealthMetrics.memoryInference
474
+ // jsdoc for the rationale.
475
+ metrics.memoryInference.freshAttempts = Math.max(0, metrics.memoryInference.yieldEligibleConsidered -
476
+ metrics.memoryInference.cacheHits -
477
+ metrics.memoryInference.skippedAborted);
478
+ metrics.memoryInference.yieldRate =
479
+ metrics.memoryInference.freshAttempts > 0
480
+ ? roundRate(metrics.memoryInference.yieldEligibleWritten / metrics.memoryInference.freshAttempts)
481
+ : 0;
482
+ metrics.graphExtraction.ran =
483
+ metrics.graphExtraction.extractedFiles > 0 ||
484
+ metrics.graphExtraction.entities > 0 ||
485
+ metrics.graphExtraction.durationMs > 0;
486
+ const cacheTotal = metrics.graphExtraction.cacheHits + metrics.graphExtraction.cacheMisses;
487
+ metrics.graphExtraction.cacheHitRate = cacheTotal > 0 ? roundRate(metrics.graphExtraction.cacheHits / cacheTotal) : 0;
488
+ metrics.sessionExtraction.ran =
489
+ metrics.sessionExtraction.sessionsScanned > 0 ||
490
+ metrics.sessionExtraction.proposalsCreated > 0 ||
491
+ metrics.sessionExtraction.durationMs > 0;
492
+ }
493
+ /**
494
+ * Merge per-row metrics from `src` into accumulator `dst`. All numeric fields
495
+ * are additive; cumulative rates are recomputed by finalizeImproveMetrics.
496
+ */
497
+ function mergeImproveMetrics(dst, src) {
498
+ dst.plannedRefs += src.plannedRefs;
499
+ // profileFilteredRefs is the count of refs the planner drops up-front for the
500
+ // active profile — recomputed against the (stable) stash every run, so it is a
501
+ // snapshot, NOT a per-run increment. Summing it re-counts the same refs each
502
+ // run (the ~2.4M bug). Set from the most recent run in summarizeImproveRuns.
503
+ dst.actions.reflect.ok += src.actions.reflect.ok;
504
+ dst.actions.reflect.failed += src.actions.reflect.failed;
505
+ dst.actions.reflect.cooldown += src.actions.reflect.cooldown;
506
+ dst.actions.reflect.skipped += src.actions.reflect.skipped;
507
+ dst.actions.reflect.guardRejected += src.actions.reflect.guardRejected;
508
+ for (const [reason, count] of Object.entries(src.actions.reflect.skippedByReason)) {
509
+ dst.actions.reflect.skippedByReason[reason] = (dst.actions.reflect.skippedByReason[reason] ?? 0) + count;
510
+ }
511
+ dst.actions.distill.queued += src.actions.distill.queued;
512
+ dst.actions.distill.llmFailed += src.actions.distill.llmFailed;
513
+ dst.actions.distill.qualityRejected += src.actions.distill.qualityRejected;
514
+ dst.actions.distill.judgeRejected += src.actions.distill.judgeRejected;
515
+ dst.actions.distill.validatorRejected += src.actions.distill.validatorRejected;
516
+ dst.actions.distill.configDisabled += src.actions.distill.configDisabled;
517
+ dst.actions.distill.skipped += src.actions.distill.skipped;
518
+ for (const [reason, count] of Object.entries(src.actions.distill.skippedByReason)) {
519
+ dst.actions.distill.skippedByReason[reason] = (dst.actions.distill.skippedByReason[reason] ?? 0) + count;
520
+ }
521
+ dst.actions.distill.deferred += src.actions.distill.deferred;
522
+ for (const [reason, count] of Object.entries(src.actions.distill.deferredByReason)) {
523
+ dst.actions.distill.deferredByReason[reason] = (dst.actions.distill.deferredByReason[reason] ?? 0) + count;
524
+ }
525
+ dst.actions.memoryPrune += src.actions.memoryPrune;
526
+ dst.actions.memoryInference += src.actions.memoryInference;
527
+ dst.actions.graphExtraction += src.actions.graphExtraction;
528
+ dst.actions.error += src.actions.error;
529
+ dst.autoAccept.promoted += src.autoAccept.promoted;
530
+ dst.autoAccept.validationFailed += src.autoAccept.validationFailed;
531
+ dst.reflectsWithErrorContext += src.reflectsWithErrorContext;
532
+ dst.coverageGapCount += src.coverageGapCount;
533
+ dst.evalCasesWritten += src.evalCasesWritten;
534
+ dst.deadUrlCount += src.deadUrlCount;
535
+ // NOTE: memorySummary (derived/eligible) is a WHOLE-STASH snapshot recorded on
536
+ // every run, NOT a per-run increment — summing it across the window inflates
537
+ // it ~N× (the 1.2M-eligible bug). It is set from the most recent run in
538
+ // summarizeImproveRuns instead, so it is intentionally not merged here.
539
+ dst.memoryCleanup.pruneCandidates += src.memoryCleanup.pruneCandidates;
540
+ dst.memoryCleanup.contradictionCandidates += src.memoryCleanup.contradictionCandidates;
541
+ dst.memoryCleanup.beliefStateTransitions += src.memoryCleanup.beliefStateTransitions;
542
+ dst.memoryCleanup.consolidationCandidates += src.memoryCleanup.consolidationCandidates;
543
+ dst.memoryCleanup.archived += src.memoryCleanup.archived;
544
+ dst.memoryCleanup.warnings += src.memoryCleanup.warnings;
545
+ dst.consolidation.processed += src.consolidation.processed;
546
+ dst.consolidation.promoted += src.consolidation.promoted;
547
+ dst.consolidation.merged += src.consolidation.merged;
548
+ dst.consolidation.deleted += src.consolidation.deleted;
549
+ dst.consolidation.contradicted += src.consolidation.contradicted;
550
+ dst.consolidation.failedChunks += src.consolidation.failedChunks;
551
+ dst.consolidation.totalChunks += src.consolidation.totalChunks;
552
+ dst.consolidation.durationMs += src.consolidation.durationMs;
553
+ dst.consolidation.judgedNoAction += src.consolidation.judgedNoAction;
554
+ dst.consolidation.mergedSecondaries += src.consolidation.mergedSecondaries;
555
+ dst.consolidation.failedChunkMemories += src.consolidation.failedChunkMemories;
556
+ for (const [reason, count] of Object.entries(src.consolidation.skipReasons)) {
557
+ dst.consolidation.skipReasons[reason] = (dst.consolidation.skipReasons[reason] ?? 0) + count;
558
+ }
559
+ dst.memoryInference.considered += src.memoryInference.considered;
560
+ dst.memoryInference.cacheHits += src.memoryInference.cacheHits;
561
+ dst.memoryInference.splitParents += src.memoryInference.splitParents;
562
+ dst.memoryInference.written += src.memoryInference.written;
563
+ dst.memoryInference.skippedNoFacts += src.memoryInference.skippedNoFacts;
564
+ dst.memoryInference.skippedChildExists += src.memoryInference.skippedChildExists;
565
+ dst.memoryInference.skippedAborted += src.memoryInference.skippedAborted;
566
+ dst.memoryInference.unaccounted += src.memoryInference.unaccounted;
567
+ dst.memoryInference.htmlErrorCount += src.memoryInference.htmlErrorCount;
568
+ dst.memoryInference.yieldEligibleRuns += src.memoryInference.yieldEligibleRuns;
569
+ dst.memoryInference.yieldEligibleConsidered += src.memoryInference.yieldEligibleConsidered;
570
+ dst.memoryInference.yieldEligibleWritten += src.memoryInference.yieldEligibleWritten;
571
+ dst.memoryInference.durationMs += src.memoryInference.durationMs;
572
+ dst.graphExtraction.extractedFiles += src.graphExtraction.extractedFiles;
573
+ dst.graphExtraction.entities += src.graphExtraction.entities;
574
+ dst.graphExtraction.relations += src.graphExtraction.relations;
575
+ dst.graphExtraction.cacheHits += src.graphExtraction.cacheHits;
576
+ dst.graphExtraction.cacheMisses += src.graphExtraction.cacheMisses;
577
+ dst.graphExtraction.truncations += src.graphExtraction.truncations;
578
+ dst.graphExtraction.failures += src.graphExtraction.failures;
579
+ dst.graphExtraction.htmlErrors += src.graphExtraction.htmlErrors;
580
+ dst.graphExtraction.nonArrayBatchFailures += src.graphExtraction.nonArrayBatchFailures;
581
+ dst.graphExtraction.durationMs += src.graphExtraction.durationMs;
582
+ dst.sessionExtraction.sessionsScanned += src.sessionExtraction.sessionsScanned;
583
+ dst.sessionExtraction.sessionsExtracted += src.sessionExtraction.sessionsExtracted;
584
+ dst.sessionExtraction.sessionsSkipped += src.sessionExtraction.sessionsSkipped;
585
+ dst.sessionExtraction.proposalsCreated += src.sessionExtraction.proposalsCreated;
586
+ dst.sessionExtraction.warnings += src.sessionExtraction.warnings;
587
+ dst.sessionExtraction.durationMs += src.sessionExtraction.durationMs;
588
+ // WS-5: merge perf telemetry (additive sums).
589
+ dst.perfTelemetry.dedupPoolSize += src.perfTelemetry.dedupPoolSize;
590
+ dst.perfTelemetry.llmPoolSize += src.perfTelemetry.llmPoolSize;
591
+ dst.perfTelemetry.judgedCacheSkipped += src.perfTelemetry.judgedCacheSkipped;
592
+ dst.perfTelemetry.embedMs += src.perfTelemetry.embedMs;
593
+ dst.perfTelemetry.embedCacheHits += src.perfTelemetry.embedCacheHits;
594
+ dst.perfTelemetry.embedCacheMisses += src.perfTelemetry.embedCacheMisses;
595
+ dst.perfTelemetry.overBudgetRuns += src.perfTelemetry.overBudgetRuns;
596
+ dst.perfTelemetry.runsWithTelemetry += src.perfTelemetry.runsWithTelemetry;
597
+ // coverage: acceptedProposals is additive; totalAssets is a snapshot (like memorySummary).
598
+ // totalAssets is intentionally NOT merged here — set from the most recent run in summarizeImproveRuns.
599
+ dst.coverage.acceptedProposals += src.coverage.acceptedProposals;
600
+ }
601
+ export function summarizeImproveRuns(db, since, until) {
602
+ const accum = createUnknownImproveMetrics();
603
+ const rows = queryImproveRuns(db, since, until);
604
+ // Per-phase wall-time samples. Each entry is one envelope's durationMs for
605
+ // that phase. Phases that did not run on a given envelope are simply
606
+ // omitted (NOT counted as 0) so the median/p95 reflect actual phase work.
607
+ const phaseDurations = {
608
+ consolidation: [],
609
+ memoryInference: [],
610
+ graphExtraction: [],
611
+ };
612
+ // memorySummary is a whole-stash snapshot per run, so the window value is the
613
+ // MOST RECENT run's snapshot (current state) — not a sum across runs.
614
+ let latestStartMs = Number.NEGATIVE_INFINITY;
615
+ let latestMemorySummary;
616
+ let latestProfileFilteredRefs = 0;
617
+ for (const row of rows) {
618
+ let result;
619
+ try {
620
+ result = JSON.parse(row.result_json);
621
+ }
622
+ catch {
623
+ continue;
624
+ }
625
+ const perRow = projectRunMetrics(result);
626
+ mergeImproveMetrics(accum, perRow);
627
+ const startMs = new Date(row.started_at).getTime();
628
+ if (Number.isFinite(startMs) && startMs >= latestStartMs) {
629
+ latestStartMs = startMs;
630
+ latestMemorySummary = perRow.memorySummary;
631
+ latestProfileFilteredRefs = perRow.profileFilteredRefs;
632
+ }
633
+ // Collect per-phase durations directly off the envelope. consolidation's
634
+ // duration lives inside the sub-object; memoryInference and graphExtraction
635
+ // expose top-level *DurationMs keys (`memoryInferenceDurationMs`,
636
+ // `graphExtractionDurationMs`) when they actually ran on that envelope.
637
+ const consol = result.consolidation;
638
+ const consolMs = toFiniteNumber(consol?.durationMs);
639
+ if (consolMs > 0)
640
+ phaseDurations.consolidation.push(consolMs);
641
+ const memMs = toFiniteNumber(result.memoryInferenceDurationMs);
642
+ if (memMs > 0)
643
+ phaseDurations.memoryInference.push(memMs);
644
+ const graphMs = toFiniteNumber(result.graphExtractionDurationMs);
645
+ if (graphMs > 0)
646
+ phaseDurations.graphExtraction.push(graphMs);
647
+ }
648
+ finalizeImproveMetrics(accum);
649
+ if (latestMemorySummary)
650
+ accum.memorySummary = latestMemorySummary;
651
+ accum.profileFilteredRefs = latestProfileFilteredRefs;
652
+ accum.wallTime.byPhase = {
653
+ consolidation: summarizePhaseDurations(phaseDurations.consolidation),
654
+ memoryInference: summarizePhaseDurations(phaseDurations.memoryInference),
655
+ graphExtraction: summarizePhaseDurations(phaseDurations.graphExtraction),
656
+ };
657
+ return { metrics: accum, runCount: rows.length };
658
+ }
659
+ /**
660
+ * Aggregate a list of per-envelope phase durations into the
661
+ * `wallTime.byPhase.*` shape: count, total, median, p95. Median/p95 use the
662
+ * same nearest-rank picker as the top-level wallTime stats so the two are
663
+ * comparable.
664
+ */
665
+ export function summarizePhaseDurations(samples) {
666
+ if (samples.length === 0)
667
+ return { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 };
668
+ const sorted = [...samples].sort((a, b) => a - b);
669
+ const pick = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? 0;
670
+ const totalMs = sorted.reduce((acc, n) => acc + n, 0);
671
+ return {
672
+ count: sorted.length,
673
+ totalMs,
674
+ medianMs: pick(0.5),
675
+ p95Ms: pick(0.95),
676
+ };
677
+ }
678
+ /**
679
+ * Project an improve_runs row + wall-time lookup into a single ImproveRunSummary.
680
+ * Used by `akm health --detail per-run`.
681
+ */
682
+ export function projectImproveRunSummary(row, wallTimeMs, taskId) {
683
+ let result = {};
684
+ try {
685
+ result = JSON.parse(row.result_json);
686
+ }
687
+ catch {
688
+ // fall through with empty result so per-stage rollups are zeros
689
+ }
690
+ const perRow = projectRunMetrics(result);
691
+ finalizeImproveMetrics(perRow);
692
+ const orphansPurged = toFiniteNumber(result.orphansPurged);
693
+ const lintSummary = result.lintSummary;
694
+ const lintFixed = lintSummary ? toFiniteNumber(lintSummary.fixed) : 0;
695
+ const lintFlagged = lintSummary ? toFiniteNumber(lintSummary.flagged) : 0;
696
+ return {
697
+ id: row.id,
698
+ startedAt: row.started_at,
699
+ completedAt: row.completed_at,
700
+ wallTimeMs,
701
+ ok: row.ok === 1,
702
+ scope: {
703
+ mode: row.scope_mode,
704
+ ...(row.scope_value ? { value: row.scope_value } : {}),
705
+ },
706
+ taskId,
707
+ actions: perRow.actions,
708
+ memorySummary: perRow.memorySummary,
709
+ memoryCleanup: perRow.memoryCleanup,
710
+ consolidation: perRow.consolidation,
711
+ memoryInference: perRow.memoryInference,
712
+ graphExtraction: perRow.graphExtraction,
713
+ reflectsWithErrorContext: perRow.reflectsWithErrorContext,
714
+ evalCasesWritten: perRow.evalCasesWritten,
715
+ orphansPurged,
716
+ lintFixed,
717
+ lintFlagged,
718
+ };
719
+ }
720
+ function emptyPhaseStats() {
721
+ return {
722
+ consolidation: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
723
+ memoryInference: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
724
+ graphExtraction: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
725
+ };
726
+ }
727
+ export function computeWallTimeStats(durationsMs, byPhase) {
728
+ const phase = byPhase ?? emptyPhaseStats();
729
+ if (durationsMs.length === 0)
730
+ return { count: 0, medianMs: 0, p95Ms: 0, minMs: 0, maxMs: 0, byPhase: phase };
731
+ const sorted = [...durationsMs].sort((a, b) => a - b);
732
+ const pick = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? 0;
733
+ return {
734
+ count: sorted.length,
735
+ medianMs: pick(0.5),
736
+ p95Ms: pick(0.95),
737
+ minMs: sorted[0] ?? 0,
738
+ maxMs: sorted[sorted.length - 1] ?? 0,
739
+ byPhase: phase,
740
+ };
741
+ }
742
+ export function buildImproveSkipSummary(events) {
743
+ // Two kinds of skip events:
744
+ // - Per-occurrence (no `count`): one event per skipped ref → SUM is correct.
745
+ // - Aggregated snapshot (carries `count`): a single per-run event whose count
746
+ // is the number of refs that hit a STABLE, whole-stash condition that run
747
+ // (`no_new_signal`, `profile_filtered_all_passes`). Each run re-counts the
748
+ // same stable set, so summing across the window re-counts it N times (the
749
+ // 2.7M / 3M inflation). For these we keep the MOST RECENT run's count — the
750
+ // current snapshot — matching how memorySummary/profileFilteredRefs are
751
+ // handled. Events arrive in chronological (offset) order, so the last
752
+ // count-bearing event per reason is the latest run's value.
753
+ const summed = {};
754
+ const latestSnapshot = {};
755
+ for (const event of events) {
756
+ const reason = typeof event.metadata?.reason === "string" && event.metadata.reason.trim() ? event.metadata.reason : "unknown";
757
+ const rawCount = event.metadata?.count;
758
+ if (typeof rawCount === "number" && Number.isFinite(rawCount) && rawCount > 0) {
759
+ latestSnapshot[reason] = rawCount; // overwrite → keeps the latest run's snapshot
760
+ }
761
+ else {
762
+ summed[reason] = (summed[reason] ?? 0) + 1;
763
+ }
764
+ }
765
+ const skipReasons = { ...summed };
766
+ for (const [reason, count] of Object.entries(latestSnapshot)) {
767
+ skipReasons[reason] = (skipReasons[reason] ?? 0) + count;
768
+ }
769
+ const skipped = Object.values(skipReasons).reduce((a, b) => a + b, 0);
770
+ return { skipped, skipReasons };
771
+ }