akm-cli 0.9.0-beta.5 → 0.9.0-beta.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/CHANGELOG.md +711 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/profiles/default.json +9 -4
  6. package/dist/assets/profiles/frequent.json +1 -1
  7. package/dist/assets/profiles/memory-focus.json +1 -1
  8. package/dist/assets/profiles/quick.json +1 -1
  9. package/dist/assets/profiles/synthesize.json +15 -0
  10. package/dist/assets/profiles/thorough.json +1 -1
  11. package/dist/assets/prompts/consolidate-system.md +23 -0
  12. package/dist/assets/prompts/contradiction-judge.md +33 -0
  13. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  14. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  15. package/dist/assets/prompts/extract-session.md +6 -2
  16. package/dist/assets/prompts/graph-extract-system.md +1 -0
  17. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  18. package/dist/assets/prompts/memory-infer-system.md +1 -0
  19. package/dist/assets/prompts/memory-infer-user.md +5 -0
  20. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  21. package/dist/assets/prompts/procedural-system.md +44 -0
  22. package/dist/assets/prompts/recombine-system.md +40 -0
  23. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  24. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  25. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  26. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  27. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  28. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  29. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  30. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  34. package/dist/assets/templates/html/health.html +281 -111
  35. package/dist/assets/wiki/ingest-workflow-template.md +38 -10
  36. package/dist/cli/parse-args.js +46 -1
  37. package/dist/cli/shared.js +28 -0
  38. package/dist/cli.js +27 -11
  39. package/dist/commands/agent/agent-dispatch.js +2 -2
  40. package/dist/commands/agent/agent-support.js +0 -7
  41. package/dist/commands/agent/contribute-cli.js +17 -4
  42. package/dist/commands/config-cli.js +18 -2
  43. package/dist/commands/env/child-env.js +47 -0
  44. package/dist/commands/env/env-cli.js +33 -26
  45. package/dist/commands/env/secret-cli.js +36 -22
  46. package/dist/commands/feedback-cli.js +15 -6
  47. package/dist/commands/graph/graph-cli.js +5 -13
  48. package/dist/commands/graph/graph.js +76 -72
  49. package/dist/commands/health/checks.js +49 -1
  50. package/dist/commands/health/html-report.js +422 -80
  51. package/dist/commands/health.js +386 -9
  52. package/dist/commands/improve/calibration.js +161 -0
  53. package/dist/commands/improve/consolidate/chunking.js +141 -0
  54. package/dist/commands/improve/consolidate/eligibility.js +81 -0
  55. package/dist/commands/improve/consolidate/merge.js +145 -0
  56. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  57. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  58. package/dist/commands/improve/consolidate.js +635 -660
  59. package/dist/commands/improve/dedup.js +482 -0
  60. package/dist/commands/improve/distill.js +159 -69
  61. package/dist/commands/improve/eligibility.js +434 -0
  62. package/dist/commands/improve/encoding-salience.js +205 -0
  63. package/dist/commands/improve/extract-cli.js +124 -2
  64. package/dist/commands/improve/extract-prompt.js +39 -2
  65. package/dist/commands/improve/extract-watch.js +140 -0
  66. package/dist/commands/improve/extract.js +389 -40
  67. package/dist/commands/improve/feedback-valence.js +54 -0
  68. package/dist/commands/improve/homeostatic.js +467 -0
  69. package/dist/commands/improve/improve-auto-accept.js +138 -7
  70. package/dist/commands/improve/improve-cli.js +36 -61
  71. package/dist/commands/improve/improve-profiles.js +14 -0
  72. package/dist/commands/improve/improve-result-file.js +14 -25
  73. package/dist/commands/improve/improve-session.js +58 -0
  74. package/dist/commands/improve/improve.js +485 -2498
  75. package/dist/commands/improve/locks.js +154 -0
  76. package/dist/commands/improve/loop-stages.js +1083 -0
  77. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  78. package/dist/commands/improve/outcome-loop.js +256 -0
  79. package/dist/commands/improve/preparation.js +1966 -0
  80. package/dist/commands/improve/proactive-maintenance.js +115 -0
  81. package/dist/commands/improve/procedural.js +418 -0
  82. package/dist/commands/improve/recombine.js +850 -0
  83. package/dist/commands/improve/reflect-noise.js +0 -0
  84. package/dist/commands/improve/reflect.js +183 -40
  85. package/dist/commands/improve/salience.js +438 -0
  86. package/dist/commands/improve/triage.js +93 -0
  87. package/dist/commands/lint/agent-linter.js +19 -24
  88. package/dist/commands/lint/base-linter.js +173 -60
  89. package/dist/commands/lint/command-linter.js +19 -24
  90. package/dist/commands/lint/env-key-rules.js +38 -1
  91. package/dist/commands/lint/fact-linter.js +39 -0
  92. package/dist/commands/lint/index.js +31 -13
  93. package/dist/commands/lint/memory-linter.js +1 -1
  94. package/dist/commands/lint/registry.js +7 -2
  95. package/dist/commands/lint/task-linter.js +3 -3
  96. package/dist/commands/lint/workflow-linter.js +26 -1
  97. package/dist/commands/proposal/drain-policies.js +5 -0
  98. package/dist/commands/proposal/drain.js +43 -50
  99. package/dist/commands/proposal/proposal-cli.js +21 -31
  100. package/dist/commands/proposal/proposal.js +5 -0
  101. package/dist/commands/proposal/propose.js +7 -2
  102. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  103. package/dist/commands/proposal/validators/proposals.js +189 -63
  104. package/dist/commands/read/curate.js +414 -94
  105. package/dist/commands/read/knowledge.js +6 -3
  106. package/dist/commands/read/search-cli.js +9 -4
  107. package/dist/commands/read/search.js +10 -6
  108. package/dist/commands/read/show.js +86 -7
  109. package/dist/commands/sources/init.js +49 -17
  110. package/dist/commands/sources/installed-stashes.js +11 -3
  111. package/dist/commands/sources/schema-repair.js +43 -45
  112. package/dist/commands/sources/self-update.js +2 -2
  113. package/dist/commands/sources/source-add.js +7 -3
  114. package/dist/commands/sources/stash-cli.js +28 -40
  115. package/dist/commands/sources/stash-skeleton.js +23 -8
  116. package/dist/commands/tasks/tasks-cli.js +19 -27
  117. package/dist/commands/tasks/tasks.js +39 -11
  118. package/dist/commands/wiki-cli.js +21 -35
  119. package/dist/core/asset/asset-registry.js +3 -1
  120. package/dist/core/asset/asset-spec.js +18 -2
  121. package/dist/core/asset/frontmatter.js +166 -167
  122. package/dist/core/asset/markdown.js +8 -0
  123. package/dist/core/authoring-rules.js +92 -0
  124. package/dist/core/common.js +0 -5
  125. package/dist/core/config/config-migration.js +12 -11
  126. package/dist/core/config/config-schema.js +340 -56
  127. package/dist/core/config/config-types.js +3 -3
  128. package/dist/core/config/config.js +28 -7
  129. package/dist/core/events.js +3 -7
  130. package/dist/core/improve-types.js +11 -8
  131. package/dist/core/logs-db.js +10 -66
  132. package/dist/core/parse.js +36 -16
  133. package/dist/core/paths.js +3 -0
  134. package/dist/core/standards/resolve-standards-context.js +87 -0
  135. package/dist/core/standards/resolve-stash-standards.js +99 -0
  136. package/dist/core/standards/resolve-type-conventions.js +66 -0
  137. package/dist/core/state/migrations.js +714 -0
  138. package/dist/core/state-db.js +525 -474
  139. package/dist/indexer/db/db.js +439 -247
  140. package/dist/indexer/db/graph-db.js +129 -86
  141. package/dist/indexer/ensure-index.js +152 -17
  142. package/dist/indexer/graph/graph-boost.js +51 -41
  143. package/dist/indexer/graph/graph-extraction.js +218 -4
  144. package/dist/indexer/index-writer-lock.js +99 -0
  145. package/dist/indexer/indexer.js +123 -221
  146. package/dist/indexer/passes/dir-staleness.js +114 -0
  147. package/dist/indexer/passes/memory-inference.js +13 -5
  148. package/dist/indexer/passes/staleness-detect.js +2 -5
  149. package/dist/indexer/search/db-search.js +19 -6
  150. package/dist/indexer/search/ranking-contributors.js +22 -0
  151. package/dist/indexer/search/ranking.js +4 -0
  152. package/dist/indexer/search/search-source.js +17 -18
  153. package/dist/indexer/search/semantic-status.js +4 -0
  154. package/dist/indexer/walk/matchers.js +9 -0
  155. package/dist/integrations/agent/config.js +6 -53
  156. package/dist/integrations/agent/index.js +2 -18
  157. package/dist/integrations/agent/prompts.js +75 -9
  158. package/dist/integrations/agent/runner-dispatch.js +59 -0
  159. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  160. package/dist/integrations/harnesses/index.js +2 -3
  161. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  162. package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
  163. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +0 -2
  164. package/dist/integrations/session-logs/index.js +16 -0
  165. package/dist/llm/client.js +45 -15
  166. package/dist/llm/embedder.js +42 -3
  167. package/dist/llm/embedders/deterministic.js +66 -0
  168. package/dist/llm/embedders/local.js +66 -2
  169. package/dist/llm/feature-gate.js +8 -4
  170. package/dist/llm/graph-extract.js +67 -44
  171. package/dist/llm/memory-infer-impl.js +138 -0
  172. package/dist/llm/memory-infer.js +1 -127
  173. package/dist/llm/metadata-enhance.js +44 -31
  174. package/dist/llm/structured-call.js +49 -0
  175. package/dist/migrate-storage-node.mjs +8 -0
  176. package/dist/output/context.js +5 -5
  177. package/dist/output/renderers.js +74 -2
  178. package/dist/output/shapes/curate.js +14 -2
  179. package/dist/output/shapes/passthrough.js +0 -1
  180. package/dist/output/text/helpers.js +16 -1
  181. package/dist/registry/providers/skills-sh.js +21 -147
  182. package/dist/registry/providers/static-index.js +15 -157
  183. package/dist/registry/resolve.js +22 -9
  184. package/dist/runtime.js +25 -1
  185. package/dist/scripts/migrate-storage.js +2617 -1961
  186. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +759 -510
  187. package/dist/setup/setup.js +29 -8
  188. package/dist/sources/include.js +6 -2
  189. package/dist/sources/providers/filesystem.js +0 -1
  190. package/dist/sources/providers/git-install.js +210 -0
  191. package/dist/sources/providers/git-provider.js +234 -0
  192. package/dist/sources/providers/git-stash.js +248 -0
  193. package/dist/sources/providers/git.js +10 -661
  194. package/dist/sources/providers/npm.js +2 -6
  195. package/dist/sources/providers/provider-utils.js +13 -7
  196. package/dist/sources/providers/sync-from-ref.js +9 -1
  197. package/dist/sources/providers/tar-utils.js +16 -8
  198. package/dist/sources/providers/website.js +9 -5
  199. package/dist/sources/website-ingest.js +187 -29
  200. package/dist/sources/wiki-fetchers/registry.js +53 -0
  201. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  202. package/dist/storage/database.js +45 -10
  203. package/dist/storage/managed-db.js +82 -0
  204. package/dist/storage/repositories/registry-cache.js +92 -0
  205. package/dist/storage/sqlite-pragmas.js +146 -0
  206. package/dist/tasks/backends/cron.js +1 -1
  207. package/dist/tasks/backends/launchd.js +1 -1
  208. package/dist/tasks/backends/schtasks.js +1 -1
  209. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  210. package/dist/tasks/runner.js +5 -13
  211. package/dist/text-import-hook.mjs +0 -0
  212. package/dist/wiki/wiki.js +37 -0
  213. package/dist/workflows/db.js +3 -4
  214. package/dist/workflows/runtime/runs.js +1 -117
  215. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  216. package/dist/workflows/validate-summary.js +2 -7
  217. package/docs/data-and-telemetry.md +3 -2
  218. package/docs/migration/release-notes/0.9.0.md +39 -0
  219. package/package.json +13 -11
  220. package/dist/commands/db-cli.js +0 -23
  221. package/dist/indexer/db/db-backup.js +0 -376
@@ -5,52 +5,50 @@ import { createHash } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
7
  import readline from "node:readline";
8
- import { parse as yamlParse } from "yaml";
8
+ import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
9
9
  import { parseAssetRef } from "../../core/asset/asset-ref.js";
10
10
  import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
11
11
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
12
12
  import { resolveStashDir, timestampForFilename } from "../../core/common.js";
13
13
  import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
14
14
  import { ConfigError } from "../../core/errors.js";
15
- import { appendEvent } from "../../core/events.js";
15
+ // Note: appendEvent import removed (WS-3a: archive TTL machinery retired)
16
16
  import { parseEmbeddedJsonResponse } from "../../core/parse.js";
17
+ import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
17
18
  import { detectTruncatedDescription } from "../../core/text-truncation.js";
18
- import { hasHotCaptureMode, hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
19
+ import { hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
19
20
  import { createProposal, isProposalSkipped, listProposals } from "../proposal/validators/proposals.js";
21
+ import { cacheHash, runDeterministicDedup, stripFrontmatterBody } from "./dedup.js";
22
+ import { checkGenerationGuard, checkLexicalDiversity, computeMergedGeneration, readAssetGeneration, runHomeostaticDemotion, shouldSkipHotProbationInLlm, } from "./homeostatic.js";
20
23
  import { writeContradictEdge } from "./memory/memory-belief.js";
21
24
  // Re-export the moved helpers so existing test imports continue to resolve.
22
25
  export { hasSupersededStatus, validateProposalFrontmatter };
26
+ import { getBodyEmbeddings, getConsolidationJudgedMap, openStateDatabase, upsertBodyEmbeddings, upsertConsolidationJudged, withStateDb, } from "../../core/state-db.js";
23
27
  import { warn } from "../../core/warn.js";
24
28
  import { commitWriteTargetBoundary, deleteAssetFromSource, resolveWriteTarget, writeAssetToSource, } from "../../core/write-source.js";
25
29
  import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../../indexer/db/db.js";
26
30
  import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
27
31
  import { chatCompletion } from "../../llm/client.js";
28
- import { cosineSimilarity, embedBatch } from "../../llm/embedder.js";
32
+ import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
29
33
  import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
34
+ // Chunk sizing + per-chunk prompt assembly live in ./consolidate/chunking.
35
+ // Imported for internal use by the orchestrator and re-exported for importers.
36
+ import { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
37
+ export { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
38
+ // LLM-output sanitization (pure string/frontmatter transforms) lives in
39
+ // ./consolidate/sanitize. Imported for internal use + re-exported for importers.
40
+ import { normalizeUpdatedField, sanitizeMergedContent } from "./consolidate/sanitize.js";
41
+ export { normalizeUpdatedField, sanitizeMergedContent, stripOuterCodeFence } from "./consolidate/sanitize.js";
42
+ // Eligibility / safety predicates live in ./consolidate/eligibility. Imported
43
+ // for internal guard use; the two public predicates are re-exported.
44
+ import { consolidateGuardStatus, isConsolidationEligibleMemoryName, isHotCapturedMemory, isSessionCaptureMemoryName, } from "./consolidate/eligibility.js";
45
+ export { isConsolidationEligibleMemoryName, isHotCapturedMemory, isSessionCaptureMemoryName, } from "./consolidate/eligibility.js";
46
+ // Plan parsing / merging (pure op-reconciliation algebra) lives in
47
+ // ./consolidate/merge. Imported for internal use; mergePlans re-exported.
48
+ import { isValidOp, mergePlans } from "./consolidate/merge.js";
49
+ export { mergePlans } from "./consolidate/merge.js";
30
50
  // ── Prompts ─────────────────────────────────────────────────────────────────
31
- const CONSOLIDATE_SYSTEM_PROMPT = `You are the akm consolidate assistant analyzing memory assets.
32
-
33
- Rules:
34
- 1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
35
- 2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
36
- 3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
37
- 4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") → mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
38
- 5. KEEP: Memory is unique and current → omit from output.
39
-
40
- Return ONLY JSON (no prose, no code fences):
41
- {
42
- "operations": [
43
- { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
44
- { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
45
- { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
46
- { "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
47
- ],
48
- "warnings": ["<optional concerns>"]
49
- }
50
-
51
- For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
52
-
53
- When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
51
+ const CONSOLIDATE_SYSTEM_PROMPT = consolidateSystemPrompt;
54
52
  /**
55
53
  * JSON Schema for structured consolidate plans (PR 1 of the asset-writers
56
54
  * decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
@@ -136,140 +134,17 @@ export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
136
134
  },
137
135
  },
138
136
  };
139
- export function isConsolidationEligibleMemoryName(name) {
140
- return !name.endsWith(".derived");
141
- }
142
- /**
143
- * Returns true when the memory file has `captureMode: hot` in its frontmatter.
144
- *
145
- * Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
146
- * The consolidate LLM is forbidden from deleting or auto-merging them — the
147
- * user wrote them on purpose and only the user can decide to retire them.
148
- *
149
- * Reads the file once per check; consolidate runs against ~10 memories per
150
- * chunk so the IO cost is trivial. Returns false on any read/parse error
151
- * (fail-safe: an unparseable file is treated as not-hot, but the broader
152
- * consolidate flow already guards against unparseable memories elsewhere).
153
- *
154
- * Defends against four observed defect classes (see
155
- * `memory:akm-improve-critical-review-2026-05-20`):
156
- * - LLM marks a memory contradicted then deletes (dangling contradictedBy)
157
- * - LLM merges two unrelated memories sharing a topic keyword
158
- * - LLM judges a recent durable design memo as "redundant"
159
- * - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
160
- */
161
- export function isHotCapturedMemory(filePath) {
162
- try {
163
- if (!fs.existsSync(filePath))
164
- return false;
165
- const content = fs.readFileSync(filePath, "utf8");
166
- const parsed = parseFrontmatter(content);
167
- return hasHotCaptureMode(parsed.data);
168
- }
169
- catch {
170
- return false;
171
- }
172
- }
173
- function consolidateGuardStatus(filePath) {
174
- if (!fs.existsSync(filePath))
175
- return "missing";
176
- let content;
177
- try {
178
- content = fs.readFileSync(filePath, "utf8");
179
- }
180
- catch {
181
- return "unparseable";
182
- }
183
- let parsed;
184
- try {
185
- parsed = parseFrontmatter(content);
186
- }
187
- catch {
188
- return "unparseable";
189
- }
190
- const data = parsed.data;
191
- if (!data || Object.keys(data).length === 0)
192
- return "unparseable";
193
- return hasHotCaptureMode(data) ? "hot" : "safe";
194
- }
195
- // ── Chunk sizing ─────────────────────────────────────────────────────────────
196
- /**
197
- * Conservative chars-per-token estimate used when computing prompt budgets.
198
- * English text averages roughly 4 chars/token for most LLM tokenizers. We use
199
- * 3 to stay conservative (shorter tokens = more tokens per char).
200
- */
201
- const CHARS_PER_TOKEN = 3;
202
- /**
203
- * Overhead budget reserved for the system prompt, chunk header lines, and per-
204
- * memory metadata lines (name, description, tags, separator). Measured at
205
- * roughly 600 chars for the system prompt + ~100 chars of header + ~50 chars
206
- * per memory × chunk size. We round up to 2 000 tokens to leave room for the
207
- * model's own output.
208
- */
209
- const PROMPT_OVERHEAD_TOKENS = 2_000;
210
- /**
211
- * Default effective token budget used when the default LLM profile's
212
- * `contextLength` is not set. This is intentionally conservative (4 096)
213
- * rather than being set to the model's actual context window, because:
214
- *
215
- * - When the agent path is used, the agent CLI (e.g. opencode)
216
- * prepends its own large system prompt + conversation history before
217
- * forwarding to the model. That overhead easily consumes 30K+ tokens on
218
- * a model with a 16K context window, leaving very little room for
219
- * chunk content.
220
- * - When the HTTP path is used (an LLM profile is selected), only the akm
221
- * system prompt and user prompt are sent, so the budget can be set to the
222
- * model's actual context length via profiles.llm[defaults.llm].contextLength.
223
- *
224
- * Set profiles.llm[defaults.llm].contextLength in your config file to the
225
- * model's actual context window to allow larger chunks on the HTTP path.
226
- */
227
- export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
228
- /**
229
- * Given the model's context window and the per-memory body truncation limit,
230
- * return the maximum number of memories that can safely fit in one chunk
231
- * without the prompt overflowing the context window.
232
- *
233
- * The formula is:
234
- * usableTokens = contextLength - PROMPT_OVERHEAD_TOKENS
235
- * tokensPerMemory = ceil(bodyTruncation / CHARS_PER_TOKEN)
236
- * chunkSize = floor(usableTokens / tokensPerMemory)
237
- *
238
- * Result is clamped between 1 and 50 to avoid degenerate values.
239
- *
240
- * @param contextLength - Model context window in tokens.
241
- * @param bodyTruncation - Max chars per memory body included in the prompt.
242
- * @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
243
- */
244
- export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
245
- const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
246
- const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
247
- const raw = Math.floor(usableTokens / tokensPerMemory);
248
- return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
249
- }
250
- // ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
251
- /**
252
- * Re-order memories so that similar ones are placed adjacent to each other
253
- * before the memories are sliced into chunks. This ensures high-similarity
254
- * memories land in the same LLM context window, allowing the consolidate
255
- * model to detect and merge duplicates that would otherwise be split across
256
- * chunks and survive indefinitely.
257
- *
258
- * Algorithm: greedy nearest-neighbour chain starting from the first memory.
259
- * Each step selects the unused memory with the highest cosine similarity to
260
- * the last-placed memory. O(n²) — acceptable for the expected N < 200.
261
- *
262
- * mem0 arXiv:2504.19413 — every candidate compared against whole store.
263
- * A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
264
- *
265
- * Returns the original order unchanged when:
266
- * - The embedding config is not present.
267
- * - Embedding requests fail (fail-open).
268
- * - There are fewer than 3 memories (no benefit to reordering).
269
- */
270
- async function clusterMemoriesBySimilarity(memories, config) {
137
+ async function clusterMemoriesBySimilarity(memories, config, stateDb) {
138
+ const noTelemetry = { embedMs: 0, cacheHits: 0, cacheMisses: 0 };
271
139
  if (memories.length < 3 || !config.embedding)
272
- return memories;
140
+ return { ordered: memories, embedTelemetry: noTelemetry };
141
+ // WS-3a: cluster uses description+tags as the embedding input (NOT the raw
142
+ // body) — this is intentionally different from the dedup/body cache because
143
+ // the clustering goal is semantic grouping, not dedup twin detection.
144
+ // The body_embeddings cache is keyed by cacheHash(body); clustering inputs
145
+ // are keyed by cacheHash(description+tags text). Re-use the same table with
146
+ // a distinct hash so the two lookup sets never collide.
147
+ const modelId = resolveEmbeddingModelId(config.embedding);
273
148
  const texts = memories.map((m) => {
274
149
  const parts = [];
275
150
  if (m.description)
@@ -278,16 +153,91 @@ async function clusterMemoriesBySimilarity(memories, config) {
278
153
  parts.push(m.tags.join(" "));
279
154
  return parts.join(". ") || m.name;
280
155
  });
281
- let embeddings = null;
282
- try {
283
- embeddings = await embedBatch(texts, config.embedding);
156
+ // Compute content hashes for the cluster texts (not bodies — different input).
157
+ const contentHashes = texts.map((t) => createHash("sha256").update(t, "utf8").digest("hex"));
158
+ // WS-5: track embed cache hits/misses for perf telemetry.
159
+ let embedMs = 0;
160
+ let cacheHits = 0;
161
+ let cacheMisses = 0;
162
+ let cachedVecs = new Map();
163
+ if (stateDb) {
164
+ try {
165
+ cachedVecs = getBodyEmbeddings(stateDb, contentHashes, modelId);
166
+ }
167
+ catch {
168
+ // Fail open.
169
+ cachedVecs = new Map();
170
+ }
284
171
  }
285
- catch {
286
- // Fail open: embedding failures degrade gracefully to original order.
287
- return memories;
172
+ const missIndices = [];
173
+ const missTexts = [];
174
+ for (let i = 0; i < texts.length; i++) {
175
+ if (!cachedVecs.has(contentHashes[i])) {
176
+ missIndices.push(i);
177
+ missTexts.push(texts[i]);
178
+ cacheMisses++;
179
+ }
180
+ else {
181
+ cacheHits++;
182
+ }
288
183
  }
184
+ let missVecs = [];
185
+ if (missTexts.length > 0) {
186
+ const embedStart = Date.now();
187
+ try {
188
+ missVecs = await embedBatch(missTexts, config.embedding);
189
+ }
190
+ catch {
191
+ // Fail open: embedding failures degrade gracefully to original order.
192
+ return { ordered: memories, embedTelemetry: { embedMs, cacheHits, cacheMisses } };
193
+ }
194
+ finally {
195
+ embedMs += Date.now() - embedStart;
196
+ }
197
+ // Upsert newly computed vectors into the cache.
198
+ if (stateDb && missVecs.length === missTexts.length) {
199
+ try {
200
+ const toUpsert = missIndices.map((idx, pos) => ({
201
+ contentHash: contentHashes[idx],
202
+ embedding: missVecs[pos],
203
+ modelId,
204
+ }));
205
+ upsertBodyEmbeddings(stateDb, toUpsert);
206
+ }
207
+ catch {
208
+ // Fail open: cache write errors are non-fatal.
209
+ }
210
+ }
211
+ }
212
+ // Assemble the full embedding array in memories order.
213
+ let embeddings = null;
214
+ {
215
+ const assembled = [];
216
+ let ok = true;
217
+ for (let i = 0; i < memories.length; i++) {
218
+ const hash = contentHashes[i];
219
+ const cached = cachedVecs.get(hash);
220
+ if (cached) {
221
+ assembled.push(cached);
222
+ continue;
223
+ }
224
+ const missPos = missIndices.indexOf(i);
225
+ const vec = missPos >= 0 ? missVecs[missPos] : undefined;
226
+ if (vec) {
227
+ assembled.push(vec);
228
+ }
229
+ else {
230
+ ok = false;
231
+ break;
232
+ }
233
+ }
234
+ if (ok && assembled.length === memories.length) {
235
+ embeddings = assembled;
236
+ }
237
+ }
238
+ const embedTelemetry = { embedMs, cacheHits, cacheMisses };
289
239
  if (!embeddings || embeddings.length !== memories.length)
290
- return memories;
240
+ return { ordered: memories, embedTelemetry };
291
241
  // Greedy nearest-neighbour chain.
292
242
  const used = new Array(memories.length).fill(false);
293
243
  const ordered = [];
@@ -313,87 +263,16 @@ async function clusterMemoriesBySimilarity(memories, config) {
313
263
  used[bestIdx] = true;
314
264
  current = bestIdx;
315
265
  }
316
- return ordered;
266
+ return { ordered, embedTelemetry };
317
267
  }
318
268
  // ── Chunk helpers ────────────────────────────────────────────────────────────
319
- /**
320
- * Build the per-chunk user prompt fed to the consolidate LLM.
321
- *
322
- * Each memory is annotated with two flags that drive the system-prompt
323
- * rules at lines 181-186:
324
- * - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
325
- * forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
326
- * stack before this annotation.
327
- * - `(already queued)` — the memory's body hash matches a pending
328
- * consolidate proposal; system prompt rule 3 forbids proposing
329
- * promote/merge/contradict. ~107/4h before this annotation.
330
- *
331
- * Both annotations are visible to the LLM. `pendingProposalBodyHashes`
332
- * is precomputed once per run by `loadPendingConsolidateProposalHashes`
333
- * so the cost stays O(memories) inside the chunk loop.
334
- */
335
- export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
336
- const start = memories[0] ? `memory:${memories[0].name}` : "";
337
- const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
338
- const annotationsByIndex = [];
339
- const hotRefs = [];
340
- for (const m of memories) {
341
- let body = "";
342
- try {
343
- body = fs.readFileSync(m.filePath, "utf8");
344
- }
345
- catch {
346
- body = "(unreadable)";
347
- }
348
- const parsed = parseFrontmatter(body);
349
- const isHot = parsed.data.captureMode === "hot";
350
- const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
351
- const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
352
- annotationsByIndex.push({ isHot, isAlreadyQueued, body });
353
- if (isHot)
354
- hotRefs.push(`memory:${m.name}`);
355
- }
356
- const lines = [
357
- `Source: ${sourceName}`,
358
- `Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
359
- "",
360
- ];
361
- // Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
362
- // op-words like "promote", "merge", "contradict" so the model doesn't
363
- // accidentally treat the warning as a hint to use that op elsewhere
364
- // (variant B leaked the word "contradict" into the control sample
365
- // during the diagnostic).
366
- if (hotRefs.length > 0) {
367
- lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
368
- for (const ref of hotRefs)
369
- lines.push(` - ${ref}`);
370
- lines.push("");
371
- }
372
- for (let i = 0; i < memories.length; i++) {
373
- const m = memories[i];
374
- const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
375
- const annotations = [];
376
- if (isHot)
377
- annotations.push("captureMode: hot");
378
- if (isAlreadyQueued)
379
- annotations.push("already queued");
380
- const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
381
- lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
382
- lines.push(`Description: ${m.description || "(none)"}`);
383
- lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
384
- lines.push("---");
385
- lines.push(body.slice(0, bodyTruncation));
386
- lines.push("");
387
- }
388
- return lines.join("\n");
389
- }
390
269
  /**
391
270
  * Precompute body-hashes of all currently-pending consolidate proposals so
392
271
  * the per-chunk prompt can annotate memories whose body would just produce
393
- * a deterministic `dedup_pending_proposal` skip. Hash domain matches the
394
- * dedup site at ~line 1510 (sha256 over the post-frontmatter content,
395
- * trimmed). Empty set on any read/parse error — fail-safe to "annotate
396
- * nothing" so the LLM still proposes, just slightly more wastefully.
272
+ * a deterministic `dedup_pending_proposal` skip. Uses `cacheHash` (case-
273
+ * preserving stripped body) the same domain used by the body-embedding
274
+ * cache and `computeMemoryContentHash`. Empty set on any read/parse error
275
+ * — fail-safe to "annotate nothing" so the LLM still proposes.
397
276
  */
398
277
  function loadPendingConsolidateProposalHashes(stashDir) {
399
278
  const hashes = new Set();
@@ -401,8 +280,7 @@ function loadPendingConsolidateProposalHashes(stashDir) {
401
280
  const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
402
281
  for (const p of pending) {
403
282
  try {
404
- const body = parseFrontmatter(p.payload.content).content.trim();
405
- hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
283
+ hashes.add(cacheHash(p.payload.content));
406
284
  }
407
285
  catch {
408
286
  // skip malformed payloads — they can't dedup anyway
@@ -414,148 +292,6 @@ function loadPendingConsolidateProposalHashes(stashDir) {
414
292
  }
415
293
  return hashes;
416
294
  }
417
- function isValidOp(op) {
418
- if (typeof op !== "object" || op === null)
419
- return false;
420
- const o = op;
421
- if (o.op === "merge") {
422
- return typeof o.primary === "string" && Array.isArray(o.secondaries);
423
- }
424
- if (o.op === "delete") {
425
- return typeof o.ref === "string";
426
- }
427
- if (o.op === "promote") {
428
- return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
429
- }
430
- if (o.op === "contradict") {
431
- return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
432
- }
433
- return false;
434
- }
435
- export function mergePlans(chunks, knownRefs) {
436
- const mergeOps = new Map();
437
- const deleteOps = new Map();
438
- const promoteOps = new Map();
439
- // C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
440
- const contradictOps = new Map();
441
- const warnings = [];
442
- for (const chunk of chunks) {
443
- for (const op of chunk) {
444
- if (op.op === "merge") {
445
- // Drop ops whose primary the LLM hallucinated (not in the loaded memory
446
- // pool). Without this guard, a hallucinated primary flows all the way to
447
- // Phase B where !memoryByRef.has(primary) fires and charges every real
448
- // secondary with merge_primary_missing — masking LLM hallucinations as
449
- // filter regressions in health metrics.
450
- if (knownRefs && !knownRefs.has(op.primary)) {
451
- warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
452
- // Use a dedicated skip reason so dashboards can distinguish
453
- // hallucinated primaries from stale-DB regressions.
454
- // Secondaries are real refs; they are NOT charged here — they remain
455
- // available for other ops to claim.
456
- continue;
457
- }
458
- // Filter hallucinated secondaries while preserving real ones.
459
- let mergeOp = op;
460
- if (knownRefs) {
461
- const filteredSecondaries = op.secondaries.filter((sec) => {
462
- if (!knownRefs.has(sec)) {
463
- warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
464
- return false;
465
- }
466
- return true;
467
- });
468
- if (filteredSecondaries.length !== op.secondaries.length) {
469
- mergeOp = { ...op, secondaries: filteredSecondaries };
470
- }
471
- }
472
- // merge wins over delete
473
- if (deleteOps.has(mergeOp.primary)) {
474
- deleteOps.delete(mergeOp.primary);
475
- }
476
- for (const sec of mergeOp.secondaries) {
477
- if (deleteOps.has(sec))
478
- deleteOps.delete(sec);
479
- }
480
- mergeOps.set(mergeOp.primary, mergeOp);
481
- }
482
- else if (op.op === "delete") {
483
- // merge and promote both win over delete. A promote is non-destructive
484
- // (creates a proposal) but the source memory is counted in `promoted`;
485
- // if a delete also fires, the ref lands in both `promoted` and
486
- // `skipReasons`, breaking the invariant by +1.
487
- if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
488
- deleteOps.set(op.ref, op);
489
- }
490
- }
491
- else if (op.op === "promote") {
492
- // C-2 / #381: when both a promote and a merge target the same ref,
493
- // queue the promote FIRST rather than discarding it. The promote op
494
- // routes through createProposal (the human-gated proposal queue), so
495
- // it is non-destructive. The merge follows after the proposal is
496
- // created. This preserves the human reviewer's ability to inspect the
497
- // promotion before the source memory is merged/deleted.
498
- // AGM K*8 — retain the maximally informative consistent subset.
499
- promoteOps.set(op.ref, op);
500
- }
501
- else if (op.op === "contradict") {
502
- // Deduplicate by ref+contradictedByRef pair.
503
- const key = `${op.ref}|${op.contradictedByRef}`;
504
- if (!contradictOps.has(key)) {
505
- contradictOps.set(key, op);
506
- }
507
- }
508
- }
509
- }
510
- // Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
511
- //
512
- // 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
513
- // for secondaries that were already in deleteOps, but misses the case
514
- // where the delete chunk came first. A full sweep here fixes both orders.
515
- //
516
- // 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
517
- // only the first (insertion-order) retains it. Without this, a successful
518
- // merge credits A to mergedSecondaries and a later merge's emitMerge-
519
- // FailureSkips also charges A to skipReasons — double-counting A while
520
- // processed has it only once.
521
- //
522
- // 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
523
- // a secondary in another, remove A from the secondary list. Both merges
524
- // would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
525
- // breaking the invariant the same way.
526
- // Also remove delete ops for any ref claimed by a promote op (handles the
527
- // case where the delete chunk appeared before the promote chunk).
528
- for (const ref of promoteOps.keys()) {
529
- deleteOps.delete(ref);
530
- }
531
- const claimedSecondaries = new Set();
532
- for (const mergeOp of mergeOps.values()) {
533
- deleteOps.delete(mergeOp.primary);
534
- mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
535
- if (mergeOps.has(sec)) {
536
- warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
537
- return false;
538
- }
539
- if (claimedSecondaries.has(sec)) {
540
- warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
541
- return false;
542
- }
543
- claimedSecondaries.add(sec);
544
- deleteOps.delete(sec);
545
- return true;
546
- });
547
- }
548
- // C-2 / #381: promote ops are ordered BEFORE merge ops so that the
549
- // human-gated proposal queue entry is created before any destructive merge.
550
- // Phase B processes ops in array order, so promote executes first.
551
- const ops = [
552
- ...promoteOps.values(),
553
- ...mergeOps.values(),
554
- ...deleteOps.values(),
555
- ...contradictOps.values(),
556
- ];
557
- return { ops, warnings };
558
- }
559
295
  function getJournalPath(stashDir) {
560
296
  return path.join(stashDir, ".akm", "consolidate-journal.json");
561
297
  }
@@ -672,6 +408,28 @@ function backupFile(filePath, backupDir, name) {
672
408
  // best-effort
673
409
  }
674
410
  }
411
+ // ── WS-3b: Generation frontmatter injection ───────────────────────────────────
412
+ /**
413
+ * Inject `generation` (and optionally `source_refs`) into merged content.
414
+ * generation = max(sourceGenerations) + 1.
415
+ * Fails open — returns original content if frontmatter can't be parsed.
416
+ */
417
+ function injectGenerationFrontmatter(mergedContent, sourceGenerations, allParticipants) {
418
+ try {
419
+ const parsed = parseFrontmatter(mergedContent);
420
+ const updatedFm = {
421
+ ...parsed.data,
422
+ generation: computeMergedGeneration(sourceGenerations),
423
+ };
424
+ if (!updatedFm.source_refs) {
425
+ updatedFm.source_refs = allParticipants;
426
+ }
427
+ return assembleAssetFromString(serializeFrontmatter(updatedFm), parsed.content);
428
+ }
429
+ catch {
430
+ return mergedContent; // fail open
431
+ }
432
+ }
675
433
  // ── Archive helper (P1-B: soft-invalidation) ─────────────────────────────────
676
434
  /**
677
435
  * Move a memory asset to `.akm/archive/` with `status: superseded` frontmatter
@@ -752,6 +510,25 @@ function resolveConsolidateLlmConfig(config) {
752
510
  // fall back to the default LLM profile rather than disabling the pass.
753
511
  return getDefaultLlmConfig(config);
754
512
  }
513
+ // ── Judged-state cache (#581) ────────────────────────────────────────────────
514
+ /**
515
+ * Stable content hash for a memory file used by the judged-state cache (#581)
516
+ * and the body-embedding cache (WS-3a). Uses `cacheHash` from dedup.ts:
517
+ * sha256 of the case-preserving stripped body. Two memories that differ only
518
+ * in frontmatter (`updated:`, `inferenceProcessed:`) hash identically, so a
519
+ * cosmetic frontmatter touch never forces a needless re-judge — only a body
520
+ * change does. Returns `undefined` on any read/parse error so callers fail
521
+ * open (treat the memory as un-cached → it stays in the LLM pool).
522
+ */
523
+ function computeMemoryContentHash(filePath) {
524
+ try {
525
+ const raw = fs.readFileSync(filePath, "utf8");
526
+ return cacheHash(raw);
527
+ }
528
+ catch {
529
+ return undefined;
530
+ }
531
+ }
755
532
  // ── Main entry point ─────────────────────────────────────────────────────────
756
533
  export async function akmConsolidate(opts = {}) {
757
534
  const startMs = Date.now();
@@ -780,6 +557,27 @@ export async function akmConsolidate(opts = {}) {
780
557
  }
781
558
  const warnings = [];
782
559
  checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
560
+ // WS-3a: open one state.db handle shared by the body-embedding cache (dedup
561
+ // + cluster) and the judged-state cache. All callers in the function body
562
+ // receive this handle; it is closed in the `finally` block below.
563
+ // Fail-open: any open error leaves it `undefined` and all cache paths skip.
564
+ let sharedStateDb;
565
+ try {
566
+ sharedStateDb = openStateDatabase();
567
+ }
568
+ catch {
569
+ // State DB unavailable → skip the embedding cache for this run.
570
+ }
571
+ try {
572
+ return await akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb);
573
+ }
574
+ finally {
575
+ sharedStateDb?.close();
576
+ }
577
+ }
578
+ // Inner implementation — all early-return paths are here; sharedStateDb is
579
+ // closed by the outer finally in `akmConsolidate`.
580
+ async function akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb) {
783
581
  let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
784
582
  // Pre-flight: filter out stale DB entries whose files no longer exist on
785
583
  // disk. Without this, memories deleted by a prior run (but not yet
@@ -791,6 +589,80 @@ export async function akmConsolidate(opts = {}) {
791
589
  warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
792
590
  }
793
591
  memories = memories.filter((m) => fs.existsSync(m.filePath));
592
+ // ── WS-3b Step 0a: Homeostatic demotion ────────────────────────────────────
593
+ // DEFAULT OFF. Before any LLM merge, demote retrievalSalience in state.db
594
+ // for stale/low-value assets so the merge pool is bounded and high-SNR.
595
+ // Demotion is state.db-only (file content untouched); re-promotable on
596
+ // re-retrieval. Only fires when `homeostaticDemotion.enabled === true`.
597
+ const homeostaticConfig = config.profiles?.improve?.default?.processes?.consolidate?.homeostaticDemotion ?? {};
598
+ if (homeostaticConfig.enabled && sharedStateDb) {
599
+ const demotionResult = runHomeostaticDemotion(sharedStateDb, homeostaticConfig);
600
+ if (demotionResult.demoted > 0) {
601
+ warnings.push(`Homeostatic demotion: demoted retrievalSalience for ${demotionResult.demoted} stale asset(s) before merge pool assembly.`);
602
+ }
603
+ warnings.push(...demotionResult.warnings);
604
+ }
605
+ // ── WS-3b Step 0c: Filter hot-probation assets from LLM merge pool ─────────
606
+ // Hot-probation assets (system-generated, not yet graduated from intake pass)
607
+ // are processed by the dedup pre-pass but excluded from the LLM clustering.
608
+ // This prevents noisy extractions from polluting LLM context. The dedup pass
609
+ // below still runs against them so they're cleaned up deterministically.
610
+ // DEFAULT OFF — only active when `processes.extract.hotProbation.enabled === true`
611
+ // (the flag that causes extract to tag new extractions as hot-probation).
612
+ // Without that flag no assets will ever carry the hot-probation marker, so
613
+ // running the filter loop would be pure unnecessary I/O over the full corpus.
614
+ const hotProbationEnabled = config.profiles?.improve?.default?.processes?.extract?.hotProbation
615
+ ?.enabled === true;
616
+ let hotProbationCount = 0;
617
+ if (hotProbationEnabled) {
618
+ const hotProbationMemories = [];
619
+ const nonProbationMemories = [];
620
+ for (const m of memories) {
621
+ try {
622
+ const raw = fs.readFileSync(m.filePath, "utf8");
623
+ const parsed = parseFrontmatter(raw);
624
+ if (shouldSkipHotProbationInLlm(parsed.data)) {
625
+ hotProbationMemories.push(m);
626
+ hotProbationCount++;
627
+ }
628
+ else {
629
+ nonProbationMemories.push(m);
630
+ }
631
+ }
632
+ catch {
633
+ nonProbationMemories.push(m); // fail open
634
+ }
635
+ }
636
+ if (hotProbationCount > 0) {
637
+ warnings.push(`Hot-probation: ${hotProbationCount} hot-probation asset(s) routed to dedup-only pass (excluded from LLM merge pool).`);
638
+ memories = nonProbationMemories;
639
+ }
640
+ }
641
+ // ── Deterministic dedup pre-pass (#617) ─────────────────────────────────────
642
+ // Cheap, no-LLM fast path that collapses the obvious near-duplicates
643
+ // (`.derived` ↔ origin pairs + content twins) BEFORE the embedding-clustered
644
+ // LLM consolidation. DEFAULT OFF — when `dedup.enabled !== true` this is a
645
+ // no-op and the pass behaves byte-identically to today. Collapsed variants
646
+ // are pruned from the LLM pool so the model only ever sees genuinely
647
+ // distinct-but-related memories. Each dropped variant is archived (soft
648
+ // invalidation) before deletion, matching the LLM merge path.
649
+ // Dry-run never mutates the filesystem, so the dedup pre-pass is skipped
650
+ // entirely under `--dry-run` (the LLM plan preview below is unaffected).
651
+ let dedupCollapsed = 0;
652
+ if (opts.dedup?.enabled && !opts.dryRun) {
653
+ const dedupTimestamp = timestampForFilename();
654
+ const dedupResult = await runDeterministicDedup(stashDir, opts.dedup, config, (variantFilePath, variantName) => {
655
+ archiveMemory(variantFilePath, stashDir, `memory:${variantName}`, "collapsed by deterministic dedup pre-pass", -1, undefined, warnings);
656
+ backupFile(variantFilePath, getBackupDir(stashDir, dedupTimestamp), variantName);
657
+ }, opts.signal, sharedStateDb);
658
+ dedupCollapsed = dedupResult.collapsed;
659
+ warnings.push(...dedupResult.warnings);
660
+ if (dedupResult.consumedRefs.length > 0) {
661
+ const consumed = new Set(dedupResult.consumedRefs);
662
+ memories = memories.filter((m) => !consumed.has(`memory:${m.name}`));
663
+ warnings.push(`Deterministic dedup: collapsed ${dedupResult.collapsed} near-duplicate memor${dedupResult.collapsed === 1 ? "y" : "ies"} (no LLM) before chunking.`);
664
+ }
665
+ }
794
666
  if (memories.length === 0) {
795
667
  return {
796
668
  schemaVersion: 1,
@@ -801,7 +673,10 @@ export async function akmConsolidate(opts = {}) {
801
673
  target: opts.target ?? stashDir,
802
674
  processed: 0,
803
675
  merged: 0,
804
- deleted: 0,
676
+ // #617: the deterministic dedup pre-pass may have emptied the pool by
677
+ // collapsing every remaining memory into a canonical. Surface those
678
+ // collapses in `deleted` so the run reports the work it actually did.
679
+ deleted: dedupCollapsed,
805
680
  promoted: [],
806
681
  contradicted: 0,
807
682
  warnings,
@@ -809,7 +684,7 @@ export async function akmConsolidate(opts = {}) {
809
684
  };
810
685
  }
811
686
  if (opts.incrementalSince) {
812
- memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings);
687
+ memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings, opts.neighborsPerChanged);
813
688
  if (memories.length === 0) {
814
689
  return {
815
690
  schemaVersion: 1,
@@ -828,6 +703,109 @@ export async function akmConsolidate(opts = {}) {
828
703
  };
829
704
  }
830
705
  }
706
+ // WS-5 perf telemetry accumulators. These are collected throughout the run and
707
+ // merged into `perfTelemetry` on the final ConsolidateResult.
708
+ // `dedupPoolSize` = memories entering judgedCache narrowing (after dedup+incremental+limit).
709
+ // `judgedCacheSkipped` = memories skipped by the cache.
710
+ // `llmPoolSize` = memories actually sent to the LLM.
711
+ // `embedMs/cacheHits/cacheMisses` = accumulated from clusterMemoriesBySimilarity.
712
+ const perfMs = { dedupPoolSize: memories.length, judgedCacheSkipped: 0 };
713
+ // ── Judged-state cache narrowing (#581) ─────────────────────────────────────
714
+ // DEFAULT OFF. When enabled, skip every memory whose current content hash
715
+ // equals the hash recorded the last time the consolidate LLM judged it
716
+ // (judged-unchanged → no re-judge). This converts coverage from O(window) to
717
+ // O(changed/new) so one run can sweep the whole corpus while the LLM only
718
+ // sees genuinely new/changed memories. `currentHashByName` is populated for
719
+ // EVERY surviving memory (whether or not the cache is on) so the post-LLM
720
+ // recording step can upsert judged state without re-reading the files; when
721
+ // the cache is off it stays empty and the recording step is a no-op.
722
+ const judgedCacheEnabled = opts.judgedCache?.enabled !== false;
723
+ const currentHashByName = new Map();
724
+ if (judgedCacheEnabled) {
725
+ for (const m of memories) {
726
+ const h = computeMemoryContentHash(m.filePath);
727
+ if (h !== undefined)
728
+ currentHashByName.set(m.name, h);
729
+ }
730
+ let cachedMap = new Map();
731
+ {
732
+ // Use the shared state.db handle if available; open a local one otherwise.
733
+ const dbForJudged = sharedStateDb;
734
+ if (dbForJudged) {
735
+ try {
736
+ cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => `memory:${m.name}`));
737
+ }
738
+ catch {
739
+ cachedMap = new Map();
740
+ }
741
+ }
742
+ else {
743
+ try {
744
+ cachedMap = withStateDb((localDb) => getConsolidationJudgedMap(localDb, memories.map((m) => `memory:${m.name}`)));
745
+ }
746
+ catch {
747
+ // State DB unavailable → fail open: judge the full pool this run.
748
+ cachedMap = new Map();
749
+ }
750
+ }
751
+ }
752
+ const beforeCount = memories.length;
753
+ memories = memories.filter((m) => {
754
+ const cur = currentHashByName.get(m.name);
755
+ // No readable hash → keep (fail open; let the LLM judge it).
756
+ if (cur === undefined)
757
+ return true;
758
+ const cached = cachedMap.get(`memory:${m.name}`);
759
+ // Skip only when previously judged AND content is byte-identical since.
760
+ return !(cached !== undefined && cached.content_hash === cur);
761
+ });
762
+ const skipped = beforeCount - memories.length;
763
+ perfMs.judgedCacheSkipped = skipped; // WS-5 perf telemetry
764
+ if (skipped > 0) {
765
+ warnings.push(`Judged-state cache: skipped ${skipped} memor${skipped === 1 ? "y" : "ies"} judged-unchanged (no LLM); ${memories.length} remain for judging.`);
766
+ }
767
+ if (memories.length === 0) {
768
+ return {
769
+ schemaVersion: 1,
770
+ ok: true,
771
+ shape: "consolidate-result",
772
+ dryRun: opts.dryRun ?? false,
773
+ previewOnly: false,
774
+ target: opts.target ?? stashDir,
775
+ processed: 0,
776
+ merged: 0,
777
+ deleted: dedupCollapsed,
778
+ promoted: [],
779
+ contradicted: 0,
780
+ warnings,
781
+ durationMs: Date.now() - startMs,
782
+ };
783
+ }
784
+ }
785
+ if (opts.limit === undefined && memories.length > 150) {
786
+ warnings.push(`Consolidation: pool has ${memories.length} memories and no limit is set. Consider adding a limit to your consolidate config to prevent timeouts on slow LLM endpoints.`);
787
+ }
788
+ if (opts.limit !== undefined && memories.length > opts.limit) {
789
+ // Order oldest-modified-first before capping so the limit selects the
790
+ // stalest memories rather than a fixed head of the (rowid-ordered) DB
791
+ // query. Consolidation rewrites surviving files, bumping their mtime, so
792
+ // processed memories drift to the back of the queue and the cap rotates
793
+ // across the whole corpus over successive runs instead of revisiting the
794
+ // same slice every time. Fail-open to 0 (front of queue) when a file can
795
+ // no longer be stat'd.
796
+ const mtimeOf = (m) => {
797
+ try {
798
+ return fs.statSync(m.filePath).mtimeMs;
799
+ }
800
+ catch {
801
+ return 0;
802
+ }
803
+ };
804
+ const mtimeCache = new Map(memories.map((m) => [m.filePath, mtimeOf(m)]));
805
+ memories = [...memories].sort((a, b) => (mtimeCache.get(a.filePath) ?? 0) - (mtimeCache.get(b.filePath) ?? 0));
806
+ warnings.push(`Consolidation: pool capped at ${opts.limit} of ${memories.length} memories (limit option, oldest-modified first).`);
807
+ memories = memories.slice(0, opts.limit);
808
+ }
831
809
  // Consolidation always uses the HTTP LLM client directly — never the agent
832
810
  // CLI. The agent CLI is for interactive agent sessions (reflect, propose);
833
811
  // structured JSON generation works better and faster via HTTP.
@@ -851,16 +829,64 @@ export async function akmConsolidate(opts = {}) {
851
829
  const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
852
830
  // -- Phase A: plan generation -----------------------------------------------
853
831
  const sourceName = opts.target ?? stashDir;
832
+ // WS-5: capture llmPoolSize = memories entering the LLM (after all filtering).
833
+ const llmPoolSize = memories.length;
854
834
  // C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
855
835
  // This ensures that semantically similar memories land in the same LLM
856
836
  // context window, allowing the model to detect and merge duplicates that
857
837
  // would otherwise be split across chunks and survive indefinitely.
858
838
  // mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
859
839
  // Fails open: if embeddings are unavailable or fail, original order is used.
860
- const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
840
+ const { ordered: clusteredMemories, embedTelemetry } = await clusterMemoriesBySimilarity(memories, config, sharedStateDb);
841
+ // WS-3b Anti-collapse step 8c: inject random (non-similar) clusters.
842
+ // A small fraction (default 5%) of the pool is shuffled into random positions
843
+ // so the pipeline isn't PURELY similarity-driven. This prevents rich-get-richer
844
+ // entrenchment where only the most-retrieved assets ever get consolidated.
845
+ // DEFAULT OFF — gated on antiCollapse.enabled.
846
+ let finalClusteredMemories = clusteredMemories;
847
+ {
848
+ const antiCollapseForCluster = config.profiles?.improve?.default?.processes?.consolidate?.antiCollapse ?? {};
849
+ if (antiCollapseForCluster.enabled && clusteredMemories.length > 2) {
850
+ const fraction = antiCollapseForCluster.randomClusterFraction ?? 0.05;
851
+ const randomCount = Math.max(1, Math.floor(clusteredMemories.length * fraction));
852
+ // Pick `randomCount` positions to inject random (un-clustered) members.
853
+ // Use a seeded-ish shuffle: sort by hash of the name so it's deterministic
854
+ // per run but not strictly similarity-driven.
855
+ const shuffled = [...clusteredMemories].sort((a, b) => {
856
+ // Deterministic shuffle: compare sha256-ish (use name hash as proxy).
857
+ const ha = a.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
858
+ const hb = b.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
859
+ return ha - hb;
860
+ });
861
+ const randomSlice = shuffled.slice(0, randomCount);
862
+ const randomSet = new Set(randomSlice.map((m) => m.name));
863
+ // Insert random members at intervals through the clustered sequence.
864
+ const withRandom = [];
865
+ const interval = Math.max(2, Math.floor(clusteredMemories.length / randomCount));
866
+ let randomIdx = 0;
867
+ for (let i = 0; i < clusteredMemories.length; i++) {
868
+ const m = clusteredMemories[i];
869
+ if (m && !randomSet.has(m.name))
870
+ withRandom.push(m);
871
+ if (i > 0 && i % interval === 0 && randomIdx < randomSlice.length) {
872
+ const r = randomSlice[randomIdx++];
873
+ if (r)
874
+ withRandom.push(r);
875
+ }
876
+ }
877
+ // Append any remaining random members not yet inserted.
878
+ while (randomIdx < randomSlice.length) {
879
+ const r = randomSlice[randomIdx++];
880
+ if (r)
881
+ withRandom.push(r);
882
+ }
883
+ finalClusteredMemories = withRandom;
884
+ warnings.push(`Anti-collapse: injected ${randomCount} random (non-similarity-driven) cluster member(s) into consolidation pool (fraction=${fraction}).`);
885
+ }
886
+ }
861
887
  const chunks = [];
862
- for (let i = 0; i < clusteredMemories.length; i += chunkSize) {
863
- chunks.push(clusteredMemories.slice(i, i + chunkSize));
888
+ for (let i = 0; i < finalClusteredMemories.length; i += chunkSize) {
889
+ chunks.push(finalClusteredMemories.slice(i, i + chunkSize));
864
890
  }
865
891
  // 2026-05-27 prompt-context fix: precompute body-hashes of pending
866
892
  // consolidate proposals once, so the per-chunk prompt can annotate
@@ -869,8 +895,46 @@ export async function akmConsolidate(opts = {}) {
869
895
  // 4h on this user's stack. See
870
896
  // /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
871
897
  const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
898
+ // ── Cold-start budget estimation ─────────────────────────────────────────────
899
+ // Estimate wall-clock cost BEFORE issuing any LLM calls. When a signal is
900
+ // provided and the estimated cost exceeds ~60% of the remaining budget we
901
+ // auto-reduce the pool and log the reduction so the run never starts work
902
+ // it cannot finish (avoiding SIGTERM mid-LLM-call).
903
+ //
904
+ // Formula: chunks.length × p90_chunk_seconds. The p90 comes from
905
+ // `opts.p90ChunkSecondsDefault` (caller-supplied, typically from the profile
906
+ // config); absent = 30 s (conservative default matching a medium local LLM).
907
+ //
908
+ // "Remaining budget" is read from a custom property on the AbortSignal if
909
+ // the caller (improve.ts) has attached one. Without it no auto-reduction
910
+ // fires but the check is still cheap to run.
911
+ if (chunks.length > 10 && opts.signal) {
912
+ const p90Chunk = opts.p90ChunkSecondsDefault ?? 30;
913
+ const estimatedSeconds = chunks.length * p90Chunk;
914
+ // remainingBudgetMs is a non-standard extension set by improve.ts when it
915
+ // creates the budget AbortController. Undefined = no budget information.
916
+ const budgetMs = opts.signal.remainingBudgetMs;
917
+ if (budgetMs !== undefined && budgetMs > 0) {
918
+ const remainingSeconds = budgetMs / 1000;
919
+ if (estimatedSeconds > remainingSeconds * 0.6) {
920
+ const safeCaps = Math.max(1, Math.floor((remainingSeconds * 0.6) / p90Chunk));
921
+ const removedChunks = chunks.length - safeCaps;
922
+ if (removedChunks > 0) {
923
+ const msg = `[consolidate] cold-start budget: estimated ${estimatedSeconds.toFixed(0)}s > 60% of remaining ${remainingSeconds.toFixed(0)}s; ` +
924
+ `reducing pool from ${chunks.length} to ${safeCaps} chunks (${removedChunks} deferred to next run).`;
925
+ warn(msg);
926
+ warnings.push(msg);
927
+ chunks.splice(safeCaps);
928
+ }
929
+ }
930
+ }
931
+ }
872
932
  warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
873
933
  ` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
934
+ // Consolidate output merges memories (non-wiki) → stash authoring standards.
935
+ // Resolved ONCE per run and passed to each chunk prompt (facts not re-read
936
+ // per chunk).
937
+ const standardsContext = resolveStashStandards(stashDir);
874
938
  const chunkOpsArrays = [];
875
939
  // Structured skip-reason histogram (2026-05-26): every deterministic
876
940
  // post-LLM op rejection site below also calls `pushSkipReason` so the
@@ -907,6 +971,13 @@ export async function akmConsolidate(opts = {}) {
907
971
  // judgedNoAction tracks memories the LLM saw inside a chunk but proposed
908
972
  // no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
909
973
  let judgedNoAction = 0;
974
+ // Judged-state cache (#581): coarse outcome per memory NAME the LLM actually
975
+ // judged in a successfully-parsed chunk this run. "actioned" = an op targeted
976
+ // it; "no_action" = the LLM saw it and proposed nothing. Populated only when
977
+ // the cache is enabled (otherwise it stays empty and the post-loop recording
978
+ // step is a no-op). Memories in failed/aborted chunks are NOT recorded, so a
979
+ // transient LLM failure never poisons the cache into skipping them next run.
980
+ const judgedOutcomeByName = new Map();
910
981
  // 2026-05-27 cross-chunk double-count fix: refs that contributed to
911
982
  // judgedNoAction in their own chunk. When a different chunk's op references
912
983
  // one of these as a secondary and that op later fails, the ref would land
@@ -931,6 +1002,19 @@ export async function akmConsolidate(opts = {}) {
931
1002
  const ABORT_MIN_CHUNKS = 4;
932
1003
  const ABORT_FAILURE_RATE = 0.5;
933
1004
  for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
1005
+ // Budget-signal check: break cleanly before the next LLM call if the
1006
+ // caller's budget has been exhausted. Commits work done so far.
1007
+ if (opts.signal?.aborted) {
1008
+ const skipped = chunks.length - chunkIdx;
1009
+ const msg = `[consolidate] budget signal aborted before chunk ${chunkIdx + 1}/${chunks.length}; ${skipped} chunk(s) not processed (partial_timeout — work done so far committed).`;
1010
+ warn(msg);
1011
+ warnings.push(msg);
1012
+ // Account for memories in unprocessed chunks.
1013
+ for (let i = chunkIdx; i < chunks.length; i++) {
1014
+ failedChunkMemories += chunks[i].length;
1015
+ }
1016
+ break;
1017
+ }
934
1018
  // Abort if failure rate >= 50% over at least 4 processed chunks.
935
1019
  if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
936
1020
  const failureRate = totalChunksFailed / totalChunksProcessed;
@@ -968,7 +1052,7 @@ export async function akmConsolidate(opts = {}) {
968
1052
  continue;
969
1053
  }
970
1054
  warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
971
- const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
1055
+ const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes, standardsContext);
972
1056
  let raw = await tryLlmFeature("memory_consolidation", config, async () => {
973
1057
  if (!llmConfig)
974
1058
  return { ok: false, error: "No LLM configured for consolidation" };
@@ -1075,11 +1159,59 @@ export async function akmConsolidate(opts = {}) {
1075
1159
  if (!targetRefs.has(memRef)) {
1076
1160
  chunkNoAction++;
1077
1161
  judgedNoActionRefs.add(memRef);
1162
+ // Judged-state cache (#581): the LLM saw this memory and proposed
1163
+ // nothing → record judged-unchanged so the next run can skip it.
1164
+ if (judgedCacheEnabled)
1165
+ judgedOutcomeByName.set(m.name, "no_action");
1166
+ }
1167
+ else if (judgedCacheEnabled) {
1168
+ // An op targeted this memory → it was judged + actioned.
1169
+ judgedOutcomeByName.set(m.name, "actioned");
1078
1170
  }
1079
1171
  }
1080
1172
  judgedNoAction += chunkNoAction;
1081
1173
  chunkOpsArrays.push(ops);
1082
1174
  }
1175
+ // ── Judged-state cache recording (#581) ─────────────────────────────────────
1176
+ // Persist judged state for every memory the LLM actually judged this run so
1177
+ // the next run can skip the unchanged ones. Keyed by current content hash so
1178
+ // a later body edit (different hash) re-enters the LLM pool. DEFAULT OFF and
1179
+ // skipped under --dry-run (dry-run mutates nothing). Failed/aborted chunks
1180
+ // contributed no entries to `judgedOutcomeByName`, so a transient LLM outage
1181
+ // never caches a memory as judged.
1182
+ if (judgedCacheEnabled && !opts.dryRun && judgedOutcomeByName.size > 0) {
1183
+ // Use the shared state.db handle; open a local one as fallback.
1184
+ const doRecord = (db) => {
1185
+ const judgedAt = new Date(startMs).toISOString();
1186
+ for (const [name, outcome] of judgedOutcomeByName) {
1187
+ const hash = currentHashByName.get(name);
1188
+ if (hash === undefined)
1189
+ continue;
1190
+ upsertConsolidationJudged(db, {
1191
+ entryKey: `memory:${name}`,
1192
+ contentHash: hash,
1193
+ judgedAt,
1194
+ outcome,
1195
+ });
1196
+ }
1197
+ };
1198
+ if (sharedStateDb) {
1199
+ try {
1200
+ doRecord(sharedStateDb);
1201
+ }
1202
+ catch (e) {
1203
+ warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
1204
+ }
1205
+ }
1206
+ else {
1207
+ try {
1208
+ withStateDb((localDb) => doRecord(localDb));
1209
+ }
1210
+ catch (e) {
1211
+ warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
1212
+ }
1213
+ }
1214
+ }
1083
1215
  // Build the known-refs set from the already-filtered memory pool so
1084
1216
  // mergePlans() can reject LLM-hallucinated primary refs before execution.
1085
1217
  const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
@@ -1266,7 +1398,7 @@ export async function akmConsolidate(opts = {}) {
1266
1398
  emitMergeFailureSkips(mergeResult.error);
1267
1399
  continue;
1268
1400
  }
1269
- const mergedContent = mergeResult.content;
1401
+ let mergedContent = mergeResult.content;
1270
1402
  // Validate frontmatter of merged content — must have a `---` block
1271
1403
  // with at minimum a `description` field. We parse via the hand-rolled
1272
1404
  // parser (cheap) AND require non-empty description. This guards against
@@ -1320,6 +1452,62 @@ export async function akmConsolidate(opts = {}) {
1320
1452
  emitMergeFailureSkips("merge_participant_blocked");
1321
1453
  continue;
1322
1454
  }
1455
+ // WS-3b: Anti-collapse generation guard (step 8a).
1456
+ // DEFAULT OFF. When antiCollapse.enabled, refuse to merge two assets both
1457
+ // above generation N (default 2). This prevents the pipeline from
1458
+ // building ever-deeper LLM-merged trees that lose the source fidelity
1459
+ // of the original episodes.
1460
+ const antiCollapseConfig = config.profiles?.improve?.default?.processes?.consolidate?.antiCollapse ??
1461
+ {};
1462
+ if (antiCollapseConfig.enabled) {
1463
+ const allParticipants = [op.primary, ...op.secondaries];
1464
+ const sourceGenerations = allParticipants.map((ref) => {
1465
+ const e = memoryByRef.get(ref);
1466
+ if (!e)
1467
+ return 0;
1468
+ try {
1469
+ const raw = fs.readFileSync(e.filePath, "utf8");
1470
+ const parsed = parseFrontmatter(raw);
1471
+ return readAssetGeneration(parsed.data);
1472
+ }
1473
+ catch {
1474
+ return 0;
1475
+ }
1476
+ });
1477
+ const generationCheck = checkGenerationGuard(sourceGenerations, antiCollapseConfig);
1478
+ if (generationCheck.refused) {
1479
+ warnings.push(`Merge: ${generationCheck.reason}`);
1480
+ emitMergeFailureSkips("merge_generation_guard");
1481
+ continue;
1482
+ }
1483
+ // WS-3b: Lexical diversity check (step 8b).
1484
+ // Low n-gram diversity ⇒ likely correlated-extraction artifact; raise merge threshold.
1485
+ if (antiCollapseConfig.lexicalDiversityCheck !== false) {
1486
+ const bodies = allParticipants
1487
+ .map((ref) => {
1488
+ const e = memoryByRef.get(ref);
1489
+ if (!e)
1490
+ return "";
1491
+ try {
1492
+ const raw = fs.readFileSync(e.filePath, "utf8");
1493
+ return stripFrontmatterBody(raw);
1494
+ }
1495
+ catch {
1496
+ return "";
1497
+ }
1498
+ })
1499
+ .filter((b) => b.length > 0);
1500
+ const diversityCheck = checkLexicalDiversity(bodies, antiCollapseConfig);
1501
+ if (diversityCheck.lowDiversity) {
1502
+ // Low-diversity cluster: just warn (don't refuse merge since the dedup
1503
+ // path handles exact twins). The warning surfaces in health telemetry.
1504
+ warnings.push(`Merge: cluster around ${op.primary} has low lexical diversity (${diversityCheck.diversity?.toFixed(2) ?? "?"} < 0.30) — likely correlated extraction; merge proceeds but review is recommended.`);
1505
+ }
1506
+ }
1507
+ // Inject generation counter into merged content frontmatter (step 8a).
1508
+ // merged.generation = max(sourceGenerations) + 1.
1509
+ mergedContent = injectGenerationFrontmatter(mergedContent, sourceGenerations, allParticipants);
1510
+ }
1323
1511
  // Backup secondaries before deleting
1324
1512
  for (const secRef of op.secondaries) {
1325
1513
  const secEntry = memoryByRef.get(secRef);
@@ -1522,11 +1710,12 @@ export async function akmConsolidate(opts = {}) {
1522
1710
  // is the load-bearing content, so dedup must hash on body only.
1523
1711
  // (b) A prior run created a proposal for the same body under a
1524
1712
  // different knowledgeRef slug.
1525
- const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
1713
+ // Use cacheHash (case-preserving stripped body) to match the canonical
1714
+ // hash domain used by the body-embedding cache and pending-proposal set.
1715
+ const bodyHash = cacheHash(sourceBody);
1526
1716
  const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1527
1717
  const contentDupProposal = allPendingConsolidateProposals.find((p) => {
1528
- const otherBody = parseFrontmatter(p.payload.content).content.trim();
1529
- return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
1718
+ return cacheHash(p.payload.content) === bodyHash;
1530
1719
  });
1531
1720
  if (contentDupProposal) {
1532
1721
  warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
@@ -1608,6 +1797,18 @@ export async function akmConsolidate(opts = {}) {
1608
1797
  }
1609
1798
  }
1610
1799
  else if (op.op === "contradict") {
1800
+ // Confidence gate: surface-level topic overlap causes false positives
1801
+ // (investigation 2026-06-18). Require ≥0.92 confidence before writing
1802
+ // contradiction edges. Missing confidence field defaults to 1.0 for
1803
+ // backward compatibility with responses that predate this field.
1804
+ const opConfidence = typeof op.confidence === "number"
1805
+ ? op.confidence
1806
+ : 1.0;
1807
+ if (opConfidence < 0.92) {
1808
+ warnings.push(`Contradict: confidence ${opConfidence.toFixed(2)} below 0.92 threshold for ${op.ref} <-> ${op.contradictedByRef} — skipping.`);
1809
+ pushSkipReason("contradict", op.ref, "contradict_low_confidence");
1810
+ continue;
1811
+ }
1611
1812
  // C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
1612
1813
  // (the SCC resolver in memory-improve.ts) has edges to work on.
1613
1814
  // Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
@@ -1647,37 +1848,19 @@ export async function akmConsolidate(opts = {}) {
1647
1848
  commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
1648
1849
  }
1649
1850
  cleanupJournal(stashDir, timestamp);
1650
- // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
1651
- // C-5 / #391: emit an `archive_cleanup` event before each deletion so the
1652
- // audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf)
1653
- // any event that is recorded must be queryable; silent deletes are an anti-pattern.
1654
- const archiveDir = path.join(stashDir, ".akm", "archive");
1655
- if (fs.existsSync(archiveDir)) {
1656
- const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
1657
- const cutoff = Date.now() - retentionMs;
1658
- for (const fname of fs.readdirSync(archiveDir)) {
1659
- const fp = path.join(archiveDir, fname);
1660
- try {
1661
- const stat = fs.statSync(fp);
1662
- if (stat.mtimeMs < cutoff) {
1663
- // Emit event before deletion so the record survives the purge.
1664
- appendEvent({
1665
- eventType: "archive_cleanup",
1666
- metadata: {
1667
- file: fname,
1668
- filePath: fp,
1669
- ageMs: Date.now() - stat.mtimeMs,
1670
- retentionMs,
1671
- },
1672
- });
1673
- fs.unlinkSync(fp);
1674
- }
1675
- }
1676
- catch {
1677
- /* ignore race conditions */
1678
- }
1679
- }
1680
- }
1851
+ // [signoff 2026-06-15] TTL archive cleanup machinery RETIRED (WS-3a).
1852
+ // The elaborate archiveRetentionDays / archive-dir scan existed only to satisfy
1853
+ // the old irrecoverability constraint. Stashes are now git-backed, so git
1854
+ // history is the recovery path no bespoke archive TTL needed. Any files in
1855
+ // .akm/archive/ will stay there harmlessly until the operator prunes them with
1856
+ // `git rm` or `find .akm/archive -mtime +90 -delete`. Changed N files this
1857
+ // run; recover any via `git show <sha>:<path>` or `git restore <path>`.
1858
+ if (merged > 0 || deleted > 0 || dedupCollapsed > 0) {
1859
+ const totalChanged = merged + deleted + dedupCollapsed;
1860
+ warnings.push(`Changed ${totalChanged} file(s) this run. Recover any via git if needed (git history is the backstop).`);
1861
+ }
1862
+ const runDurationMs = Date.now() - startMs;
1863
+ const budgetFraction = opts.runBudgetMs !== undefined && opts.runBudgetMs > 0 ? runDurationMs / opts.runBudgetMs : undefined;
1681
1864
  return {
1682
1865
  schemaVersion: 1,
1683
1866
  ok: true,
@@ -1687,7 +1870,10 @@ export async function akmConsolidate(opts = {}) {
1687
1870
  target: sourceName,
1688
1871
  processed: memories.length,
1689
1872
  merged,
1690
- deleted,
1873
+ // #617: fold the deterministic dedup pre-pass collapses into the reported
1874
+ // deleted count. Each collapse removed exactly one variant file with NO
1875
+ // LLM call before the LLM pass ran on the pruned pool.
1876
+ deleted: deleted + dedupCollapsed,
1691
1877
  promoted,
1692
1878
  contradicted,
1693
1879
  failedChunks: totalChunksFailed,
@@ -1697,235 +1883,19 @@ export async function akmConsolidate(opts = {}) {
1697
1883
  mergedSecondaries,
1698
1884
  failedChunkMemories,
1699
1885
  warnings,
1700
- durationMs: Date.now() - startMs,
1886
+ durationMs: runDurationMs,
1887
+ perfTelemetry: {
1888
+ dedupPoolSize: perfMs.dedupPoolSize,
1889
+ llmPoolSize,
1890
+ judgedCacheSkipped: perfMs.judgedCacheSkipped,
1891
+ embedMs: embedTelemetry.embedMs,
1892
+ embedCacheHits: embedTelemetry.cacheHits,
1893
+ embedCacheMisses: embedTelemetry.cacheMisses,
1894
+ ...(budgetFraction !== undefined ? { estimatedBudgetFractionUsed: budgetFraction } : {}),
1895
+ },
1701
1896
  };
1702
1897
  }
1703
1898
  // ── Helpers ─────────────────────────────────────────────────────────────────
1704
- // ── LLM-output sanitization ─────────────────────────────────────────────────
1705
- //
1706
- // Three classes of LLM defect have been observed across hundreds of
1707
- // consolidate proposals (see audit notes in this branch):
1708
- //
1709
- // 1. Code-fence leakage: the entire merged asset is wrapped in
1710
- // ```markdown … ``` (or ```yaml … ```) despite the prompt forbidding
1711
- // fences. The post-processor used to pass this through verbatim, so the
1712
- // first character of the asset content became a backtick rather than
1713
- // `---`, defeating the frontmatter parser.
1714
- // 2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
1715
- // with unbalanced quotes that break the YAML reader. The post-processor
1716
- // historically passed the LLM's raw scalar straight into a manually
1717
- // assembled `description: <raw>` line.
1718
- // 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
1719
- // runs out mid-sentence, leaving things like
1720
- // `description: "Tables in narrow column containers need max-width:100% +"`
1721
- // with no closing context.
1722
- //
1723
- // `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
1724
- // all three at the point where LLM output is consumed.
1725
- /**
1726
- * Attempt to recover a frontmatter block that is missing its closing `---`.
1727
- *
1728
- * Scans lines after the opening `---` for the first blank line or the first
1729
- * line that cannot be a YAML scalar (i.e. not a key-value, indented
1730
- * continuation, comment, or list item). Injects `---` before that line so
1731
- * the normal parser can proceed.
1732
- *
1733
- * Returns the patched string on success, or `null` if the structure is too
1734
- * ambiguous to recover safely (e.g. no opening `---`, or no body content
1735
- * found after the frontmatter key-value lines).
1736
- */
1737
- function recoverMalformedFrontmatter(raw) {
1738
- if (!raw.startsWith("---"))
1739
- return null;
1740
- const lines = raw.split(/\r?\n/);
1741
- // Skip the opening `---` line (index 0).
1742
- let insertAt = -1;
1743
- for (let i = 1; i < lines.length; i++) {
1744
- const line = lines[i];
1745
- // A blank line marks the end of the frontmatter block in many YAML variants.
1746
- if (line.trim() === "") {
1747
- insertAt = i;
1748
- break;
1749
- }
1750
- // A line that is clearly body content: doesn't look like a YAML key, an
1751
- // indented continuation, a comment, or a sequence item.
1752
- const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
1753
- /^\s+\S/.test(line) || // indented continuation / nested
1754
- /^\s*#/.test(line) || // YAML comment
1755
- /^\s*-\s/.test(line); // sequence item
1756
- if (!isYaml) {
1757
- insertAt = i;
1758
- break;
1759
- }
1760
- }
1761
- if (insertAt < 0)
1762
- return null;
1763
- const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
1764
- return result;
1765
- }
1766
- /**
1767
- * Outer-fence stripper specific to consolidate. Unlike the shared
1768
- * `stripMarkdownFences` helper (which only handles markdown fences), this
1769
- * variant additionally recognises `yaml` and bare-language fences and refuses
1770
- * to strip an unbalanced fence — i.e. a leading ``` with no trailing ``` is
1771
- * treated as a malformed response, not partially sanitized.
1772
- *
1773
- * Returns `null` when only one half of a fence pair is present (caller
1774
- * should reject the response entirely).
1775
- */
1776
- export function stripOuterCodeFence(raw) {
1777
- const trimmed = raw.trim();
1778
- const leading = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n/i);
1779
- const trailing = trimmed.match(/\r?\n```\s*$/);
1780
- if (!leading && !trailing)
1781
- return { content: trimmed, stripped: false };
1782
- if (!leading || !trailing)
1783
- return null; // unbalanced — refuse
1784
- const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
1785
- return { content: inner, stripped: true };
1786
- }
1787
- export function sanitizeMergedContent(raw) {
1788
- // Step 1: Strip outer code fence.
1789
- // Recovery path: if only the leading fence is present, strip it and continue
1790
- // provided the inner content starts with `---`. Trailing-only fences are NOT
1791
- // recovered — a trailing ``` is more likely a body code block than a forgotten
1792
- // wrapper, so recovering would silently corrupt the body.
1793
- let body;
1794
- {
1795
- const fenceResult = stripOuterCodeFence(raw);
1796
- if (fenceResult) {
1797
- body = fenceResult.content;
1798
- }
1799
- else {
1800
- const trimmed = raw.trim();
1801
- const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
1802
- const inner = leadingMatch ? leadingMatch[1].trim() : null;
1803
- if (!inner?.startsWith("---")) {
1804
- return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
1805
- }
1806
- body = inner;
1807
- }
1808
- }
1809
- // Strip <think> blocks (some local models still emit them despite system prompts).
1810
- body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
1811
- // Step 2: Verify frontmatter sentinel.
1812
- // Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
1813
- // merged content:") before the `---`. Accept if `---` appears within 300 chars.
1814
- // Beyond that it's more likely a body section divider, not a frontmatter start.
1815
- if (!body.startsWith("---")) {
1816
- const nlIdx = body.indexOf("\n---");
1817
- if (nlIdx >= 0 && nlIdx < 300) {
1818
- body = body.slice(nlIdx + 1);
1819
- }
1820
- else {
1821
- return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
1822
- }
1823
- }
1824
- // Extract frontmatter block.
1825
- // Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
1826
- // by scanning lines after the opening `---` for the first blank line or the
1827
- // first line that isn't a YAML key-value pair, then inject `---` there.
1828
- let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1829
- if (!match) {
1830
- const recovered = recoverMalformedFrontmatter(body);
1831
- if (recovered) {
1832
- match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1833
- }
1834
- if (!match) {
1835
- return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
1836
- }
1837
- }
1838
- // Re-parse via the yaml library so any quote-escaping mistakes either get
1839
- // normalised or surface as a parse error we can reject.
1840
- // Recovery: if the strict yaml library fails, fall back to the lenient
1841
- // hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
1842
- // quirks (unescaped special chars, bare scalars, etc.). If it recovers
1843
- // at least one key, proceed — serializeFrontmatter below will re-serialize
1844
- // cleanly. Only reject if both parsers fail to extract any data.
1845
- let parsedFm;
1846
- try {
1847
- parsedFm = yamlParse(match[1]);
1848
- }
1849
- catch (e) {
1850
- const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
1851
- if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
1852
- parsedFm = fallback.data;
1853
- }
1854
- else {
1855
- return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
1856
- }
1857
- }
1858
- if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
1859
- return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
1860
- }
1861
- const fm = parsedFm;
1862
- // Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
1863
- // `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
1864
- // these, but small models still do. Replace any such leak with today's ISO
1865
- // date OR drop the field if we can't safely normalise it.
1866
- normalizeUpdatedField(fm);
1867
- // Re-serialise via yaml.stringify to fix any quoting quirks.
1868
- let serialized;
1869
- try {
1870
- serialized = serializeFrontmatter(fm);
1871
- }
1872
- catch (e) {
1873
- return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
1874
- }
1875
- const cleaned = assembleAssetFromString(serialized, match[2]);
1876
- return { ok: true, result: { content: cleaned, frontmatter: fm } };
1877
- }
1878
- /**
1879
- * Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
1880
- * LLM. The consolidate prompt forbids these, but small models still produce
1881
- * literal `today` / `{today: null}` / `now` values.
1882
- *
1883
- * Rules:
1884
- * - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
1885
- * - A Date object (some YAML parsers materialise dates) is converted to its
1886
- * ISO yyyy-mm-dd form.
1887
- * - A placeholder string ("today", "now", "{today}", "${today}", template
1888
- * variables) is replaced with today's ISO date.
1889
- * - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
1890
- * - `null`, empty string, missing → left alone (no field added; reviewers
1891
- * should not silently gain metadata they didn't write).
1892
- *
1893
- * Exported for unit testing.
1894
- */
1895
- export function normalizeUpdatedField(fm) {
1896
- if (!("updated" in fm))
1897
- return;
1898
- const v = fm.updated;
1899
- if (v === null || v === undefined || v === "")
1900
- return;
1901
- const todayIso = new Date().toISOString().slice(0, 10);
1902
- if (v instanceof Date) {
1903
- fm.updated = v.toISOString().slice(0, 10);
1904
- return;
1905
- }
1906
- if (typeof v === "string") {
1907
- const trimmed = v.trim().toLowerCase();
1908
- if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
1909
- return; // already a real date
1910
- if (trimmed === "today" ||
1911
- trimmed === "now" ||
1912
- trimmed === "{today}" ||
1913
- // biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
1914
- trimmed === "${today}" ||
1915
- trimmed === "{{today}}" ||
1916
- /^\{?\s*today\s*\}?$/.test(trimmed)) {
1917
- fm.updated = todayIso;
1918
- return;
1919
- }
1920
- // Unknown string format — leave alone so it's visible in the diff.
1921
- return;
1922
- }
1923
- if (typeof v === "object") {
1924
- // Maps like `{today: null}`, `{now: null}` — clearly a template leak.
1925
- fm.updated = todayIso;
1926
- return;
1927
- }
1928
- }
1929
1899
  /**
1930
1900
  * Normalise a knowledge slug for variant-aware deduplication. Collapses:
1931
1901
  * - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
@@ -2004,7 +1974,7 @@ function parseSinceToIso(since) {
2004
1974
  const multiplier = { m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2]];
2005
1975
  return new Date(Date.now() - parseInt(m[1], 10) * multiplier).toISOString();
2006
1976
  }
2007
- export function narrowToIncrementalCandidates(memories, since, warnings) {
1977
+ export function narrowToIncrementalCandidates(memories, since, warnings, neighborsPerChanged = 5) {
2008
1978
  const sinceIso = parseSinceToIso(since);
2009
1979
  const isChanged = (m) => {
2010
1980
  try {
@@ -2019,7 +1989,6 @@ export function narrowToIncrementalCandidates(memories, since, warnings) {
2019
1989
  return [];
2020
1990
  if (changed.length === memories.length)
2021
1991
  return memories;
2022
- const NEIGHBORS_PER_CHANGED = 5;
2023
1992
  const byName = new Map(memories.map((m) => [m.name, m]));
2024
1993
  const keep = new Set(changed.map((m) => m.name));
2025
1994
  let db;
@@ -2029,7 +1998,7 @@ export function narrowToIncrementalCandidates(memories, since, warnings) {
2029
1998
  const id = findEntryIdByRef(db, `memory:${m.name}`);
2030
1999
  if (id === undefined)
2031
2000
  continue;
2032
- for (const hit of getNeighborsByEntryId(db, id, NEIGHBORS_PER_CHANGED + 1)) {
2001
+ for (const hit of getNeighborsByEntryId(db, id, neighborsPerChanged + 1)) {
2033
2002
  if (hit.id === id)
2034
2003
  continue;
2035
2004
  const entry = getEntryById(db, hit.id);
@@ -2067,6 +2036,10 @@ function loadMemoriesForSource(source, stashDir, warnings) {
2067
2036
  return path.resolve(e.stashDir) === path.resolve(source);
2068
2037
  })
2069
2038
  .filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
2039
+ // #632 — exclude session-capture telemetry (checkpoints) from the
2040
+ // consolidation pool, mirroring recombine. Their bodies are pipeline
2041
+ // bookkeeping, so consolidating them burns LLM calls on pure noise.
2042
+ .filter((e) => !isSessionCaptureMemoryName(e.entry.name))
2070
2043
  // Skip stale DB entries whose file was deleted by a prior run but not yet
2071
2044
  // re-indexed. Without this guard the deleted file's ref appears in chunks
2072
2045
  // sent to the LLM, which then proposes a second delete → delete_failed
@@ -2100,6 +2073,8 @@ function loadMemoriesForSource(source, stashDir, warnings) {
2100
2073
  const name = fname.replace(/\.md$/, "");
2101
2074
  if (!isConsolidationEligibleMemoryName(name))
2102
2075
  continue;
2076
+ if (isSessionCaptureMemoryName(name))
2077
+ continue; // #632 — skip telemetry checkpoints
2103
2078
  memories.push({ name, filePath, description: "", tags: [], stashDir: fsStashDir });
2104
2079
  }
2105
2080
  }
@@ -2231,8 +2206,8 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
2231
2206
  }
2232
2207
  normalizeUpdatedField(repairedFmData);
2233
2208
  const repairedYaml = serializeFrontmatter(repairedFmData);
2234
- const bodyPart = mergedFm.content ?? "";
2235
- return { content: `---\n${repairedYaml}\n---\n${bodyPart}` };
2209
+ const bodyPart = typeof mergedFm.content === "string" ? mergedFm.content : "";
2210
+ return { content: assembleAssetFromString(repairedYaml, bodyPart) };
2236
2211
  }
2237
2212
  }
2238
2213
  catch {