akm-cli 0.9.0-beta.11 → 0.9.0-beta.26

package/dist/commands/improve/consolidate.js

@@ -6,51 +6,33 @@ import fs from "node:fs";
 import path from "node:path";
 import readline from "node:readline";
 import { parse as yamlParse } from "yaml";
+import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
 import { resolveStashDir, timestampForFilename } from "../../core/common.js";
 import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
 import { ConfigError } from "../../core/errors.js";
-import { appendEvent } from "../../core/events.js";
+// Note: appendEvent import removed (WS-3a: archive TTL machinery retired)
 import { parseEmbeddedJsonResponse } from "../../core/parse.js";
 import { detectTruncatedDescription } from "../../core/text-truncation.js";
 import { hasHotCaptureMode, hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
 import { createProposal, isProposalSkipped, listProposals } from "../proposal/validators/proposals.js";
+import { cacheHash, runDeterministicDedup, stripFrontmatterBody } from "./dedup.js";
+import { checkGenerationGuard, checkLexicalDiversity, computeMergedGeneration, readAssetGeneration, runHomeostaticDemotion, shouldSkipHotProbationInLlm, } from "./homeostatic.js";
 import { writeContradictEdge } from "./memory/memory-belief.js";
 // Re-export the moved helpers so existing test imports continue to resolve.
 export { hasSupersededStatus, validateProposalFrontmatter };
+import { getBodyEmbeddings, getConsolidationJudgedMap, openStateDatabase, upsertBodyEmbeddings, upsertConsolidationJudged, } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
 import { commitWriteTargetBoundary, deleteAssetFromSource, resolveWriteTarget, writeAssetToSource, } from "../../core/write-source.js";
 import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../../indexer/db/db.js";
 import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
 import { chatCompletion } from "../../llm/client.js";
-import { cosineSimilarity, embedBatch } from "../../llm/embedder.js";
+import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
 import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
 // ── Prompts ─────────────────────────────────────────────────────────────────
-const CONSOLIDATE_SYSTEM_PROMPT = `You are the akm consolidate assistant analyzing memory assets.
-
-Rules:
-1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
-2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
-3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
-4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") → mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
-5. KEEP: Memory is unique and current → omit from output.
-
-Return ONLY JSON (no prose, no code fences):
-{
-  "operations": [
-    { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
-    { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
-    { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
-    { "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
-  ],
-  "warnings": ["<optional concerns>"]
-}
-
-For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
-
-When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
+const CONSOLIDATE_SYSTEM_PROMPT = consolidateSystemPrompt;
 /**
  * JSON Schema for structured consolidate plans (PR 1 of the asset-writers
  * decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
@@ -247,29 +229,17 @@ export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize
     const raw = Math.floor(usableTokens / tokensPerMemory);
     return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
 }
-// ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
-/**
- * Re-order memories so that similar ones are placed adjacent to each other
- * before the memories are sliced into chunks. This ensures high-similarity
- * memories land in the same LLM context window, allowing the consolidate
- * model to detect and merge duplicates that would otherwise be split across
- * chunks and survive indefinitely.
- *
- * Algorithm: greedy nearest-neighbour chain starting from the first memory.
- * Each step selects the unused memory with the highest cosine similarity to
- * the last-placed memory. O(n²) — acceptable for the expected N < 200.
- *
- * mem0 arXiv:2504.19413 — every candidate compared against whole store.
- * A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
- *
- * Returns the original order unchanged when:
- *   - The embedding config is not present.
- *   - Embedding requests fail (fail-open).
- *   - There are fewer than 3 memories (no benefit to reordering).
- */
-async function clusterMemoriesBySimilarity(memories, config) {
+async function clusterMemoriesBySimilarity(memories, config, stateDb) {
+    const noTelemetry = { embedMs: 0, cacheHits: 0, cacheMisses: 0 };
     if (memories.length < 3 || !config.embedding)
-        return memories;
+        return { ordered: memories, embedTelemetry: noTelemetry };
+    // WS-3a: cluster uses description+tags as the embedding input (NOT the raw
+    // body) — this is intentionally different from the dedup/body cache because
+    // the clustering goal is semantic grouping, not dedup twin detection.
+    // The body_embeddings cache is keyed by cacheHash(body); clustering inputs
+    // are keyed by cacheHash(description+tags text). Re-use the same table with
+    // a distinct hash so the two lookup sets never collide.
+    const modelId = resolveEmbeddingModelId(config.embedding);
     const texts = memories.map((m) => {
         const parts = [];
         if (m.description)
@@ -278,16 +248,91 @@ async function clusterMemoriesBySimilarity(memories, config) {
             parts.push(m.tags.join(" "));
         return parts.join(". ") || m.name;
     });
-    let embeddings = null;
-    try {
-        embeddings = await embedBatch(texts, config.embedding);
+    // Compute content hashes for the cluster texts (not bodies — different input).
+    const contentHashes = texts.map((t) => createHash("sha256").update(t, "utf8").digest("hex"));
+    // WS-5: track embed cache hits/misses for perf telemetry.
+    let embedMs = 0;
+    let cacheHits = 0;
+    let cacheMisses = 0;
+    let cachedVecs = new Map();
+    if (stateDb) {
+        try {
+            cachedVecs = getBodyEmbeddings(stateDb, contentHashes, modelId);
+        }
+        catch {
+            // Fail open.
+            cachedVecs = new Map();
+        }
     }
-    catch {
-        // Fail open: embedding failures degrade gracefully to original order.
-        return memories;
+    const missIndices = [];
+    const missTexts = [];
+    for (let i = 0; i < texts.length; i++) {
+        if (!cachedVecs.has(contentHashes[i])) {
+            missIndices.push(i);
+            missTexts.push(texts[i]);
+            cacheMisses++;
+        }
+        else {
+            cacheHits++;
+        }
     }
+    let missVecs = [];
+    if (missTexts.length > 0) {
+        const embedStart = Date.now();
+        try {
+            missVecs = await embedBatch(missTexts, config.embedding);
+        }
+        catch {
+            // Fail open: embedding failures degrade gracefully to original order.
+            return { ordered: memories, embedTelemetry: { embedMs, cacheHits, cacheMisses } };
+        }
+        finally {
+            embedMs += Date.now() - embedStart;
+        }
+        // Upsert newly computed vectors into the cache.
+        if (stateDb && missVecs.length === missTexts.length) {
+            try {
+                const toUpsert = missIndices.map((idx, pos) => ({
+                    contentHash: contentHashes[idx],
+                    embedding: missVecs[pos],
+                    modelId,
+                }));
+                upsertBodyEmbeddings(stateDb, toUpsert);
+            }
+            catch {
+                // Fail open: cache write errors are non-fatal.
+            }
+        }
+    }
+    // Assemble the full embedding array in memories order.
+    let embeddings = null;
+    {
+        const assembled = [];
+        let ok = true;
+        for (let i = 0; i < memories.length; i++) {
+            const hash = contentHashes[i];
+            const cached = cachedVecs.get(hash);
+            if (cached) {
+                assembled.push(cached);
+                continue;
+            }
+            const missPos = missIndices.indexOf(i);
+            const vec = missPos >= 0 ? missVecs[missPos] : undefined;
+            if (vec) {
+                assembled.push(vec);
+            }
+            else {
+                ok = false;
+                break;
+            }
+        }
+        if (ok && assembled.length === memories.length) {
+            embeddings = assembled;
+        }
+    }
+    const embedTelemetry = { embedMs, cacheHits, cacheMisses };
     if (!embeddings || embeddings.length !== memories.length)
-        return memories;
+        return { ordered: memories, embedTelemetry };
     // Greedy nearest-neighbour chain.
     const used = new Array(memories.length).fill(false);
     const ordered = [];
@@ -313,7 +358,7 @@ async function clusterMemoriesBySimilarity(memories, config) {
         used[bestIdx] = true;
         current = bestIdx;
     }
-    return ordered;
+    return { ordered, embedTelemetry };
 }
 // ── Chunk helpers ────────────────────────────────────────────────────────────
 /**
@@ -347,7 +392,9 @@ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks,
         }
         const parsed = parseFrontmatter(body);
         const isHot = parsed.data.captureMode === "hot";
-        const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
+        // Use cacheHash (case-preserving stripped body) to match the domain used
+        // by loadPendingConsolidateProposalHashes and the body-embedding cache.
+        const bodyHash = cacheHash(body);
         const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
         annotationsByIndex.push({ isHot, isAlreadyQueued, body });
         if (isHot)
@@ -390,10 +437,10 @@ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks,
 /**
  * Precompute body-hashes of all currently-pending consolidate proposals so
  * the per-chunk prompt can annotate memories whose body would just produce
- * a deterministic `dedup_pending_proposal` skip. Hash domain matches the
- * dedup site at ~line 1510 (sha256 over the post-frontmatter content,
- * trimmed). Empty set on any read/parse error — fail-safe to "annotate
- * nothing" so the LLM still proposes, just slightly more wastefully.
+ * a deterministic `dedup_pending_proposal` skip. Uses `cacheHash` (case-
+ * preserving stripped body) — the same domain used by the body-embedding
+ * cache and `computeMemoryContentHash`. Empty set on any read/parse error
+ * — fail-safe to "annotate nothing" so the LLM still proposes.
  */
 function loadPendingConsolidateProposalHashes(stashDir) {
     const hashes = new Set();
@@ -401,8 +448,7 @@ function loadPendingConsolidateProposalHashes(stashDir) {
         const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
         for (const p of pending) {
             try {
-                const body = parseFrontmatter(p.payload.content).content.trim();
-                hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
+                hashes.add(cacheHash(p.payload.content));
             }
             catch {
                 // skip malformed payloads — they can't dedup anyway
@@ -672,6 +718,28 @@ function backupFile(filePath, backupDir, name) {
         // best-effort
     }
 }
+// ── WS-3b: Generation frontmatter injection ───────────────────────────────────
+/**
+ * Inject `generation` (and optionally `source_refs`) into merged content.
+ * generation = max(sourceGenerations) + 1.
+ * Fails open — returns original content if frontmatter can't be parsed.
+ */
+function injectGenerationFrontmatter(mergedContent, sourceGenerations, allParticipants) {
+    try {
+        const parsed = parseFrontmatter(mergedContent);
+        const updatedFm = {
+            ...parsed.data,
+            generation: computeMergedGeneration(sourceGenerations),
+        };
+        if (!updatedFm.source_refs) {
+            updatedFm.source_refs = allParticipants;
+        }
+        return assembleAssetFromString(serializeFrontmatter(updatedFm), parsed.content);
+    }
+    catch {
+        return mergedContent; // fail open
+    }
+}
 // ── Archive helper (P1-B: soft-invalidation) ─────────────────────────────────
 /**
  * Move a memory asset to `.akm/archive/` with `status: superseded` frontmatter
@@ -752,6 +820,25 @@ function resolveConsolidateLlmConfig(config) {
     // fall back to the default LLM profile rather than disabling the pass.
     return getDefaultLlmConfig(config);
 }
+// ── Judged-state cache (#581) ────────────────────────────────────────────────
+/**
+ * Stable content hash for a memory file used by the judged-state cache (#581)
+ * and the body-embedding cache (WS-3a). Uses `cacheHash` from dedup.ts:
+ * sha256 of the case-preserving stripped body. Two memories that differ only
+ * in frontmatter (`updated:`, `inferenceProcessed:`) hash identically, so a
+ * cosmetic frontmatter touch never forces a needless re-judge — only a body
+ * change does. Returns `undefined` on any read/parse error so callers fail
+ * open (treat the memory as un-cached → it stays in the LLM pool).
+ */
+function computeMemoryContentHash(filePath) {
+    try {
+        const raw = fs.readFileSync(filePath, "utf8");
+        return cacheHash(raw);
+    }
+    catch {
+        return undefined;
+    }
+}
 // ── Main entry point ─────────────────────────────────────────────────────────
 export async function akmConsolidate(opts = {}) {
     const startMs = Date.now();
@@ -780,6 +867,27 @@ export async function akmConsolidate(opts = {}) {
     }
     const warnings = [];
     checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
+    // WS-3a: open one state.db handle shared by the body-embedding cache (dedup
+    // + cluster) and the judged-state cache. All callers in the function body
+    // receive this handle; it is closed in the `finally` block below.
+    // Fail-open: any open error leaves it `undefined` and all cache paths skip.
+    let sharedStateDb;
+    try {
+        sharedStateDb = openStateDatabase();
+    }
+    catch {
+        // State DB unavailable → skip the embedding cache for this run.
+    }
+    try {
+        return await akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb);
+    }
+    finally {
+        sharedStateDb?.close();
+    }
+}
+// Inner implementation — all early-return paths are here; sharedStateDb is
+// closed by the outer finally in `akmConsolidate`.
+async function akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb) {
     let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
     // Pre-flight: filter out stale DB entries whose files no longer exist on
     // disk. Without this, memories deleted by a prior run (but not yet
@@ -791,6 +899,80 @@ export async function akmConsolidate(opts = {}) {
         warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
     }
     memories = memories.filter((m) => fs.existsSync(m.filePath));
+    // ── WS-3b Step 0a: Homeostatic demotion ────────────────────────────────────
+    // DEFAULT OFF. Before any LLM merge, demote retrievalSalience in state.db
+    // for stale/low-value assets so the merge pool is bounded and high-SNR.
+    // Demotion is state.db-only (file content untouched); re-promotable on
+    // re-retrieval. Only fires when `homeostaticDemotion.enabled === true`.
+    const homeostaticConfig = config.profiles?.improve?.default?.processes?.consolidate?.homeostaticDemotion ?? {};
+    if (homeostaticConfig.enabled && sharedStateDb) {
+        const demotionResult = runHomeostaticDemotion(sharedStateDb, homeostaticConfig);
+        if (demotionResult.demoted > 0) {
+            warnings.push(`Homeostatic demotion: demoted retrievalSalience for ${demotionResult.demoted} stale asset(s) before merge pool assembly.`);
+        }
+        warnings.push(...demotionResult.warnings);
+    }
+    // ── WS-3b Step 0c: Filter hot-probation assets from LLM merge pool ─────────
+    // Hot-probation assets (system-generated, not yet graduated from intake pass)
+    // are processed by the dedup pre-pass but excluded from the LLM clustering.
+    // This prevents noisy extractions from polluting LLM context. The dedup pass
+    // below still runs against them so they're cleaned up deterministically.
+    // DEFAULT OFF — only active when `processes.extract.hotProbation.enabled === true`
+    // (the flag that causes extract to tag new extractions as hot-probation).
+    // Without that flag no assets will ever carry the hot-probation marker, so
+    // running the filter loop would be pure unnecessary I/O over the full corpus.
+    const hotProbationEnabled = config.profiles?.improve?.default?.processes?.extract?.hotProbation
+        ?.enabled === true;
+    let hotProbationCount = 0;
+    if (hotProbationEnabled) {
+        const hotProbationMemories = [];
+        const nonProbationMemories = [];
+        for (const m of memories) {
+            try {
+                const raw = fs.readFileSync(m.filePath, "utf8");
+                const parsed = parseFrontmatter(raw);
+                if (shouldSkipHotProbationInLlm(parsed.data)) {
+                    hotProbationMemories.push(m);
+                    hotProbationCount++;
+                }
+                else {
+                    nonProbationMemories.push(m);
+                }
+            }
+            catch {
+                nonProbationMemories.push(m); // fail open
+            }
+        }
+        if (hotProbationCount > 0) {
+            warnings.push(`Hot-probation: ${hotProbationCount} hot-probation asset(s) routed to dedup-only pass (excluded from LLM merge pool).`);
+            memories = nonProbationMemories;
+        }
+    }
+    // ── Deterministic dedup pre-pass (#617) ─────────────────────────────────────
+    // Cheap, no-LLM fast path that collapses the obvious near-duplicates
+    // (`.derived` ↔ origin pairs + content twins) BEFORE the embedding-clustered
+    // LLM consolidation. DEFAULT OFF — when `dedup.enabled !== true` this is a
+    // no-op and the pass behaves byte-identically to today. Collapsed variants
+    // are pruned from the LLM pool so the model only ever sees genuinely
+    // distinct-but-related memories. Each dropped variant is archived (soft
+    // invalidation) before deletion, matching the LLM merge path.
+    // Dry-run never mutates the filesystem, so the dedup pre-pass is skipped
+    // entirely under `--dry-run` (the LLM plan preview below is unaffected).
+    let dedupCollapsed = 0;
+    if (opts.dedup?.enabled && !opts.dryRun) {
+        const dedupTimestamp = timestampForFilename();
+        const dedupResult = await runDeterministicDedup(stashDir, opts.dedup, config, (variantFilePath, variantName) => {
+            archiveMemory(variantFilePath, stashDir, `memory:${variantName}`, "collapsed by deterministic dedup pre-pass", -1, undefined, warnings);
+            backupFile(variantFilePath, getBackupDir(stashDir, dedupTimestamp), variantName);
+        }, opts.signal, sharedStateDb);
+        dedupCollapsed = dedupResult.collapsed;
+        warnings.push(...dedupResult.warnings);
+        if (dedupResult.consumedRefs.length > 0) {
+            const consumed = new Set(dedupResult.consumedRefs);
+            memories = memories.filter((m) => !consumed.has(`memory:${m.name}`));
+            warnings.push(`Deterministic dedup: collapsed ${dedupResult.collapsed} near-duplicate memor${dedupResult.collapsed === 1 ? "y" : "ies"} (no LLM) before chunking.`);
+        }
+    }
     if (memories.length === 0) {
         return {
             schemaVersion: 1,
@@ -801,7 +983,10 @@ export async function akmConsolidate(opts = {}) {
             target: opts.target ?? stashDir,
             processed: 0,
             merged: 0,
-            deleted: 0,
+            // #617: the deterministic dedup pre-pass may have emptied the pool by
+            // collapsing every remaining memory into a canonical. Surface those
+            // collapses in `deleted` so the run reports the work it actually did.
+            deleted: dedupCollapsed,
             promoted: [],
             contradicted: 0,
             warnings,
@@ -828,6 +1013,93 @@ export async function akmConsolidate(opts = {}) {
             };
         }
     }
+    // WS-5 perf telemetry accumulators. These are collected throughout the run and
+    // merged into `perfTelemetry` on the final ConsolidateResult.
+    // `dedupPoolSize` = memories entering judgedCache narrowing (after dedup+incremental+limit).
+    // `judgedCacheSkipped` = memories skipped by the cache.
+    // `llmPoolSize` = memories actually sent to the LLM.
+    // `embedMs/cacheHits/cacheMisses` = accumulated from clusterMemoriesBySimilarity.
+    const perfMs = { dedupPoolSize: memories.length, judgedCacheSkipped: 0 };
+    // ── Judged-state cache narrowing (#581) ─────────────────────────────────────
+    // DEFAULT OFF. When enabled, skip every memory whose current content hash
+    // equals the hash recorded the last time the consolidate LLM judged it
+    // (judged-unchanged → no re-judge). This converts coverage from O(window) to
+    // O(changed/new) so one run can sweep the whole corpus while the LLM only
+    // sees genuinely new/changed memories. `currentHashByName` is populated for
+    // EVERY surviving memory (whether or not the cache is on) so the post-LLM
+    // recording step can upsert judged state without re-reading the files; when
+    // the cache is off it stays empty and the recording step is a no-op.
+    const judgedCacheEnabled = opts.judgedCache?.enabled !== false;
+    const currentHashByName = new Map();
+    if (judgedCacheEnabled) {
+        for (const m of memories) {
+            const h = computeMemoryContentHash(m.filePath);
+            if (h !== undefined)
+                currentHashByName.set(m.name, h);
+        }
+        let cachedMap = new Map();
+        {
+            // Use the shared state.db handle if available; open a local one otherwise.
+            const dbForJudged = sharedStateDb;
+            if (dbForJudged) {
+                try {
+                    cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => `memory:${m.name}`));
+                }
+                catch {
+                    cachedMap = new Map();
+                }
+            }
+            else {
+                let localDb;
+                try {
+                    localDb = openStateDatabase();
+                    cachedMap = getConsolidationJudgedMap(localDb, memories.map((m) => `memory:${m.name}`));
+                }
+                catch {
+                    // State DB unavailable → fail open: judge the full pool this run.
+                    cachedMap = new Map();
+                }
+                finally {
+                    localDb?.close();
+                }
+            }
+        }
+        const beforeCount = memories.length;
+        memories = memories.filter((m) => {
+            const cur = currentHashByName.get(m.name);
+            // No readable hash → keep (fail open; let the LLM judge it).
+            if (cur === undefined)
+                return true;
+            const cached = cachedMap.get(`memory:${m.name}`);
+            // Skip only when previously judged AND content is byte-identical since.
+            return !(cached !== undefined && cached.content_hash === cur);
+        });
+        const skipped = beforeCount - memories.length;
+        perfMs.judgedCacheSkipped = skipped; // WS-5 perf telemetry
+        if (skipped > 0) {
+            warnings.push(`Judged-state cache: skipped ${skipped} memor${skipped === 1 ? "y" : "ies"} judged-unchanged (no LLM); ${memories.length} remain for judging.`);
+        }
+        if (memories.length === 0) {
+            return {
+                schemaVersion: 1,
+                ok: true,
+                shape: "consolidate-result",
+                dryRun: opts.dryRun ?? false,
+                previewOnly: false,
+                target: opts.target ?? stashDir,
+                processed: 0,
+                merged: 0,
+                deleted: dedupCollapsed,
+                promoted: [],
+                contradicted: 0,
+                warnings,
+                durationMs: Date.now() - startMs,
+            };
+        }
+    }
+    if (opts.limit === undefined && memories.length > 150) {
+        warnings.push(`Consolidation: pool has ${memories.length} memories and no limit is set. Consider adding a limit to your consolidate config to prevent timeouts on slow LLM endpoints.`);
+    }
     if (opts.limit !== undefined && memories.length > opts.limit) {
         // Order oldest-modified-first before capping so the limit selects the
         // stalest memories rather than a fixed head of the (rowid-ordered) DB
@@ -872,16 +1144,64 @@ export async function akmConsolidate(opts = {}) {
     const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
     // -- Phase A: plan generation -----------------------------------------------
     const sourceName = opts.target ?? stashDir;
+    // WS-5: capture llmPoolSize = memories entering the LLM (after all filtering).
+    const llmPoolSize = memories.length;
     // C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
     // This ensures that semantically similar memories land in the same LLM
     // context window, allowing the model to detect and merge duplicates that
     // would otherwise be split across chunks and survive indefinitely.
     // mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
     // Fails open: if embeddings are unavailable or fail, original order is used.
-    const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
+    const { ordered: clusteredMemories, embedTelemetry } = await clusterMemoriesBySimilarity(memories, config, sharedStateDb);
+    // WS-3b Anti-collapse step 8c: inject random (non-similar) clusters.
+    // A small fraction (default 5%) of the pool is shuffled into random positions
+    // so the pipeline isn't PURELY similarity-driven. This prevents rich-get-richer
+    // entrenchment where only the most-retrieved assets ever get consolidated.
+    // DEFAULT OFF — gated on antiCollapse.enabled.
+    let finalClusteredMemories = clusteredMemories;
+    {
+        const antiCollapseForCluster = config.profiles?.improve?.default?.processes?.consolidate?.antiCollapse ?? {};
+        if (antiCollapseForCluster.enabled && clusteredMemories.length > 2) {
+            const fraction = antiCollapseForCluster.randomClusterFraction ?? 0.05;
+            const randomCount = Math.max(1, Math.floor(clusteredMemories.length * fraction));
+            // Pick `randomCount` positions to inject random (un-clustered) members.
+            // Use a seeded-ish shuffle: sort by hash of the name so it's deterministic
+            // per run but not strictly similarity-driven.
+            const shuffled = [...clusteredMemories].sort((a, b) => {
+                // Deterministic shuffle: compare sha256-ish (use name hash as proxy).
+                const ha = a.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
+                const hb = b.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
+                return ha - hb;
+            });
+            const randomSlice = shuffled.slice(0, randomCount);
+            const randomSet = new Set(randomSlice.map((m) => m.name));
+            // Insert random members at intervals through the clustered sequence.
+            const withRandom = [];
+            const interval = Math.max(2, Math.floor(clusteredMemories.length / randomCount));
+            let randomIdx = 0;
+            for (let i = 0; i < clusteredMemories.length; i++) {
+                const m = clusteredMemories[i];
+                if (m && !randomSet.has(m.name))
+                    withRandom.push(m);
+                if (i > 0 && i % interval === 0 && randomIdx < randomSlice.length) {
+                    const r = randomSlice[randomIdx++];
+                    if (r)
+                        withRandom.push(r);
+                }
+            }
+            // Append any remaining random members not yet inserted.
+            while (randomIdx < randomSlice.length) {
+                const r = randomSlice[randomIdx++];
+                if (r)
+                    withRandom.push(r);
+            }
+            finalClusteredMemories = withRandom;
+            warnings.push(`Anti-collapse: injected ${randomCount} random (non-similarity-driven) cluster member(s) into consolidation pool (fraction=${fraction}).`);
+        }
+    }
     const chunks = [];
-    for (let i = 0; i < clusteredMemories.length; i += chunkSize) {
-        chunks.push(clusteredMemories.slice(i, i + chunkSize));
+    for (let i = 0; i < finalClusteredMemories.length; i += chunkSize) {
+        chunks.push(finalClusteredMemories.slice(i, i + chunkSize));
     }
     // 2026-05-27 prompt-context fix: precompute body-hashes of pending
     // consolidate proposals once, so the per-chunk prompt can annotate
@@ -890,6 +1210,40 @@ export async function akmConsolidate(opts = {}) {
     // 4h on this user's stack. See
     // /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
     const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
+    // ── Cold-start budget estimation ─────────────────────────────────────────────
+    // Estimate wall-clock cost BEFORE issuing any LLM calls. When a signal is
+    // provided and the estimated cost exceeds ~60% of the remaining budget we
+    // auto-reduce the pool and log the reduction so the run never starts work
+    // it cannot finish (avoiding SIGTERM mid-LLM-call).
+    //
+    // Formula: chunks.length × p90_chunk_seconds. The p90 comes from
+    // `opts.p90ChunkSecondsDefault` (caller-supplied, typically from the profile
+    // config); absent = 30 s (conservative default matching a medium local LLM).
+    //
+    // "Remaining budget" is read from a custom property on the AbortSignal if
+    // the caller (improve.ts) has attached one. Without it no auto-reduction
+    // fires but the check is still cheap to run.
+    if (chunks.length > 10 && opts.signal) {
+        const p90Chunk = opts.p90ChunkSecondsDefault ?? 30;
+        const estimatedSeconds = chunks.length * p90Chunk;
+        // remainingBudgetMs is a non-standard extension set by improve.ts when it
+        // creates the budget AbortController. Undefined = no budget information.
+        const budgetMs = opts.signal.remainingBudgetMs;
+        if (budgetMs !== undefined && budgetMs > 0) {
+            const remainingSeconds = budgetMs / 1000;
+            if (estimatedSeconds > remainingSeconds * 0.6) {
+                const safeCaps = Math.max(1, Math.floor((remainingSeconds * 0.6) / p90Chunk));
+                const removedChunks = chunks.length - safeCaps;
+                if (removedChunks > 0) {
+                    const msg = `[consolidate] cold-start budget: estimated ${estimatedSeconds.toFixed(0)}s > 60% of remaining ${remainingSeconds.toFixed(0)}s; ` +
+                        `reducing pool from ${chunks.length} to ${safeCaps} chunks (${removedChunks} deferred to next run).`;
+                    warn(msg);
+                    warnings.push(msg);
+                    chunks.splice(safeCaps);
+                }
+            }
+        }
+    }
     warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
         ` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
     const chunkOpsArrays = [];
@@ -928,6 +1282,13 @@ export async function akmConsolidate(opts = {}) {
     // judgedNoAction tracks memories the LLM saw inside a chunk but proposed
     // no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
     let judgedNoAction = 0;
+    // Judged-state cache (#581): coarse outcome per memory NAME the LLM actually
+    // judged in a successfully-parsed chunk this run. "actioned" = an op targeted
+    // it; "no_action" = the LLM saw it and proposed nothing. Populated only when
+    // the cache is enabled (otherwise it stays empty and the post-loop recording
+    // step is a no-op). Memories in failed/aborted chunks are NOT recorded, so a
+    // transient LLM failure never poisons the cache into skipping them next run.
+    const judgedOutcomeByName = new Map();
     // 2026-05-27 cross-chunk double-count fix: refs that contributed to
     // judgedNoAction in their own chunk. When a different chunk's op references
     // one of these as a secondary and that op later fails, the ref would land
@@ -952,6 +1313,19 @@ export async function akmConsolidate(opts = {}) {
     const ABORT_MIN_CHUNKS = 4;
     const ABORT_FAILURE_RATE = 0.5;
     for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
+        // Budget-signal check: break cleanly before the next LLM call if the
+        // caller's budget has been exhausted. Commits work done so far.
+        if (opts.signal?.aborted) {
+            const skipped = chunks.length - chunkIdx;
+            const msg = `[consolidate] budget signal aborted before chunk ${chunkIdx + 1}/${chunks.length}; ${skipped} chunk(s) not processed (partial_timeout — work done so far committed).`;
+            warn(msg);
+            warnings.push(msg);
+            // Account for memories in unprocessed chunks.
+            for (let i = chunkIdx; i < chunks.length; i++) {
+                failedChunkMemories += chunks[i].length;
+            }
+            break;
+        }
         // Abort if failure rate >= 50% over at least 4 processed chunks.
         if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
             const failureRate = totalChunksFailed / totalChunksProcessed;
@@ -1096,11 +1470,64 @@ export async function akmConsolidate(opts = {}) {
             if (!targetRefs.has(memRef)) {
                 chunkNoAction++;
                 judgedNoActionRefs.add(memRef);
+                // Judged-state cache (#581): the LLM saw this memory and proposed
+                // nothing → record judged-unchanged so the next run can skip it.
+                if (judgedCacheEnabled)
+                    judgedOutcomeByName.set(m.name, "no_action");
+            }
+            else if (judgedCacheEnabled) {
+                // An op targeted this memory → it was judged + actioned.
+                judgedOutcomeByName.set(m.name, "actioned");
             }
         }
         judgedNoAction += chunkNoAction;
         chunkOpsArrays.push(ops);
     }
+    // ── Judged-state cache recording (#581) ─────────────────────────────────────
+    // Persist judged state for every memory the LLM actually judged this run so
+    // the next run can skip the unchanged ones. Keyed by current content hash so
+    // a later body edit (different hash) re-enters the LLM pool. DEFAULT OFF and
+    // skipped under --dry-run (dry-run mutates nothing). Failed/aborted chunks
+    // contributed no entries to `judgedOutcomeByName`, so a transient LLM outage
+    // never caches a memory as judged.
+    if (judgedCacheEnabled && !opts.dryRun && judgedOutcomeByName.size > 0) {
+        // Use the shared state.db handle; open a local one as fallback.
+        const doRecord = (db) => {
+            const judgedAt = new Date(startMs).toISOString();
+            for (const [name, outcome] of judgedOutcomeByName) {
+                const hash = currentHashByName.get(name);
+                if (hash === undefined)
+                    continue;
+                upsertConsolidationJudged(db, {
+                    entryKey: `memory:${name}`,
+                    contentHash: hash,
+                    judgedAt,
+                    outcome,
+                });
+            }
+        };
+        if (sharedStateDb) {
+            try {
+                doRecord(sharedStateDb);
+            }
+            catch (e) {
+                warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
+            }
+        }
+        else {
+            let localDb;
+            try {
+                localDb = openStateDatabase();
+                doRecord(localDb);
+            }
+            catch (e) {
+                warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
+            }
+            finally {
+                localDb?.close();
+            }
+        }
+    }
     // Build the known-refs set from the already-filtered memory pool so
     // mergePlans() can reject LLM-hallucinated primary refs before execution.
     const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
@@ -1287,7 +1714,7 @@ export async function akmConsolidate(opts = {}) {
                 emitMergeFailureSkips(mergeResult.error);
                 continue;
             }
-            const mergedContent = mergeResult.content;
+            let mergedContent = mergeResult.content;
             // Validate frontmatter of merged content — must have a `---` block
             // with at minimum a `description` field. We parse via the hand-rolled
             // parser (cheap) AND require non-empty description. This guards against
@@ -1341,6 +1768,62 @@ export async function akmConsolidate(opts = {}) {
                 emitMergeFailureSkips("merge_participant_blocked");
                 continue;
             }
+            // WS-3b: Anti-collapse generation guard (step 8a).
+            // DEFAULT OFF. When antiCollapse.enabled, refuse to merge two assets both
+            // above generation N (default 2). This prevents the pipeline from
+            // building ever-deeper LLM-merged trees that lose the source fidelity
+            // of the original episodes.
+            const antiCollapseConfig = config.profiles?.improve?.default?.processes?.consolidate?.antiCollapse ??
+                {};
+            if (antiCollapseConfig.enabled) {
+                const allParticipants = [op.primary, ...op.secondaries];
+                const sourceGenerations = allParticipants.map((ref) => {
+                    const e = memoryByRef.get(ref);
+                    if (!e)
+                        return 0;
+                    try {
+                        const raw = fs.readFileSync(e.filePath, "utf8");
+                        const parsed = parseFrontmatter(raw);
+                        return readAssetGeneration(parsed.data);
+                    }
+                    catch {
+                        return 0;
+                    }
+                });
+                const generationCheck = checkGenerationGuard(sourceGenerations, antiCollapseConfig);
+                if (generationCheck.refused) {
+                    warnings.push(`Merge: ${generationCheck.reason}`);
+                    emitMergeFailureSkips("merge_generation_guard");
+                    continue;
+                }
+                // WS-3b: Lexical diversity check (step 8b).
+                // Low n-gram diversity ⇒ likely correlated-extraction artifact; raise merge threshold.
+                if (antiCollapseConfig.lexicalDiversityCheck !== false) {
+                    const bodies = allParticipants
+                        .map((ref) => {
+                        const e = memoryByRef.get(ref);
+                        if (!e)
+                            return "";
+                        try {
+                            const raw = fs.readFileSync(e.filePath, "utf8");
+                            return stripFrontmatterBody(raw);
+                        }
+                        catch {
+                            return "";
+                        }
+                    })
+                        .filter((b) => b.length > 0);
+                    const diversityCheck = checkLexicalDiversity(bodies, antiCollapseConfig);
+                    if (diversityCheck.lowDiversity) {
+                        // Low-diversity cluster: just warn (don't refuse merge since the dedup
+                        // path handles exact twins). The warning surfaces in health telemetry.
+                        warnings.push(`Merge: cluster around ${op.primary} has low lexical diversity (${diversityCheck.diversity?.toFixed(2) ?? "?"} < 0.30) — likely correlated extraction; merge proceeds but review is recommended.`);
+                    }
+                }
+                // Inject generation counter into merged content frontmatter (step 8a).
+                // merged.generation = max(sourceGenerations) + 1.
+                mergedContent = injectGenerationFrontmatter(mergedContent, sourceGenerations, allParticipants);
+            }
             // Backup secondaries before deleting
             for (const secRef of op.secondaries) {
                 const secEntry = memoryByRef.get(secRef);
@@ -1543,11 +2026,12 @@ export async function akmConsolidate(opts = {}) {
             //       is the load-bearing content, so dedup must hash on body only.
             //   (b) A prior run created a proposal for the same body under a
             //       different knowledgeRef slug.
-            const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
+            // Use cacheHash (case-preserving stripped body) to match the canonical
+            // hash domain used by the body-embedding cache and pending-proposal set.
+            const bodyHash = cacheHash(sourceBody);
             const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
             const contentDupProposal = allPendingConsolidateProposals.find((p) => {
-                const otherBody = parseFrontmatter(p.payload.content).content.trim();
-                return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
+                return cacheHash(p.payload.content) === bodyHash;
             });
             if (contentDupProposal) {
                 warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
@@ -1629,6 +2113,18 @@ export async function akmConsolidate(opts = {}) {
             }
         }
         else if (op.op === "contradict") {
+            // Confidence gate: surface-level topic overlap causes false positives
+            // (investigation 2026-06-18). Require ≥0.92 confidence before writing
+            // contradiction edges. Missing confidence field defaults to 1.0 for
+            // backward compatibility with responses that predate this field.
+            const opConfidence = typeof op.confidence === "number"
+                ? op.confidence
+                : 1.0;
+            if (opConfidence < 0.92) {
+                warnings.push(`Contradict: confidence ${opConfidence.toFixed(2)} below 0.92 threshold for ${op.ref} <-> ${op.contradictedByRef} — skipping.`);
+                pushSkipReason("contradict", op.ref, "contradict_low_confidence");
+                continue;
+            }
             // C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
             // (the SCC resolver in memory-improve.ts) has edges to work on.
             // Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
@@ -1668,37 +2164,19 @@ export async function akmConsolidate(opts = {}) {
         commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
     }
     cleanupJournal(stashDir, timestamp);
-    // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
-    // C-5 / #391: emit an `archive_cleanup` event before each deletion so the
-    // audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf) —
-    // any event that is recorded must be queryable; silent deletes are an anti-pattern.
-    const archiveDir = path.join(stashDir, ".akm", "archive");
-    if (fs.existsSync(archiveDir)) {
-        const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
-        const cutoff = Date.now() - retentionMs;
-        for (const fname of fs.readdirSync(archiveDir)) {
-            const fp = path.join(archiveDir, fname);
-            try {
-                const stat = fs.statSync(fp);
-                if (stat.mtimeMs < cutoff) {
-                    // Emit event before deletion so the record survives the purge.
-                    appendEvent({
-                        eventType: "archive_cleanup",
-                        metadata: {
-                            file: fname,
-                            filePath: fp,
-                            ageMs: Date.now() - stat.mtimeMs,
-                            retentionMs,
-                        },
-                    });
-                    fs.unlinkSync(fp);
-                }
-            }
-            catch {
-                /* ignore race conditions */
-            }
-        }
-    }
+    // [signoff 2026-06-15] TTL archive cleanup machinery RETIRED (WS-3a).
+    // The elaborate archiveRetentionDays / archive-dir scan existed only to satisfy
+    // the old irrecoverability constraint. Stashes are now git-backed, so git
+    // history is the recovery path — no bespoke archive TTL needed. Any files in
+    // .akm/archive/ will stay there harmlessly until the operator prunes them with
+    // `git rm` or `find .akm/archive -mtime +90 -delete`. Changed N files this
+    // run; recover any via `git show <sha>:<path>` or `git restore <path>`.
+    if (merged > 0 || deleted > 0 || dedupCollapsed > 0) {
+        const totalChanged = merged + deleted + dedupCollapsed;
+        warnings.push(`Changed ${totalChanged} file(s) this run. Recover any via git if needed (git history is the backstop).`);
+    }
+    const runDurationMs = Date.now() - startMs;
+    const budgetFraction = opts.runBudgetMs !== undefined && opts.runBudgetMs > 0 ? runDurationMs / opts.runBudgetMs : undefined;
     return {
         schemaVersion: 1,
         ok: true,
@@ -1708,7 +2186,10 @@ export async function akmConsolidate(opts = {}) {
         target: sourceName,
         processed: memories.length,
         merged,
-        deleted,
+        // #617: fold the deterministic dedup pre-pass collapses into the reported
+        // deleted count. Each collapse removed exactly one variant file with NO
+        // LLM call before the LLM pass ran on the pruned pool.
+        deleted: deleted + dedupCollapsed,
         promoted,
         contradicted,
         failedChunks: totalChunksFailed,
@@ -1718,7 +2199,16 @@ export async function akmConsolidate(opts = {}) {
         mergedSecondaries,
         failedChunkMemories,
         warnings,
-        durationMs: Date.now() - startMs,
+        durationMs: runDurationMs,
+        perfTelemetry: {
+            dedupPoolSize: perfMs.dedupPoolSize,
+            llmPoolSize,
+            judgedCacheSkipped: perfMs.judgedCacheSkipped,
+            embedMs: embedTelemetry.embedMs,
+            embedCacheHits: embedTelemetry.cacheHits,
+            embedCacheMisses: embedTelemetry.cacheMisses,
+            ...(budgetFraction !== undefined ? { estimatedBudgetFractionUsed: budgetFraction } : {}),
+        },
     };
 }
 // ── Helpers ─────────────────────────────────────────────────────────────────
@@ -2251,8 +2741,8 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
             }
             normalizeUpdatedField(repairedFmData);
             const repairedYaml = serializeFrontmatter(repairedFmData);
-            const bodyPart = mergedFm.content ?? "";
-            return { content: `---\n${repairedYaml}\n---\n${bodyPart}` };
+            const bodyPart = typeof mergedFm.content === "string" ? mergedFm.content : "";
+            return { content: assembleAssetFromString(repairedYaml, bodyPart) };
         }
     }
     catch {