akm-cli 0.9.0-rc.1 → 0.9.0-rc.13

package/dist/commands/improve/collapse-detector.js

@@ -2,21 +2,27 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * R5 — Longitudinal collapse/churn detector
- * (docs/design/improve-collapse-churn-detector-design.md).
+ * R5 — Longitudinal collapse detector
+ * (docs/architecture/specs/improve-collapse-churn-detector-design.md).
  *
- * Detects the two measured failure modes of LLM-consolidated memory stores:
+ * Detects the measured collapse failure mode of LLM-consolidated memory
+ * stores: repeated merges destroy information — canary retrieval recall
+ * downtrends, distinct-content entropy downtrends, or the store shrinks
+ * while generation counts rise.
  *
- *   COLLAPSE — repeated merges destroy information: canary retrieval recall
- *     downtrends, distinct-content entropy downtrends, or the store shrinks
- *     while generation counts rise.
- *   CHURN — real accepted-change volume with zero retrieval-visible or
- *     shape-visible movement (LLM budget burned rewriting to no effect).
+ * The CHURN alert class described in the design doc was removed: production
+ * never threaded a real accepted-change-volume signal in (the value was a
+ * hardcoded 0 from the 0.9.0 confidence-gate deletion onward), so the alert
+ * could never fire. See the design doc's status note for the removal
+ * rationale. `improve_cycle_metrics.accepted_actions` stays in the schema and
+ * is written as 0: its migration body is released, and 0.8 ships `state.db`,
+ * so a deployed ledger may already have sealed that body's checksum. Editing
+ * it would fail those installs closed.
  *
  * Hard invariants: deterministic only (FTS BM25 + hashing — never an LLM,
  * never an embedding model); bounded storage (< 2 KB per qualifying cycle,
  * 365-day retention); fail-open (an error warns and skips, never breaks an
- * improve run); runs only on cycles where consolidate/recombine did work.
+ * improve run); runs only on cycles where consolidate did work.
  *
  * Observe-only in v1: alerts land in `improve_cycle_metrics.alerts_json`, the
  * events log (`collapse_detector_alert`), and the `akm health` advisory —
@@ -25,13 +31,16 @@
  * @module collapse-detector
  */
 import { randomBytes } from "node:crypto";
-import { makeAssetRef } from "../../core/asset/asset-ref.js";
+import { parseBundleRef } from "../../core/asset/asset-ref.js";
+import { conceptIdFromTypeName } from "../../core/asset/resolve-ref.js";
 import { getImproveProcessConfig } from "../../core/config/config.js";
 import { appendEvent } from "../../core/events.js";
 import { withStateDb } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
-import { closeDatabase, getAllEntries, openExistingDatabase, searchFts, } from "../../indexer/db/db.js";
 import { deactivateCanarySet, getActiveCanaries, getCanariesBySetId, insertCanaries, insertCycleMetrics, listActiveCanarySetIds, queryRecentCycleMetrics, } from "../../storage/repositories/canaries-repository.js";
+import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
+import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
+import { searchFts } from "../../storage/repositories/index-fts-repository.js";
 import { computeBigramDiversity, DEFAULT_MAX_GENERATION } from "./anti-collapse.js";
 import { getAllRankScores } from "./salience.js";
 // ── Defaults (mirrored in config-schema.ts ImproveCollapseDetectorSchema) ────
@@ -40,7 +49,6 @@ export const DEFAULT_CANARY_K = 10;
 export const DEFAULT_WINDOW_CYCLES = 5;
 export const DEFAULT_RECALL_DROP_THRESHOLD = 0.15;
 export const DEFAULT_ENTROPY_DROP_THRESHOLD = 0.05;
-export const DEFAULT_CHURN_MIN_ACCEPTED = 25;
 export const DEFAULT_RETENTION_DAYS = 365;
 /** Deterministic bigram-diversity sample cap (cost bound at 10k assets). */
 const DIVERSITY_SAMPLE_CAP = 2000;
@@ -66,12 +74,13 @@ function buildCanaryQuery(entry) {
 function buildMintList(stateDb, entries, cfg) {
     const canaryCount = cfg.canaryCount ?? DEFAULT_CANARY_COUNT;
     const rankScores = getAllRankScores(stateDb);
-    // NOTE: entryKey is stash-prefixed ("<stashDir>:type:name"); asset_salience
-    // and the canary scoring both key on the bare "type:name" ref.
+    // Chunk-8 WI-8.5c: the canary anchor + salience correlation both key on the
+    // SHORT conceptId (`<stash-subdir>/<name>`, D-R2) — the same spelling the
+    // improve candidate refs and content xrefs now carry (post WI-8.5a).
     const candidates = entries
         .filter((e) => LEARNING_TYPES.has(e.entry.type))
         .map((e) => {
-        const ref = makeAssetRef(e.entry.type, e.entry.name);
+        const ref = conceptIdFromTypeName(e.entry.type, e.entry.name);
         return { e, ref, score: rankScores.get(ref) ?? 0 };
     })
         .sort((a, b) => b.score - a.score || (a.ref < b.ref ? -1 : 1));
@@ -114,9 +123,10 @@ function newCanarySetId() {
  * mean the trend window never fills and recall reads as a fake 0).
  *
  * NEVER auto-refreshes: once minted the set is frozen until an explicit
- * `akm improve canary --refresh` — silent re-baselining is how a slow collapse
- * hides. Rows are read back BY OUR OWN set id (never "newest active") so a
- * concurrent mint in another process cannot relabel this run's metrics.
+ * `bun scripts/refresh-canary-set.ts --refresh` — silent re-baselining is how
+ * a slow collapse hides. Rows are read back BY OUR OWN set id (never "newest
+ * active") so a concurrent mint in another process cannot relabel this run's
+ * metrics.
  */
 export function ensureCanarySet(stateDb, indexDb, cfg, preloadedEntries) {
     const existing = getActiveCanaries(stateDb);
@@ -131,11 +141,11 @@ export function ensureCanarySet(stateDb, indexDb, cfg, preloadedEntries) {
     return { canarySetId, canaries: getCanariesBySetId(stateDb, canarySetId) };
 }
 /**
- * Explicit canary re-mint (the ONLY refresh path — `akm improve canary
- * --refresh`). Mint-first, deactivate-after: when the index is empty or
- * unreadable the current baseline is left untouched instead of destroyed.
- * Deactivates ALL other active sets (not just the newest) so stragglers from
- * an interrupted refresh can never resurrect.
+ * Explicit canary re-mint (the ONLY refresh path —
+ * `bun scripts/refresh-canary-set.ts --refresh`). Mint-first, deactivate-after:
+ * when the index is empty or unreadable the current baseline is left untouched
+ * instead of destroyed. Deactivates ALL other active sets (not just the
+ * newest) so stragglers from an interrupted refresh can never resurrect.
  */
 export function refreshCanarySet(stateDb, indexDb, cfg) {
     const minted = buildMintList(stateDb, getAllEntries(indexDb), cfg);
@@ -175,24 +185,36 @@ export function normHash(text) {
     return hash.toString(16);
 }
 /**
- * Score one canary against the live index, merge-following via `source_refs`:
- * a hit is the anchor ref itself OR any returned entry whose `source_refs`
- * frontmatter contains the anchor (ONE level — provenance dropped on a
- * second-generation merge is a miss by design; that IS the information loss).
+ * Score one canary against the live index, merge-following via canonical
+ * `xrefs`. A hit is the anchor ref itself or any returned entry whose
+ * provenance contains the anchor.
  * Returns the 0-based rank of the first hit, or -1.
  */
 function scoreCanary(indexDb, canary, k) {
     const results = searchFts(indexDb, canary.query, k);
+    const anchorConceptId = canaryConceptId(canary.anchor_ref);
     for (let i = 0; i < Math.min(results.length, k); i++) {
         const r = results[i];
-        const ref = makeAssetRef(r.entry.type, r.entry.name);
-        if (ref === canary.anchor_ref)
+        // Persisted canaries may be fully qualified after a state migration while
+        // index refs and xrefs remain short conceptIds. Compare identity, not display
+        // spelling, so migration alone cannot collapse recall to zero.
+        const ref = conceptIdFromTypeName(r.entry.type, r.entry.name);
+        if (ref === anchorConceptId)
             return i;
-        if (r.entry.sourceRefs?.includes(canary.anchor_ref))
+        const provenance = r.entry.xrefs ?? [];
+        if (provenance.some((sourceRef) => canaryConceptId(sourceRef) === anchorConceptId))
             return i;
     }
     return -1;
 }
+function canaryConceptId(ref) {
+    try {
+        return parseBundleRef(ref).conceptId;
+    }
+    catch {
+        return ref;
+    }
+}
 /**
  * Compute one qualifying cycle's store-health snapshot. One `entries` scan +
  * `canaryCount` FTS queries; no LLM, no embedding model, no filesystem reads.
@@ -266,13 +288,17 @@ export function computeCycleMetrics(stateDb, indexDb, args) {
         distinct_content_ratio: learningTotal === 0 ? 1 : contentHashes.size / learningTotal,
         mean_bigram_diversity: diversityCount === 0 ? 1 : diversitySum / diversityCount,
         over_generation_count: overGeneration,
-        accepted_actions: args.acceptedActions,
+        // Always 0 — the CHURN alert this fed was removed. The column is NOT NULL
+        // in a released migration body, so it is written rather than dropped.
+        accepted_actions: 0,
         merge_floor_violations: args.mergeFloorViolations,
         alerts_json: "[]",
     };
 }
 // ── Alert evaluation (pure) ───────────────────────────────────────────────────
 function median(values) {
+    if (values.length === 0)
+        return Number.NaN;
     const sorted = [...values].sort((a, b) => a - b);
     const mid = Math.floor(sorted.length / 2);
     return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
@@ -299,9 +325,11 @@ export function evaluateCollapseAlerts(history, current, cfg) {
     const hist = history.slice(-W);
     if (hist.length < W)
         return alerts; // no baseline yet
+    // Window is non-empty here: `windowCycles` is schema-bounded to ≥2 (default 5),
+    // so `hist.length >= W >= 2`. `windowStart` is the oldest row in the window.
+    const windowStart = hist[0];
     const recallDrop = cfg.recallDropThreshold ?? DEFAULT_RECALL_DROP_THRESHOLD;
     const entropyDrop = cfg.entropyDropThreshold ?? DEFAULT_ENTROPY_DROP_THRESHOLD;
-    const churnMin = cfg.churnMinAcceptedActions ?? DEFAULT_CHURN_MIN_ACCEPTED;
     // COLLAPSE 1 — canary recall drop vs window median (median, not previous
     // cycle, so one noisy cycle can neither fire nor mask the alert).
     const medianRecall = median(hist.map((h) => h.mean_recall));
@@ -315,13 +343,13 @@ export function evaluateCollapseAlerts(history, current, cfg) {
     // COLLAPSE 2 — monotonic distinct-content-ratio decline over the window.
     const series = [...hist.map((h) => h.distinct_content_ratio), current.distinct_content_ratio];
     const monotonicNonIncreasing = series.every((v, i) => i === 0 || v <= series[i - 1]);
-    const totalDecline = hist[0].distinct_content_ratio - current.distinct_content_ratio;
+    const totalDecline = windowStart.distinct_content_ratio - current.distinct_content_ratio;
     if (monotonicNonIncreasing && totalDecline >= entropyDrop) {
         alerts.push({
             kind: "collapse-entropy",
             detail: `distinct-content ratio declined monotonically by ${totalDecline.toFixed(3)} (≥${entropyDrop}) over ${W} cycles — store content is converging`,
             metrics: {
-                windowStart: hist[0].distinct_content_ratio,
+                windowStart: windowStart.distinct_content_ratio,
                 current: current.distinct_content_ratio,
                 decline: totalDecline,
             },
@@ -329,10 +357,10 @@ export function evaluateCollapseAlerts(history, current, cfg) {
     }
     // COLLAPSE 3 — store shrinking BECAUSE of re-merging (not deletion hygiene).
     const maxStore = Math.max(...hist.map((h) => h.store_total));
-    if (current.store_total < 0.8 * maxStore && current.over_generation_count > hist[0].over_generation_count) {
+    if (current.store_total < 0.8 * maxStore && current.over_generation_count > windowStart.over_generation_count) {
         alerts.push({
             kind: "collapse-shrink",
-            detail: `store shrank >20% (${current.store_total} vs window max ${maxStore}) while over-generation count rose (${hist[0].over_generation_count} → ${current.over_generation_count})`,
+            detail: `store shrank >20% (${current.store_total} vs window max ${maxStore}) while over-generation count rose (${windowStart.over_generation_count} → ${current.over_generation_count})`,
             metrics: {
                 storeTotal: current.store_total,
                 windowMax: maxStore,
@@ -340,20 +368,6 @@ export function evaluateCollapseAlerts(history, current, cfg) {
             },
         });
     }
-    // CHURN — real write volume, zero retrieval- or shape-visible movement.
-    // Flatness is measured against the window MEDIAN (consistent with the
-    // recall rule): endpoint-only comparison would call a window that swung
-    // wildly but happened to land near its start "flat".
-    const acceptedSum = hist.reduce((a, h) => a + h.accepted_actions, 0);
-    const scoreFlat = Math.abs(current.mean_ndcg - median(hist.map((h) => h.mean_ndcg))) < 0.02;
-    const entropyFlat = Math.abs(current.distinct_content_ratio - median(hist.map((h) => h.distinct_content_ratio))) < 0.02;
-    if (acceptedSum >= churnMin && scoreFlat && entropyFlat) {
-        alerts.push({
-            kind: "churn",
-            detail: `${acceptedSum} accepted actions over ${W} cycles with flat canary score and flat entropy — write volume with no retrieval-visible effect`,
-            metrics: { acceptedSum, ndcgDelta: current.mean_ndcg - hist[0].mean_ndcg },
-        });
-    }
     return alerts;
 }
 // ── Orchestrator ─────────────────────────────────────────────────────────────
@@ -375,13 +389,12 @@ export function runCollapseDetector(args) {
             const db = indexDb;
             // Over-generation threshold mirrors the guard actually in effect —
             // reading the same config key keeps the two aligned when tuned.
-            const antiCollapse = getImproveProcessConfig(args.config, "consolidate", args.improveProfile)?.antiCollapse;
+            const antiCollapse = getImproveProcessConfig("consolidate", args.improveProfile)?.antiCollapse;
             const maxGeneration = antiCollapse?.maxGeneration ?? DEFAULT_MAX_GENERATION;
             return withStateDb((stateDb) => {
                 const row = computeCycleMetrics(stateDb, db, {
                     runId: args.runId,
                     pass: args.pass,
-                    acceptedActions: args.acceptedActions,
                     mergeFloorViolations: args.mergeFloorViolations,
                     cfg,
                     maxGeneration,

package/dist/commands/improve/consolidate/chunking.js

@@ -6,7 +6,7 @@
 // no orchestrator coupling.
 import fs from "node:fs";
 import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
-import { cacheHash } from "../dedup.js";
+import { cacheHash } from "../content-hash.js";
 /**
  * Conservative chars-per-token estimate used when computing prompt budgets.
  * English text averages roughly 4 chars/token for most LLM tokenizers. We use
@@ -33,9 +33,9 @@ const PROMPT_OVERHEAD_TOKENS = 2_000;
  *     chunk content.
  *   - When the HTTP path is used (an LLM profile is selected), only the akm
  *     system prompt and user prompt are sent, so the budget can be set to the
- *     model's actual context length via profiles.llm[defaults.llm].contextLength.
+ *     model's actual context length via the selected LLM engine's contextLength.
  *
- * Set profiles.llm[defaults.llm].contextLength in your config file to the
+ * Set the selected LLM engine's contextLength in your config file to the
  * model's actual context window to allow larger chunks on the HTTP path.
  */
 export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
@@ -78,8 +78,9 @@ export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize
  * so the cost stays O(memories) inside the chunk loop.
  */
 export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set(), standardsContext = "") {
-    const start = memories[0] ? `memory:${memories[0].name}` : "";
-    const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
+    const start = memories[0] ? `memories/${memories[0].name}` : "";
+    const lastMemory = memories[memories.length - 1];
+    const end = lastMemory ? `memories/${lastMemory.name}` : "";
     const annotationsByIndex = [];
     const hotRefs = [];
     for (const m of memories) {
@@ -98,7 +99,7 @@ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks,
         const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
         annotationsByIndex.push({ isHot, isAlreadyQueued, body });
         if (isHot)
-            hotRefs.push(`memory:${m.name}`);
+            hotRefs.push(`memories/${m.name}`);
     }
     const lines = [
         `Source: ${sourceName}`,
@@ -123,6 +124,7 @@ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks,
     }
     for (let i = 0; i < memories.length; i++) {
         const m = memories[i];
+        // `annotationsByIndex` has exactly one entry per memory (built in the loop above).
         const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
         const annotations = [];
         if (isHot)
@@ -130,7 +132,7 @@ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks,
         if (isAlreadyQueued)
             annotations.push("already queued");
         const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
-        lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
+        lines.push(`[${i + 1}] memories/${m.name}${annotationSuffix}`);
         lines.push(`Description: ${m.description || "(none)"}`);
         lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
         lines.push("---");

package/dist/commands/improve/consolidate/eligibility.js

@@ -22,7 +22,7 @@ export function isConsolidationEligibleMemoryName(name) {
  * consolidate flow already guards against unparseable memories elsewhere).
  *
  * Defends against four observed defect classes (see
- * `memory:akm-improve-critical-review-2026-05-20`):
+ * `memories/akm-improve-critical-review-2026-05-20`):
  *   - LLM marks a memory contradicted then deletes (dangling contradictedBy)
  *   - LLM merges two unrelated memories sharing a topic keyword
  *   - LLM judges a recent durable design memo as "redundant"
@@ -40,25 +40,3 @@ export function isHotCapturedMemory(filePath) {
         return false;
     }
 }
-export function consolidateGuardStatus(filePath) {
-    if (!fs.existsSync(filePath))
-        return "missing";
-    let content;
-    try {
-        content = fs.readFileSync(filePath, "utf8");
-    }
-    catch {
-        return "unparseable";
-    }
-    let parsed;
-    try {
-        parsed = parseFrontmatter(content);
-    }
-    catch {
-        return "unparseable";
-    }
-    const data = parsed.data;
-    if (!data || Object.keys(data).length === 0)
-        return "unparseable";
-    return hasHotCaptureMode(data) ? "hot" : "safe";
-}

package/dist/commands/improve/consolidate/merge.js

@@ -56,6 +56,10 @@ export function mergePlans(chunks, knownRefs) {
                         mergeOp = { ...op, secondaries: filteredSecondaries };
                     }
                 }
+                if (mergeOp.secondaries.length > 1) {
+                    warnings.push(`Merge: ${mergeOp.primary} proposed multiple secondaries; retaining one secondary for review.`);
+                    mergeOp = { ...mergeOp, secondaries: mergeOp.secondaries.slice(0, 1) };
+                }
                 // merge wins over delete
                 if (deleteOps.has(mergeOp.primary)) {
                     deleteOps.delete(mergeOp.primary);