akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/indexer/passes/staleness-detect.js

@@ -1,488 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * Staleness-detection pass for `akm improve` (Phase 4A,
- * `.plans/0.8.0/self-improvement-enhancements-plan.md` lines 132-145).
- *
- * Activates the `deprecated` belief-state machinery shipped in Phase 1A
- * (commit 7b6fffe). Without this pass, nothing in the pipeline ever wrote
- * `beliefState: deprecated`, so the -0.15 ranking penalty and the
- * `matchBeliefFilter("historical")` inclusion were dormant.
- *
- * Pipeline
- * --------
- *   1. Walk every memory under `<stash>/memories/` and select candidates
- *      whose belief state is NOT already excluded
- *      ({contradicted, archived, deprecated}) AND whose `lastConfirmedAt`
- *      is absent or older than the configured threshold (default 90 days).
- *      Files without `lastConfirmedAt` fall back to file `mtime`.
- *   2. For each candidate, ask the configured validation-tier LLM
- *      (`resolveValidationRunner`) whether the candidate is still current
- *      given the top-K most-similar memories from the stash.
- *      Strict response contract: `YES\nSUPERSEDED_BY: <ref>` or `NO`.
- *      Anything else is treated as a parse error and the candidate is skipped.
- *   3. YES → write `beliefState: "deprecated"`, `supersededBy: [<ref>]`,
- *      `lastConfirmedAt: <now>`. The supersededBy ref MUST exist in the
- *      stash (DB lookup); if it doesn't, treat as NO.
- *      NO → write only `lastConfirmedAt: <now>` (refreshes the staleness
- *      window). All other frontmatter fields stay untouched.
- *
- * Caching
- * -------
- * Uses the standard `withLlmCache` wrapper with cacheVariant
- * `"staleness_detect"`, so re-running the pass on an unchanged file is a
- * no-op (no LLM call).
- *
- * Feature gate
- * ------------
- * Default OFF. Enable via `features.index.staleness_detection.enabled` (or
- * the boolean shorthand `features.index.staleness_detection = true`).
- * Threshold-days knob lives at
- * `features.index.staleness_detection.options.thresholdDays` (default 90).
- */
-import { createHash } from "node:crypto";
-import fs from "node:fs";
-import path from "node:path";
-import { assembleAsset } from "../../core/asset/asset-serialize.js";
-import { parseFrontmatter, parseFrontmatterBlock } from "../../core/asset/frontmatter.js";
-import { concurrentMap } from "../../core/concurrent.js";
-import { warn } from "../../core/warn.js";
-import { resolveValidationRunner, runnerIsLlm } from "../../integrations/agent/runner.js";
-import { chatCompletion } from "../../llm/client.js";
-import { isProcessEnabled } from "../../llm/feature-gate.js";
-import { findEntryIdByRef } from "../db/db.js";
-import { withLlmCache } from "../db/llm-cache.js";
-import { walkMarkdownFiles } from "../walk/walker.js";
-/** Frontmatter keys this pass touches. Constants so a future rename only needs to touch one site. */
-const FM_BELIEF_STATE = "beliefState";
-const FM_SUPERSEDED_BY = "supersededBy";
-const FM_LAST_CONFIRMED_AT = "lastConfirmedAt";
-/** Cache variant for `withLlmCache`. Keeps staleness results isolated from memory-inference cache rows. */
-const CACHE_VARIANT = "staleness_detect";
-/** Belief states excluded from staleness detection — already historical / archived. */
-const EXCLUDED_BELIEF_STATES = new Set(["contradicted", "archived", "deprecated"]);
-/** Default threshold in days before a memory is re-evaluated. */
-const DEFAULT_THRESHOLD_DAYS = 90;
-/** Top-K similar memories included in the LLM prompt. */
-const TOP_K_SIMILAR = 5;
-/**
- * Top-level entry point. Returns a zero-counters result when the feature is
- * disabled or no validation-tier runner is configured.
- */
-export async function runStalenessDetectionPass(ctx) {
-    const { config, sources, signal, db } = ctx;
-    const start = Date.now();
-    const result = {
-        considered: 0,
-        deprecated: 0,
-        confirmed: 0,
-        skipped: 0,
-        durationMs: 0,
-        warnings: [],
-    };
-    // Feature gate — default OFF.
-    if (!isProcessEnabled("index", "staleness_detection", config)) {
-        result.durationMs = Date.now() - start;
-        return result;
-    }
-    // The pass only writes to the primary (writable) stash. Read-only sources
-    // would be clobbered by the next sync(), so we skip them entirely.
-    const primary = sources[0];
-    if (!primary) {
-        result.durationMs = Date.now() - start;
-        return result;
-    }
-    const runner = resolveValidationRunner(config);
-    if (!runner) {
-        result.warnings.push("staleness_detection: no validation runner configured; skipping pass");
-        result.durationMs = Date.now() - start;
-        return result;
-    }
-    if (!runnerIsLlm(runner)) {
-        // MVP scope: only the LLM runner kind is supported. Agent/SDK runners
-        // would require a different prompt-dispatch path that is out of scope
-        // for the initial Phase 4A implementation.
-        result.warnings.push(`staleness_detection: validation runner kind "${runner.kind}" not supported by MVP; configure an llm-kind validation profile`);
-        result.durationMs = Date.now() - start;
-        return result;
-    }
-    const configuredThreshold = config.index?.stalenessDetection?.thresholdDays;
-    const thresholdDays = typeof configuredThreshold === "number" && configuredThreshold >= 0 ? configuredThreshold : DEFAULT_THRESHOLD_DAYS;
-    const thresholdMs = thresholdDays * 24 * 60 * 60 * 1000;
-    const now = Date.now();
-    const candidates = collectStaleCandidates(primary.path, now, thresholdMs);
-    result.considered = candidates.length;
-    if (candidates.length === 0) {
-        result.durationMs = Date.now() - start;
-        return result;
-    }
-    const allMemories = collectAllMemoriesForSimilarity(primary.path);
-    const nowIso = new Date(now).toISOString();
-    const concurrency = runner.connection.concurrency ?? 1;
-    const validate = (raw) => {
-        if (!raw || typeof raw !== "object")
-            return undefined;
-        const r = raw;
-        if (r.decision === "deprecated") {
-            if (typeof r.supersededBy === "string" && r.supersededBy.trim().length > 0) {
-                return { decision: "deprecated", supersededBy: r.supersededBy.trim() };
-            }
-            return undefined;
-        }
-        if (r.decision === "confirmed")
-            return { decision: "confirmed" };
-        return undefined;
-    };
-    const perResult = await concurrentMap(candidates, async (candidate) => {
-        if (signal?.aborted)
-            return undefined;
-        const cacheKey = candidate.filePath;
-        const cacheBody = `${candidate.filePath}\n${candidate.body}`;
-        const decision = db
-            ? await withLlmCache(db, cacheKey, cacheBody, false, () => askValidator(runner.connection, candidate, allMemories, signal, runner.timeoutMs), validate, undefined, CACHE_VARIANT)
-            : await askValidator(runner.connection, candidate, allMemories, signal, runner.timeoutMs);
-        return { candidate, decision };
-    }, concurrency);
-    for (const entry of perResult) {
-        if (!entry)
-            continue;
-        const { candidate, decision } = entry;
-        if (!decision) {
-            result.skipped += 1;
-            continue;
-        }
-        if (decision.decision === "deprecated") {
-            const targetRef = decision.supersededBy ?? "";
-            const validatedRef = validateSupersedingRef(targetRef, primary.path, db);
-            if (!validatedRef) {
-                // Spec line 153: never mark deprecated unless SUPERSEDED_BY exists.
-                // Refresh lastConfirmedAt instead — the candidate is still our best
-                // record until a real superseder shows up — and emit a warning.
-                result.warnings.push(`staleness_detection: ${candidate.ref} reported superseded by "${targetRef}" but that ref does not exist; refreshing instead`);
-                try {
-                    writeLastConfirmed(candidate, nowIso);
-                    result.confirmed += 1;
-                }
-                catch (err) {
-                    result.warnings.push(`staleness_detection: failed to refresh ${candidate.ref}: ${err instanceof Error ? err.message : String(err)}`);
-                    result.skipped += 1;
-                }
-                continue;
-            }
-            try {
-                writeDeprecated(candidate, validatedRef, nowIso);
-                result.deprecated += 1;
-            }
-            catch (err) {
-                result.warnings.push(`staleness_detection: failed to deprecate ${candidate.ref}: ${err instanceof Error ? err.message : String(err)}`);
-                result.skipped += 1;
-            }
-        }
-        else {
-            try {
-                writeLastConfirmed(candidate, nowIso);
-                result.confirmed += 1;
-            }
-            catch (err) {
-                result.warnings.push(`staleness_detection: failed to refresh ${candidate.ref}: ${err instanceof Error ? err.message : String(err)}`);
-                result.skipped += 1;
-            }
-        }
-    }
-    result.durationMs = Date.now() - start;
-    return result;
-}
-function collectStaleCandidates(stashRoot, now, thresholdMs) {
-    const memoriesDir = path.join(stashRoot, "memories");
-    if (!fs.existsSync(memoriesDir))
-        return [];
-    const out = [];
-    for (const filePath of walkMarkdownFiles(memoriesDir)) {
-        let raw;
-        let stat;
-        try {
-            raw = fs.readFileSync(filePath, "utf8");
-            stat = fs.statSync(filePath);
-        }
-        catch {
-            continue;
-        }
-        const parsed = parseFrontmatter(raw);
-        const belief = typeof parsed.data[FM_BELIEF_STATE] === "string" ? parsed.data[FM_BELIEF_STATE] : "";
-        if (EXCLUDED_BELIEF_STATES.has(belief))
-            continue;
-        const lastConfirmedMs = parseDateMs(parsed.data[FM_LAST_CONFIRMED_AT]);
-        const signalMs = lastConfirmedMs ?? stat.mtimeMs;
-        const ageMs = now - signalMs;
-        if (ageMs < thresholdMs)
-            continue;
-        const name = toMemoryName(memoriesDir, filePath);
-        if (!name)
-            continue;
-        out.push({
-            filePath,
-            ref: `memory:${name}`,
-            name,
-            data: parsed.data,
-            body: parsed.content,
-            lastSignalMs: signalMs,
-        });
-    }
-    return out;
-}
-function collectAllMemoriesForSimilarity(stashRoot) {
-    const memoriesDir = path.join(stashRoot, "memories");
-    if (!fs.existsSync(memoriesDir))
-        return [];
-    const out = [];
-    const now = Date.now();
-    for (const filePath of walkMarkdownFiles(memoriesDir)) {
-        let raw;
-        let stat;
-        try {
-            raw = fs.readFileSync(filePath, "utf8");
-            stat = fs.statSync(filePath);
-        }
-        catch {
-            continue;
-        }
-        const parsed = parseFrontmatter(raw);
-        const name = toMemoryName(memoriesDir, filePath);
-        if (!name)
-            continue;
-        const title = typeof parsed.data.title === "string" ? parsed.data.title : "";
-        const description = typeof parsed.data.description === "string" ? parsed.data.description : "";
-        const body = parsed.content;
-        out.push({
-            ref: `memory:${name}`,
-            name,
-            filePath,
-            body,
-            title,
-            description,
-            tokens: tokenize(`${title} ${description} ${body}`),
-            ageMs: now - stat.mtimeMs,
-        });
-    }
-    return out;
-}
-function toMemoryName(memoriesDir, filePath) {
-    const rel = path.relative(memoriesDir, filePath);
-    if (!rel || rel.startsWith(".."))
-        return undefined;
-    return rel.replace(/\\/g, "/").replace(/\.md$/i, "");
-}
-function parseDateMs(value) {
-    if (typeof value !== "string")
-        return undefined;
-    const ms = Date.parse(value);
-    return Number.isFinite(ms) ? ms : undefined;
-}
-// ── Similarity (lightweight FTS-style fallback) ─────────────────────────────
-/**
- * Token-overlap similarity is the FTS fallback for the prompt's "similar
- * memories" payload. Embedding-aware nearest-neighbor lookup is a future
- * enhancement; the prompt only needs enough context to ground a YES/NO
- * decision, which token overlap of body + description provides.
- */
-function tokenize(text) {
-    const out = new Set();
-    const lower = text.toLowerCase();
-    const tokens = lower.match(/[a-z0-9][a-z0-9_-]{2,}/g);
-    if (!tokens)
-        return out;
-    for (const t of tokens)
-        out.add(t);
-    return out;
-}
-function pickSimilar(candidate, all) {
-    const candTokens = tokenize(`${typeof candidate.data.title === "string" ? candidate.data.title : ""} ${typeof candidate.data.description === "string" ? candidate.data.description : ""} ${candidate.body}`);
-    const candMs = candidate.lastSignalMs;
-    const scored = [];
-    for (const snap of all) {
-        if (snap.ref === candidate.ref)
-            continue;
-        // Prefer memories more recent than the candidate so the validator can
-        // see what may have superseded it.
-        if (snap.ageMs >= Date.now() - candMs)
-            continue;
-        let overlap = 0;
-        for (const t of candTokens)
-            if (snap.tokens.has(t))
-                overlap += 1;
-        if (overlap === 0)
-            continue;
-        scored.push({ snap, score: overlap });
-    }
-    scored.sort((a, b) => b.score - a.score);
-    return scored.slice(0, TOP_K_SIMILAR).map((s) => s.snap);
-}
-// ── LLM dispatch ────────────────────────────────────────────────────────────
-const SYSTEM_PROMPT = "You are a belief-state classifier for a memory store. Given a candidate memory and a list of more-recent similar memories from the same store, decide whether the candidate is still current or has been superseded.\n\n" +
-    "Respond on the first line with exactly YES or NO.\n" +
-    "If YES, the second line MUST be of the form `SUPERSEDED_BY: <ref>` where <ref> is the exact ref of the superseding memory from the list provided. Do NOT invent refs.\n" +
-    "If NO, do not include any additional lines.\n" +
-    "No prose, no preamble, no markdown.";
-async function askValidator(connection, candidate, allMemories, signal, timeoutMs) {
-    const similar = pickSimilar(candidate, allMemories);
-    if (similar.length === 0) {
-        // No more-recent similar memories — there is nothing the candidate could
-        // have been superseded by. Treat as confirmed without paying for an LLM call.
-        return { decision: "confirmed" };
-    }
-    const messages = [
-        { role: "system", content: SYSTEM_PROMPT },
-        { role: "user", content: buildPrompt(candidate, similar) },
-    ];
-    let raw;
-    try {
-        raw = await chatCompletion(connection, messages, {
-            ...(typeof timeoutMs === "number" ? { timeoutMs } : {}),
-            ...(signal ? { signal } : {}),
-            temperature: 0,
-        });
-    }
-    catch (err) {
-        warn(`[improve] staleness detection LLM call failed for ${candidate.ref}: ${err instanceof Error ? err.message : String(err)}`);
-        return undefined;
-    }
-    return parseStalenessResponse(raw);
-}
-export function buildPrompt(candidate, similar) {
-    const lines = [];
-    lines.push(`Candidate memory: ${candidate.ref}`);
-    if (typeof candidate.data.title === "string" && candidate.data.title.trim().length > 0) {
-        lines.push(`Title: ${candidate.data.title.trim()}`);
-    }
-    if (typeof candidate.data.description === "string" && candidate.data.description.trim().length > 0) {
-        lines.push(`Description: ${candidate.data.description.trim()}`);
-    }
-    lines.push("Body:");
-    lines.push(candidate.body.trim());
-    lines.push("");
-    lines.push(`Similar more-recent memories (top ${similar.length}):`);
-    for (const s of similar) {
-        lines.push("---");
-        lines.push(`Ref: ${s.ref}`);
-        if (s.title)
-            lines.push(`Title: ${s.title}`);
-        if (s.description)
-            lines.push(`Description: ${s.description}`);
-        lines.push("Body:");
-        lines.push(s.body.trim());
-    }
-    lines.push("");
-    lines.push("Question: Given the more-recent similar memories above, has the candidate memory been superseded, or is it still current?");
-    return lines.join("\n");
-}
-/** Exported for direct unit testing. */
-export function parseStalenessResponse(raw) {
-    if (typeof raw !== "string")
-        return undefined;
-    const trimmed = raw.trim();
-    if (trimmed.length === 0)
-        return undefined;
-    const lines = trimmed
-        .split(/\r?\n/)
-        .map((l) => l.trim())
-        .filter((l) => l.length > 0);
-    if (lines.length === 0)
-        return undefined;
-    const head = lines[0]?.toUpperCase();
-    if (head === "NO")
-        return { decision: "confirmed" };
-    if (head === "YES") {
-        // Find the SUPERSEDED_BY line; tolerate it being anywhere in the body
-        // but require an exact prefix match.
-        for (const line of lines.slice(1)) {
-            const m = line.match(/^SUPERSEDED_BY:\s*(\S.*)$/i);
-            if (m?.[1]) {
-                const ref = m[1].trim();
-                if (ref.length > 0)
-                    return { decision: "deprecated", supersededBy: ref };
-            }
-        }
-        return undefined; // YES without a SUPERSEDED_BY line is a parse error.
-    }
-    return undefined;
-}
-// ── Ref validation ──────────────────────────────────────────────────────────
-/**
- * Validate that the proposed `supersededBy` ref actually exists. We first try
- * the indexed DB (canonical source of truth when available), then fall back to
- * an on-disk filesystem probe under `<stash>/memories/<name>.md` so the pass
- * works even on stashes that have never been indexed.
- */
-function validateSupersedingRef(refStr, stashRoot, db) {
-    const trimmed = refStr.trim();
-    if (!trimmed)
-        return undefined;
-    if (db) {
-        try {
-            const id = findEntryIdByRef(db, trimmed);
-            if (typeof id === "number")
-                return trimmed;
-        }
-        catch {
-            // Fall through to filesystem probe.
-        }
-    }
-    const m = trimmed.match(/^memory:(.+)$/);
-    if (!m)
-        return undefined;
-    const filePath = path.join(stashRoot, "memories", `${m[1]}.md`);
-    if (fs.existsSync(filePath))
-        return trimmed;
-    return undefined;
-}
-// ── Frontmatter writes ──────────────────────────────────────────────────────
-/**
- * Write `beliefState: deprecated`, `supersededBy`, and `lastConfirmedAt` to
- * the candidate's frontmatter. All other fields are preserved verbatim. Uses
- * the same atomic-write shape as `markParentProcessed()` in
- * `memory-inference.ts`: re-read the file from disk, parse the YAML block,
- * stitch a new block in front of the original body bytes.
- */
-function writeDeprecated(candidate, supersededByRef, nowIso) {
-    const raw = fs.readFileSync(candidate.filePath, "utf8");
-    const block = parseFrontmatterBlock(raw);
-    const baseFm = block ? { ...parseFrontmatter(raw).data } : {};
-    const nextFm = {
-        ...baseFm,
-        [FM_BELIEF_STATE]: "deprecated",
-        [FM_SUPERSEDED_BY]: dedupeStringArray([...stringArrayOrEmpty(baseFm[FM_SUPERSEDED_BY]), supersededByRef]),
-        [FM_LAST_CONFIRMED_AT]: nowIso,
-    };
-    writeFrontmatterAtomic(candidate.filePath, nextFm, block?.content ?? raw);
-}
-/**
- * Strict additive frontmatter write: ONLY `lastConfirmedAt` is touched.
- * Every other field in the file is preserved as-is.
- */
-function writeLastConfirmed(candidate, nowIso) {
-    const raw = fs.readFileSync(candidate.filePath, "utf8");
-    const block = parseFrontmatterBlock(raw);
-    const baseFm = block ? { ...parseFrontmatter(raw).data } : {};
-    const nextFm = {
-        ...baseFm,
-        [FM_LAST_CONFIRMED_AT]: nowIso,
-    };
-    writeFrontmatterAtomic(candidate.filePath, nextFm, block?.content ?? raw);
-}
-function writeFrontmatterAtomic(filePath, frontmatter, body) {
-    fs.writeFileSync(filePath, assembleAsset(frontmatter, body), "utf8");
-}
-function stringArrayOrEmpty(value) {
-    if (!Array.isArray(value))
-        return [];
-    return value.filter((v) => typeof v === "string" && v.trim().length > 0);
-}
-function dedupeStringArray(values) {
-    return [...new Set(values.map((v) => v.trim()).filter((v) => v.length > 0))];
-}
-// ── Body hash helper (exported for tests) ───────────────────────────────────
-/** Internal helper exported only for testing — mirrors `computeBodyHash`. */
-export function _stalenessBodyHash(filePath, body) {
-    return createHash("sha256").update(`${filePath}\n${body}`).digest("hex");
-}