akm-cli 0.9.0-beta.6 → 0.9.0-rc.0

package/dist/commands/health/advisories.js

@@ -0,0 +1,151 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Improve-pipeline advisories for `akm health`: projects the computed
+ * {@link ImproveHealthMetrics} plus a few direct event reads into the
+ * ordered advisory list.
+ */
+import { readEvents } from "../../core/events.js";
+import { getLatestCycleMetrics } from "../../storage/repositories/canaries-repository.js";
+import { ENRICHMENT_MINTED_FAIL_SHARE, ENRICHMENT_MINTED_WARN_SHARE, } from "./types.js";
+/**
+ * Build the improve-pipeline advisories for the health window from the already
+ * computed {@link ImproveHealthMetrics} plus a few direct event reads. Pure
+ * projection of state → advisories; emission order is preserved so the health
+ * report is byte-identical to the previous inline construction.
+ */
+export function collectImproveAdvisories(db, stateDbPath, since, improveSummary) {
+    const advisories = [];
+    // WS-2 proxy-adequacy tripwire: surface any outcome_proxy_inverted events
+    // in the health window as an advisory so operators know when the 0.10+
+    // rich in-session signal is no longer deferrable.
+    const proxyInvertedEvents = readEvents({ since, type: "outcome_proxy_inverted" }, { dbPath: stateDbPath }).events;
+    if (proxyInvertedEvents.length > 0) {
+        const lastEvent = proxyInvertedEvents[proxyInvertedEvents.length - 1];
+        const correlation = typeof lastEvent.metadata?.correlation === "number" ? lastEvent.metadata.correlation.toFixed(3) : "unknown";
+        advisories.push({
+            name: "outcome-proxy-adequacy",
+            status: "warn",
+            kind: "deterministic",
+            confidence: "high",
+            message: `WS-2 outcome proxy inverted (${proxyInvertedEvents.length} event(s) in window). ` +
+                `corr(outcome_score, accepted_change_rate) = ${correlation} < −0.3. ` +
+                "Popular assets are also the most-needing-improvement assets — " +
+                "the retrieval-based proxy is inverted. " +
+                "The 0.10+ rich in-session outcome signal is no longer deferrable. See plan §WS-2.",
+        });
+    }
+    // Two-tailed companion: a proxy that decays to noise (|corr| < 0.1 at scale)
+    // is as much a failure as an inverted one — it just fails silently.
+    const proxyDeadEvents = readEvents({ since, type: "outcome_proxy_dead" }, { dbPath: stateDbPath, db }).events;
+    if (proxyDeadEvents.length > 0) {
+        const lastEvent = proxyDeadEvents[proxyDeadEvents.length - 1];
+        const correlation = typeof lastEvent.metadata?.correlation === "number" ? lastEvent.metadata.correlation.toFixed(3) : "unknown";
+        advisories.push({
+            name: "outcome-proxy-dead",
+            status: "warn",
+            kind: "deterministic",
+            confidence: "high",
+            message: `WS-2 outcome proxy is DEAD (${proxyDeadEvents.length} event(s) in window). ` +
+                `|corr(outcome_score, accepted_change_rate)| = ${correlation} < 0.1 at n ≥ 500. ` +
+                "outcome_score is statistically unrelated to improvement outcomes — " +
+                "treat outcome-derived rank contributions as noise until a real usage/outcome signal lands.",
+        });
+    }
+    // Salience-distribution collapse: Gini below the uniform baseline means
+    // ranking no longer discriminates between assets.
+    if (improveSummary.degradation?.salienceUniformityFlagged) {
+        advisories.push({
+            name: "salience-uniformity-collapse",
+            status: "warn",
+            kind: "deterministic",
+            confidence: "high",
+            message: `Salience distribution collapsed toward uniform: top-100 retrieval_salience Gini = ` +
+                `${improveSummary.degradation.corpusCentroidDistance} < 0.08 (uniform baseline ≈ 0.1). ` +
+                "Ranking currently carries little to no discrimination between assets.",
+        });
+    }
+    // Enrichment-vs-minting policy: enrichment lanes edit existing assets;
+    // a rising minted share means a lane is generating new content instead.
+    const minting = improveSummary.enrichmentMinting;
+    if (minting && Number.isFinite(minting.share) && minting.share > ENRICHMENT_MINTED_WARN_SHARE) {
+        advisories.push({
+            name: "enrichment-lane-minting",
+            status: minting.share > ENRICHMENT_MINTED_FAIL_SHARE ? "fail" : "warn",
+            kind: "deterministic",
+            confidence: "high",
+            message: `Enrichment lanes minted ${minting.minted} NEW asset(s) vs ${minting.updated} update(s) ` +
+                `(${Math.round(minting.share * 100)}% minted, threshold ${Math.round(ENRICHMENT_MINTED_WARN_SHARE * 100)}%). ` +
+                "Enrichment-classed lanes (proactive/high-salience/high-retrieval/signal-delta) are ratified to edit " +
+                "existing assets only — new-asset generation belongs to the signal-gated minting lanes.",
+        });
+    }
+    // Churn: accepted proposals far exceeding distinct touched refs means the
+    // loop is repeatedly rewriting the same assets, not covering the corpus.
+    if (Number.isFinite(improveSummary.coverage.churnRatio) && improveSummary.coverage.churnRatio > 1.5) {
+        advisories.push({
+            name: "improve-churn-ratio",
+            status: "warn",
+            kind: "deterministic",
+            confidence: "high",
+            message: `Improve churn ratio ${improveSummary.coverage.churnRatio} > 1.5: ` +
+                `${improveSummary.coverage.acceptedProposals} accepted proposals touched only ` +
+                `${improveSummary.coverage.distinctRefs} distinct assets in the window — ` +
+                "repeated rewrites of the same refs count as churn, not coverage.",
+        });
+    }
+    // R5 collapse/churn detector: surface any collapse_detector_alert events
+    // in the health window, plus the latest cycle row's headline numbers so
+    // the operator can act without opening the DB. `unknown` when the detector
+    // has never produced a cycle row (no consolidate/recombine work yet).
+    try {
+        // Reuse the already-open state.db handle (readEvents supports a
+        // borrowed connection) — no extra open/migrate/close per health call.
+        const collapseAlertEvents = readEvents({ since, type: "collapse_detector_alert" }, { dbPath: stateDbPath, db }).events;
+        const latestCycle = getLatestCycleMetrics(db);
+        const cycleSummary = latestCycle
+            ? `Latest cycle (${latestCycle.ts}, ${latestCycle.pass}): mean canary recall ${latestCycle.mean_recall.toFixed(3)}, ` +
+                `distinct-content ratio ${latestCycle.distinct_content_ratio.toFixed(3)}, ` +
+                `${latestCycle.accepted_actions} accepted action(s).`
+            : "";
+        if (collapseAlertEvents.length > 0) {
+            const kinds = [...new Set(collapseAlertEvents.map((e) => String(e.metadata?.kind ?? "unknown")))];
+            const collapseKinds = kinds.filter((k) => k.startsWith("collapse"));
+            advisories.push({
+                name: "collapse-churn-detector",
+                status: "warn",
+                kind: "deterministic",
+                // Collapse kinds are measured, not inferred; churn/merge-floor
+                // volume thresholds are still being tuned (design doc §7).
+                confidence: collapseKinds.length > 0 ? "high" : "medium",
+                message: `R5 detector fired ${collapseAlertEvents.length} alert(s) in window (kinds: ${kinds.join(", ")}). ` +
+                    `${cycleSummary} See docs/design/improve-collapse-churn-detector-design.md §6.3 runbook queries.`,
+            });
+        }
+        else if (latestCycle) {
+            advisories.push({
+                name: "collapse-churn-detector",
+                status: "pass",
+                kind: "deterministic",
+                confidence: "high",
+                message: `No collapse/churn alerts in window. ${cycleSummary}`,
+            });
+        }
+        else {
+            advisories.push({
+                name: "collapse-churn-detector",
+                status: "unknown",
+                kind: "deterministic",
+                confidence: "high",
+                message: "No detector cycle rows yet — the collapse/churn detector runs only on improve cycles " +
+                    "where consolidate/recombine did work (synthesis lanes may be idle).",
+            });
+        }
+    }
+    catch {
+        // Table may predate migration 016 in odd mixed-version setups — advisory
+        // is best-effort and must never fail the health command.
+    }
+    return advisories;
+}

package/dist/commands/health/checks.js

@@ -4,6 +4,7 @@
 import { spawnSync } from "node:child_process";
 import { loadConfig } from "../../core/config/config.js";
 import { detectAgentCliProfiles, requireAgentProfile } from "../../integrations/agent/index.js";
+import { TASK_FAIL_RATE_WARN, } from "./types.js";
 const ACTIVE_RUN_WARN_MS = 15 * 60 * 1000;
 /**
  * Probe the configured agent profile. Self-contained (reads config + PATH); the
@@ -188,24 +189,62 @@ export const HEALTH_CHECKS = [
         channel: "hard",
         run: () => runAgentProbe(),
     },
+    {
+        // C2 (13-bus-factor): the cron task-failure rate was computed and rendered
+        // in the HTML report but never surfaced as an advisory, so a sustained
+        // 15–16% fail rate stayed invisible on `akm health`. Warn at/above the SAME
+        // 5% threshold the html-report badge uses (see TASK_FAIL_RATE_WARN).
+        name: "task-fail-rate",
+        channel: "advisory",
+        run: (ctx) => {
+            const pctStr = `${(ctx.taskFailRate * 100).toFixed(1)}%`;
+            const thresholdPct = `${(TASK_FAIL_RATE_WARN * 100).toFixed(0)}%`;
+            const warn = ctx.taskFailRate >= TASK_FAIL_RATE_WARN;
+            return {
+                name: "task-fail-rate",
+                kind: "deterministic",
+                status: warn ? "warn" : "pass",
+                confidence: "high",
+                message: ctx.taskRowCount === 0
+                    ? `No cron tasks ran since ${ctx.since} — no task-fail-rate signal.`
+                    : warn
+                        ? `Cron task fail rate ${pctStr} across ${ctx.taskRowCount} task(s) since ${ctx.since} ≥ ${thresholdPct} threshold — inspect failed runs (ok=false) for early-exit/harness errors.`
+                        : `Cron task fail rate ${pctStr} across ${ctx.taskRowCount} task(s) since ${ctx.since} (below ${thresholdPct} threshold).`,
+                evidence: { taskFailRate: ctx.taskFailRate, taskRowCount: ctx.taskRowCount, threshold: TASK_FAIL_RATE_WARN },
+            };
+        },
+    },
     {
         name: "semantic-search-runtime",
         channel: "advisory",
-        run: (ctx) => ({
-            name: "semantic-search-runtime",
-            kind: "deterministic",
-            status: !ctx.semanticStatus ||
-                ctx.semanticStatus.status === "pending" ||
-                ctx.semanticStatus.status === "ready-js" ||
-                ctx.semanticStatus.status === "ready-vec"
-                ? "pass"
-                : "warn",
-            confidence: "medium",
-            message: ctx.semanticStatus
-                ? `Semantic search status: ${ctx.semanticStatus.status}`
-                : "No semantic-search runtime status recorded yet.",
-            evidence: ctx.semanticStatus ? { ...ctx.semanticStatus } : undefined,
-        }),
+        run: (ctx) => {
+            const blocked = ctx.semanticStatus?.status === "blocked";
+            // The generic "status: blocked" line is not actionable when the real
+            // problem is a configured remote embedding endpoint that is down while
+            // semanticSearchMode leaves semantic search enabled — every index run
+            // burns time failing against it and searches silently degrade to
+            // keyword-only. Name the endpoint and the two ways out.
+            const remoteReason = ctx.semanticStatus?.reason?.startsWith("remote-") === true;
+            const endpointAdvisory = blocked && remoteReason && ctx.embeddingEndpoint
+                ? `Configured embedding endpoint ${ctx.embeddingEndpoint} is failing ` +
+                    `(${ctx.semanticStatus?.reason}${ctx.semanticStatus?.message ? `: ${ctx.semanticStatus.message}` : ""}) ` +
+                    `while semanticSearchMode is "${ctx.semanticSearchMode ?? "auto"}". Searches fall back to keyword-only. ` +
+                    `Restore the endpoint, or set semanticSearchMode to "off" (or remove embedding.endpoint to use the local model).`
+                : undefined;
+            return {
+                name: "semantic-search-runtime",
+                kind: "deterministic",
+                status: !ctx.semanticStatus || !blocked ? "pass" : "warn",
+                confidence: "medium",
+                message: endpointAdvisory ??
+                    (ctx.semanticStatus
+                        ? `Semantic search status: ${ctx.semanticStatus.status}`
+                        : "No semantic-search runtime status recorded yet."),
+                evidence: ctx.semanticStatus
+                    ? { ...ctx.semanticStatus, ...(ctx.embeddingEndpoint ? { embeddingEndpoint: ctx.embeddingEndpoint } : {}) }
+                    : undefined,
+            };
+        },
     },
     {
         // session-log-failures: demoted to informational — the ERROR_PATTERNS regex
@@ -257,6 +296,54 @@ export const HEALTH_CHECKS = [
             };
         },
     },
+    {
+        // #603: pool-saturation advisory. The raw `sessionsScanned` count fired on
+        // normal cadence changes (the Jun 12 false alarm). Instead track the ratio
+        // of NEW (unseen) sessions to the total session pool extract evaluated in
+        // the window: a low ratio is the *expected* steady state, only a near-zero
+        // ratio signals a possible discovery/dedup bug.
+        //
+        // unseen ≈ `sessionsScanned` (extract only processes new sessions; already-
+        // seen ones are deduped into `sessionsSkipped`). total = scanned + skipped.
+        // This is a heuristic approximation — `sessionsSkipped` also folds in
+        // too-short skips — so the check is informational and never gates status.
+        name: "pool-saturation",
+        channel: "advisory",
+        run: (ctx) => {
+            const sx = ctx.sessionExtraction;
+            const total = sx.sessionsScanned + sx.sessionsSkipped;
+            const unseen = sx.sessionsScanned;
+            const ratio = total > 0 ? unseen / total : null;
+            const pct = ratio === null ? null : Math.round(ratio * 1000) / 10;
+            let status = "pass";
+            let confidence = "low";
+            let message;
+            if (!sx.ran || ratio === null) {
+                message = "Pool saturation: no extract activity in the window — no signal.";
+            }
+            else if (ratio < 0.02) {
+                status = "warn";
+                confidence = "medium";
+                message = `Session pool near-exhausted: only ${pct}% of the ${total}-session pool was new (<2%). Possible discovery/dedup bug — verify extract is still finding new sessions.`;
+            }
+            else if (ratio < 0.1) {
+                confidence = "medium";
+                message = `Session pool saturation: ${pct}% of ${total} sessions were new (<10%, steady-state expected — informational).`;
+            }
+            else {
+                confidence = "medium";
+                message = `Session pool healthy: ${pct}% of ${total} sessions were new.`;
+            }
+            return {
+                name: "pool-saturation",
+                kind: "heuristic",
+                status,
+                confidence,
+                message,
+                evidence: { totalSessions: total, unseenSessions: unseen, saturationRatio: ratio },
+            };
+        },
+    },
     {
         name: "auto-accept-validation",
         channel: "advisory",
@@ -268,7 +355,7 @@ export const HEALTH_CHECKS = [
                 status: aa.validationFailed > 0 ? "warn" : "pass",
                 confidence: aa.promoted + aa.validationFailed > 0 ? "high" : "low",
                 message: aa.validationFailed > 0
-                    ? `${aa.validationFailed} proposal(s) passed confidence threshold but failed auto-accept validation (truncated description, invalid frontmatter, etc.) — they remain in the queue for manual review.`
+                    ? `${aa.validationFailed} auto-accept validation attempt(s) failed after passing the confidence threshold (truncated description, invalid frontmatter, etc.) — the affected proposals remain pending for manual review.`
                     : aa.promoted > 0
                         ? `Auto-accept healthy: ${aa.promoted} proposal(s) promoted, 0 validation failures.`
                         : "Auto-accept gate did not run (disabled or no proposals above threshold).",