clawmem 0.13.0 → 0.15.0

package/src/health/rerank-health.ts

@@ -0,0 +1,150 @@
+/**
+ * Reranker health probe — asserts the reranker DISCRIMINATES, not just that it responds.
+ *
+ * Background: the deployed zerank-2 GGUF was mis-converted (no score head), so it returned
+ * HTTP 200 + valid JSON + finite positive scores (~1e-11) yet ranked near-randomly and silently
+ * collapsed the final ranking to RRF. Liveness checks all passed. This probe instead runs a small
+ * golden set of same-topic (query, relevant, hard-negative) triples through the LIVE reranker
+ * (cache-bypassed) and asserts three things:
+ *
+ *   1. coverage     — the reranker scored every probe doc (store.rerank throws RerankCoverageError
+ *                     otherwise, before its zero-fill would hide an omitted score);
+ *   2. calibration  — the best relevant-doc score lands in a sane band (>= CALIB_FLOOR);
+ *   3. discrimination — EVERY pair clears score(relevant) - score(hardNegative) >= DISCRIM_MARGIN.
+ *
+ * See RERANKER-HEALTH-GUARD-DESIGN.md.
+ */
+import { readFileSync } from "fs";
+import { join } from "path";
+import { DEFAULT_RERANK_MODEL, RerankCoverageError, RerankMalformedResponseError, type Store } from "../store.ts";
+
+// Thresholds — LOCKED from a live zerank-2-seq baseline (2026-06-26, 8-pair golden set):
+//   relevant scores 0.9233-0.9700, hard-neg max 0.3120, min margin 0.6417, 0/8 inverted.
+//   broken zerank-2 GGUF regime: every score <= 8.03e-7 (32-query probe). 5-6 OOM of separation.
+// CALIB_FLOOR is the band that catches the ~0 collapse (kept conservative — the margin does the
+// discrimination, so the band must not false-fail a working-but-lower reranker). DISCRIM_MARGIN is
+// 2.5x below the live min margin (0.64) so healthy never trips, far above the degenerate ~0.
+export const RERANK_CALIB_FLOOR = 0.05; // band: best relevant-doc score across pairs must clear this
+export const RERANK_DISCRIM_MARGIN = 0.25; // per-pair: score(relevant) - score(hardNegative) >= this
+// Default per-probe-request timeout (the production remote fetch is otherwise untimed; a hung
+// reranker must not hang the healthcheck).
+export const RERANK_PROBE_TIMEOUT_MS = 10_000;
+
+export interface GoldenTriple {
+  query: string;
+  relevant: string;
+  hardNegative: string;
+  note?: string;
+}
+
+export interface RerankHealthResult {
+  ok: boolean;
+  coverageOk: boolean;
+  maxScore: number; // best relevant-doc score across pairs (calibration-band input)
+  minMargin: number; // smallest (relevant - hardNegative) margin across pairs
+  pairsTotal: number;
+  pairsScored: number; // pairs where both docs were scored (full coverage)
+  failures: string[]; // human-readable failure reasons (empty iff ok)
+  thresholds: { calibFloor: number; discrimMargin: number };
+}
+
+/** Load the shipped golden set (or an explicit path, for tests). */
+export function loadGoldenSet(path?: string): GoldenTriple[] {
+  const p = path ?? join(import.meta.dir, "rerank-golden.json");
+  const parsed = JSON.parse(readFileSync(p, "utf-8")) as { triples: GoldenTriple[] };
+  return parsed.triples;
+}
+
+/**
+ * Probe the reranker behind `store` for discrimination + calibration. Routes through store.rerank
+ * with { noCache, requireLiveCoverage, timeoutMs } so it exercises the FULL production path
+ * (remote → local fallback, dedup, intent, 400-char truncation) while bypassing the cache and
+ * enforcing live coverage. `store` is structurally typed so tests can pass a fake reranker.
+ */
+export async function probeRerankHealth(
+  store: Pick<Store, "rerank">,
+  opts: {
+    thresholds?: { calibFloor?: number; discrimMargin?: number };
+    timeoutMs?: number;
+    model?: string;
+    triples?: GoldenTriple[];
+  } = {},
+): Promise<RerankHealthResult> {
+  const calibFloor = opts.thresholds?.calibFloor ?? RERANK_CALIB_FLOOR;
+  const discrimMargin = opts.thresholds?.discrimMargin ?? RERANK_DISCRIM_MARGIN;
+  const timeoutMs = opts.timeoutMs ?? RERANK_PROBE_TIMEOUT_MS;
+  const model = opts.model ?? DEFAULT_RERANK_MODEL;
+  const triples = opts.triples ?? loadGoldenSet();
+
+  const failures: string[] = [];
+  let maxScore = 0;
+  let minMargin = Infinity;
+  let pairsScored = 0;
+
+  for (let i = 0; i < triples.length; i++) {
+    const t = triples[i]!;
+    const relFile = `golden-${i}-rel`;
+    const negFile = `golden-${i}-neg`;
+    const docs = [
+      { file: relFile, text: t.relevant },
+      { file: negFile, text: t.hardNegative },
+    ];
+    const label = `pair ${i} ("${t.query.slice(0, 40)}")`;
+
+    let scored: { file: string; score: number }[];
+    try {
+      // intent omitted (4th arg undefined); options force a live, coverage-checked call.
+      scored = await store.rerank(t.query, docs, model, undefined, {
+        noCache: true,
+        requireLiveCoverage: true,
+        timeoutMs,
+      });
+    } catch (err) {
+      if (err instanceof RerankCoverageError) {
+        failures.push(`${label}: coverage — reranker did not score ${err.missing.length} doc(s)`);
+      } else if (err instanceof RerankMalformedResponseError) {
+        failures.push(`${label}: malformed response — ${err.problems.join("; ")}`);
+      } else {
+        failures.push(`${label}: probe error — ${(err as Error).message}`);
+      }
+      continue;
+    }
+
+    const scoreMap = new Map(scored.map((s) => [s.file, s.score]));
+    const relScore = scoreMap.get(relFile);
+    const negScore = scoreMap.get(negFile);
+    if (relScore === undefined || negScore === undefined || !Number.isFinite(relScore) || !Number.isFinite(negScore)) {
+      // requireLiveCoverage should have thrown already; defensive.
+      failures.push(`${label}: missing or non-finite score after rerank`);
+      continue;
+    }
+
+    pairsScored++;
+    maxScore = Math.max(maxScore, relScore);
+    const margin = relScore - negScore;
+    minMargin = Math.min(minMargin, margin);
+    if (margin < discrimMargin) {
+      failures.push(
+        `${label}: margin ${margin.toFixed(3)} < ${discrimMargin} (rel ${relScore.toFixed(3)} vs neg ${negScore.toFixed(3)})`,
+      );
+    }
+  }
+
+  if (maxScore < calibFloor) {
+    failures.push(
+      `calibration: max relevant-doc score ${maxScore.toExponential(2)} < floor ${calibFloor} — reranker is inert/degenerate (likely the deprecated zerank-2 GGUF; re-deploy the seq-cls sidecar)`,
+    );
+  }
+  if (minMargin === Infinity) minMargin = 0; // no pair scored
+
+  return {
+    ok: failures.length === 0,
+    coverageOk: pairsScored === triples.length,
+    maxScore,
+    minMargin,
+    pairsTotal: triples.length,
+    pairsScored,
+    failures,
+    thresholds: { calibFloor, discrimMargin },
+  };
+}

package/src/mcp.ts

@@ -47,6 +47,25 @@ import {
 import { listVaults, loadVaultConfig } from "./config.ts";
 import { getEntityGraphNeighbors, searchEntities } from "./entity.ts";
 
+// =============================================================================
+// Reranker fallback telemetry
+// =============================================================================
+
+// blendRerank silently degrades to RRF order when the reranker is degenerate (the deprecated
+// zerank-2 GGUF emitted ~1e-11 scores that contributed nothing at weight 0.9). This surfaces that
+// otherwise-invisible regression. Rate-limited to at most one stderr line per minute; the running
+// count is included so a persistent failure is obvious. Run `clawmem doctor` for the full probe.
+let rerankFallbackCount = 0;
+let lastRerankFallbackWarnAt = 0;
+function onRerankFallback(reason: string): void {
+  rerankFallbackCount++;
+  const now = Date.now();
+  if (now - lastRerankFallbackWarnAt > 60_000) {
+    lastRerankFallbackWarnAt = now;
+    console.error(`[clawmem] reranker degraded → RRF fallback (${reason}); ${rerankFallbackCount} occurrence(s) this process. Run 'clawmem doctor' to check the reranker.`);
+  }
+}
+
 // =============================================================================
 // Types
 // =============================================================================
@@ -768,7 +787,7 @@ This is the recommended entry point for ALL memory queries.`,
       // prior w·(1/rrfRank) blend made RRF rank-1 immovable by the reranker. Harness-validated
       // 2026-06-25 (NL+KW known-item recall): lifts recall@1-5 + MRR@10 with no material pooled
       // recall@10 regression.
-      const blended = blendRerank(candidates, reranked);
+      const blended = blendRerank(candidates, reranked, { onFallback: onRerankFallback });
 
       // Map to SearchResults for composite scoring — hydrate from DB when needed
       const allSearchResults = [...store.searchFTS(query, 30)];

package/src/memory.ts

@@ -22,6 +22,7 @@ export const HALF_LIVES: Record<string, number> = {
   decision: Infinity,
   deductive: Infinity,
   hub: Infinity,
+  antipattern: Infinity,
 };
 
 // =============================================================================
@@ -33,6 +34,7 @@ export const TYPE_BASELINES: Record<string, number> = {
   deductive: 0.85,
   preference: 0.80,
   hub: 0.80,
+  antipattern: 0.75,
   problem: 0.75,
   research: 0.70,
   milestone: 0.70,

package/src/search-utils.ts

@@ -7,6 +7,23 @@
 import type { Store, SearchResult } from "./store.ts";
 import type { EnrichedResult } from "./memory.ts";
 
+/**
+ * Runtime floor below which the whole reranker output is treated as degenerate (→ RRF fallback).
+ * Permissive by design: its only job is to catch the near-zero collapse (the deprecated zerank-2
+ * GGUF maxed at 8.03e-7), NOT to grade quality. A working zerank-2-seq scores >= ~0.1, so it never
+ * false-trips a healthy reranker. Distinct from the stricter doctor CALIB_FLOOR (rerank-health.ts).
+ */
+export const RERANK_DEGENERATE_FLOOR = 1e-4;
+
+export interface BlendRerankOptions {
+  /** Reranker dominance in the blend (default 0.9). */
+  rerankWeight?: number;
+  /** Reranker is treated as unusable (→ RRF fallback) unless some score exceeds this floor. */
+  degenerateFloor?: number;
+  /** Invoked when the reranker is unusable and the blend silently falls back to pure RRF order. */
+  onFallback?: (reason: string) => void;
+}
+
 // =============================================================================
 // Result Enrichment
 // =============================================================================
@@ -135,17 +152,31 @@ export function reciprocalRankFusion(
  * preserve their relative RRF order among themselves.
  *
  * @param candidates - RRF-ordered candidates; `score` is the RRF fusion score (positive).
- * @param reranked - reranker output `{file, score in [0,1]}`; may be empty/partial/all-zero.
- * @param rerankWeight - weight on the reranker term (default 0.9; `1-rerankWeight` on RRF).
+ * @param reranked - reranker output `{file, score in [0,1]}`; may be empty/partial/all-zero/degenerate.
+ * @param options - bare number (rerankWeight, back-compat) OR { rerankWeight, degenerateFloor, onFallback }.
  * @returns candidates re-scored and sorted by blended score descending.
  */
 export function blendRerank(
   candidates: { file: string; score: number }[],
   reranked: { file: string; score: number }[],
-  rerankWeight: number = 0.9
+  options: number | BlendRerankOptions = {}
 ): { file: string; score: number }[] {
+  const opts: BlendRerankOptions = typeof options === "number" ? { rerankWeight: options } : options;
+  const rerankWeight = opts.rerankWeight ?? 0.9;
+  const degenerateFloor = opts.degenerateFloor ?? RERANK_DEGENERATE_FLOOR;
+
   const rerankScoreMap = new Map(reranked.map(r => [r.file, r.score]));
-  const rerankUsable = reranked.length > 0 && reranked.some(r => Number.isFinite(r.score) && r.score > 0);
+  // Usable iff at least one score clears the degenerate floor. The old check (`> 0`) let the broken
+  // reranker's ~1e-11 scores through as "usable", contributing ~nothing at weight 0.9 — a silent
+  // collapse to RRF order. The floor closes that hole; onFallback makes the degrade visible.
+  const rerankUsable = reranked.length > 0 && reranked.some(r => Number.isFinite(r.score) && r.score > degenerateFloor);
+  if (!rerankUsable && opts.onFallback) {
+    opts.onFallback(
+      reranked.length === 0
+        ? "reranker returned no scores"
+        : `all ${reranked.length} rerank scores <= degenerate floor ${degenerateFloor}`
+    );
+  }
   const maxRrf = candidates.reduce((m, c) => Math.max(m, c.score), 0) || 1;
   return candidates
     .map(c => {

package/src/store.ts

@@ -1143,7 +1143,7 @@ export type Store = {
 
   // Query expansion & reranking
   expandQuery: (query: string, model?: string, intent?: string) => Promise<ExpandedQuery[]>;
-  rerank: (query: string, documents: { file: string; text: string }[], model?: string, intent?: string) => Promise<{ file: string; score: number }[]>;
+  rerank: (query: string, documents: { file: string; text: string }[], model?: string, intent?: string, options?: RerankProbeOptions) => Promise<{ file: string; score: number }[]>;
 
   // Document retrieval
   findDocument: (filename: string, options?: { includeBody?: boolean }) => DocumentResult | DocumentNotFound;
@@ -1337,7 +1337,7 @@ export function createStore(dbPath?: string, opts?: { readonly?: boolean; busyTi
 
     // Query expansion & reranking
     expandQuery: (query: string, model?: string, intent?: string) => expandQuery(query, model, db, intent),
-    rerank: (query: string, documents: { file: string; text: string }[], model?: string, intent?: string) => rerank(query, documents, model, db, intent),
+    rerank: (query: string, documents: { file: string; text: string }[], model?: string, intent?: string, options?: RerankProbeOptions) => rerank(query, documents, model, db, intent, options),
 
     // Document retrieval
     findDocument: (filename: string, options?: { includeBody?: boolean }) => findDocument(db, filename, options),
@@ -3630,9 +3630,46 @@ export async function expandQuery(query: string, model: string = DEFAULT_QUERY_M
 // Reranking
 // =============================================================================
 
-export async function rerank(query: string, documents: { file: string; text: string }[], model: string = DEFAULT_RERANK_MODEL, db: Database, intent?: string): Promise<{ file: string; score: number }[]> {
+/** Options for the reranker health probe. Production query/hook callers omit all of these. */
+export type RerankProbeOptions = {
+  /** Skip the rerank cache entirely — forces a live endpoint call (health probes). */
+  noCache?: boolean;
+  /** Throw RerankCoverageError if any input doc was not scored by the reranker (checked before zero-fill). */
+  requireLiveCoverage?: boolean;
+  /** Abort signal for the remote fetch. */
+  signal?: AbortSignal;
+  /** Convenience: derive AbortSignal.timeout(timeoutMs) for the remote fetch when no signal is given. */
+  timeoutMs?: number;
+};
+
+/** Thrown by rerank() when requireLiveCoverage is set and the reranker did not score every input doc. */
+export class RerankCoverageError extends Error {
+  constructor(public readonly missing: string[]) {
+    super(`rerank coverage incomplete: ${missing.length} document(s) not scored by the reranker`);
+    this.name = "RerankCoverageError";
+  }
+}
+
+/**
+ * Thrown by rerank() when requireLiveCoverage is set and the reranker's raw response violates the
+ * coverage contract: wrong result count, duplicate/out-of-range index, or a non-finite score. A
+ * malformed-but-responding reranker is exactly the failure a health probe must catch — it must not
+ * be silently accepted (and a duplicate index can otherwise leave a doc unscored, or an out-of-range
+ * index can crash the score apply).
+ */
+export class RerankMalformedResponseError extends Error {
+  constructor(public readonly problems: string[]) {
+    super(`rerank response malformed: ${problems.join("; ")}`);
+    this.name = "RerankMalformedResponseError";
+  }
+}
+
+export async function rerank(query: string, documents: { file: string; text: string }[], model: string = DEFAULT_RERANK_MODEL, db: Database, intent?: string, options?: RerankProbeOptions): Promise<{ file: string; score: number }[]> {
   // Prepend intent to rerank query so the reranker scores with domain context
   const rerankQuery = intent ? `${intent}\n\n${query}` : query;
+  const noCache = options?.noCache === true;
+  // Health probes thread a timeout to the remote fetch (the production path is otherwise untimed).
+  const fetchSignal = options?.signal ?? (options?.timeoutMs ? AbortSignal.timeout(options.timeoutMs) : undefined);
 
   // Deduplicate identical chunk texts — same content from different files shares a single score
   const textToFiles = new Map<string, string[]>();
@@ -3650,16 +3687,22 @@ export async function rerank(query: string, documents: { file: string; text: str
   const cachedResults: Map<string, number> = new Map();
   const uncachedDocs: RerankDocument[] = [];
 
-  // Check cache for each unique document
-  for (const doc of uniqueDocs) {
-    const cacheKey = getCacheKey("rerank", { query: rerankQuery, file: doc.file, model });
-    const cached = getCachedResult(db, cacheKey);
-    if (cached !== null) {
-      const score = parseFloat(cached);
-      // Apply score to all files sharing this text
-      for (const file of textToFiles.get(doc.text)!) cachedResults.set(file, score);
-    } else {
-      uncachedDocs.push({ file: doc.file, text: doc.text });
+  // Check cache for each unique document. noCache (health probes) skips the cache entirely so the
+  // call always exercises the live endpoint — a cached probe would mask an endpoint silently
+  // reverted to a broken reranker.
+  if (noCache) {
+    for (const doc of uniqueDocs) uncachedDocs.push({ file: doc.file, text: doc.text });
+  } else {
+    for (const doc of uniqueDocs) {
+      const cacheKey = getCacheKey("rerank", { query: rerankQuery, file: doc.file, model });
+      const cached = getCachedResult(db, cacheKey);
+      if (cached !== null) {
+        const score = parseFloat(cached);
+        // Apply score to all files sharing this text
+        for (const file of textToFiles.get(doc.text)!) cachedResults.set(file, score);
+      } else {
+        uncachedDocs.push({ file: doc.file, text: doc.text });
+      }
     }
   }
 
@@ -3684,13 +3727,50 @@ export async function rerank(query: string, documents: { file: string; text: str
               query: rerankQuery,
               documents: batch.map(d => d.text.slice(0, 400)),
             }),
+            signal: fetchSignal,
           });
           if (resp.ok) {
-            const data = await resp.json() as { results: { index: number; relevance_score: number }[] };
-            for (const r of data.results) {
-              const doc = batch[r.index]!;
-              const cacheKey = getCacheKey("rerank", { query: rerankQuery, file: doc.file, model });
-              setCachedResult(db, cacheKey, r.relevance_score.toString());
+            let data: { results: { index: number; relevance_score: number }[] };
+            try {
+              data = await resp.json() as { results: { index: number; relevance_score: number }[] };
+            } catch {
+              // Invalid JSON from a 200 response. For a probe this is a malformed remote → surface it;
+              // for production treat it like a transport failure and fall through to local.
+              if (options?.requireLiveCoverage) throw new RerankMalformedResponseError(["response body is not valid JSON"]);
+              break;
+            }
+            // Strict contract for health probes: a results array of exactly batch.length, each with a
+            // unique in-range integer index and a finite numeric score. A malformed-but-responding
+            // reranker (including a null/primitive body) must surface, not be silently accepted.
+            if (options?.requireLiveCoverage) {
+              const problems: string[] = [];
+              if (data === null || typeof data !== "object" || !Array.isArray(data.results)) {
+                problems.push("response is not an object with a results array");
+              } else {
+                if (data.results.length !== batch.length) {
+                  problems.push(`batch expected ${batch.length} results, got ${data.results.length}`);
+                }
+                const seen = new Set<number>();
+                for (const r of data.results) {
+                  if (!Number.isInteger(r?.index) || r.index < 0 || r.index >= batch.length) problems.push(`index ${r?.index} out of range`);
+                  else if (seen.has(r.index)) problems.push(`duplicate index ${r.index}`);
+                  else seen.add(r.index);
+                  if (typeof r?.relevance_score !== "number" || !Number.isFinite(r.relevance_score)) problems.push(`non-finite score at index ${r?.index}`);
+                }
+              }
+              if (problems.length > 0) throw new RerankMalformedResponseError(problems);
+            }
+            // Defensive (all callers): guard a non-array body, and skip out-of-range/non-finite entries
+            // so a malformed response can never crash the score apply or store garbage. Under
+            // requireLiveCoverage the strict check above has already thrown; here a skipped entry just
+            // leaves the doc unscored (→ coverage error for probes, → zero-fill for production).
+            for (const r of (Array.isArray(data?.results) ? data.results : [])) {
+              const doc = batch[r.index];
+              if (!doc || typeof r.relevance_score !== "number" || !Number.isFinite(r.relevance_score)) continue;
+              if (!noCache) {
+                const cacheKey = getCacheKey("rerank", { query: rerankQuery, file: doc.file, model });
+                setCachedResult(db, cacheKey, r.relevance_score.toString());
+              }
               // Apply score to all files sharing this text
               for (const file of textToFiles.get(doc.text)!) cachedResults.set(file, r.relevance_score);
             }
@@ -3699,8 +3779,11 @@ export async function rerank(query: string, documents: { file: string; text: str
           }
         }
         scored = cachedResults.size > 0;
-      } catch {
-        // Remote failed, fall through to local
+      } catch (e) {
+        // Network/transport failure → fall through to local. But a malformed-response error (only
+        // raised under requireLiveCoverage) is a probe FINDING about the remote endpoint — propagate
+        // it instead of masking it with the local fallback.
+        if (e instanceof RerankMalformedResponseError) throw e;
       }
     }
 
@@ -3712,8 +3795,10 @@ export async function rerank(query: string, documents: { file: string; text: str
         const rerankResult = await llm.rerank(rerankQuery, remaining, { model });
         for (const result of rerankResult.results) {
           const doc = remaining.find(d => d.file === result.file);
-          const cacheKey = getCacheKey("rerank", { query: rerankQuery, file: result.file, model });
-          setCachedResult(db, cacheKey, result.score.toString());
+          if (!noCache) {
+            const cacheKey = getCacheKey("rerank", { query: rerankQuery, file: result.file, model });
+            setCachedResult(db, cacheKey, result.score.toString());
+          }
           // Apply score to all files sharing this text
           if (doc) {
             for (const file of textToFiles.get(doc.text)!) cachedResults.set(file, result.score);
@@ -3725,6 +3810,14 @@ export async function rerank(query: string, documents: { file: string; text: str
     }
   }
 
+  // Coverage check BEFORE the zero-fill below (health probes only, via requireLiveCoverage).
+  // After the map, an omitted score and a true 0 are indistinguishable, so a partial endpoint
+  // would otherwise look fully covered. See RERANKER-HEALTH-GUARD-DESIGN.md §5 (H1/M4).
+  if (options?.requireLiveCoverage) {
+    const missing = documents.filter(doc => !cachedResults.has(doc.file)).map(doc => doc.file);
+    if (missing.length > 0) throw new RerankCoverageError(missing);
+  }
+
   // Return all results sorted by score
   return documents
     .map(doc => ({ file: doc.file, score: cachedResults.get(doc.file) || 0 }))