claude-mem-lite 3.3.1 → 3.5.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "3.3.1",
+      "version": "3.5.0",
       "source": "./",
       "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark)."
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "3.3.1",
+  "version": "3.5.0",
   "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
   "author": {
     "name": "sdsrss"

package/README.md

@@ -644,6 +644,59 @@ Benchmarked on 200 observations across 30 queries (standard + hard-negative cate
 
 The benchmark suite runs as a CI gate (`npm run benchmark:gate`) to prevent search quality regressions.
 
+### Recall on LongMemEval (standard benchmark)
+
+Beyond the in-repo micro-benchmark above, claude-mem-lite is measured on
+[LongMemEval](https://github.com/xiaowu0162/LongMemEval) (Wu et al.) — a
+500-question long-term-memory benchmark — so its recall is comparable to the
+field, not just to itself. Metric is **recall_any@k**: does *any* gold evidence session appear in the
+top *k* retrieved? This is the same session-level definition the systems we
+compare against report on this split — [agentmemory](https://github.com/rohitg00/agentmemory)
+(BM25 + vector + graph) and dense-embedding systems like MemPalace — so the rows
+below sit on one axis, not metric-shopped. (Note: 65% of the 500 questions have
+multiple gold sessions, so `recall_any@k` is looser than fractional recall there;
+all systems in this comparison report the any-hit form.) Corpus is user-turns-only
+(the standard raw-baseline rule). Runners: `benchmark/longmemeval.mjs` (lexical)
+and `benchmark/longmemeval-rerank.mjs` (rerank).
+
+| Retriever (zero embeddings) | @1 | @5 | @10 |
+|---|---|---|---|
+| Lexical hybrid — FTS5 + TF-IDF + RRF | 76.8% | 90.6% | 95.2% |
+| + one top-20 LLM rerank pass | **92.8%** | **96.8%** | **97.4%** |
+
+*n = 500 questions; 99.8% JSON parse-rate at concurrency 3.* The rerank pass
+hands the top 20 lexical candidates to a single Haiku call (~1.4 s/query) that
+reorders them. It is **never worse than the lexical baseline by construction** —
+any LLM or parse failure falls back to the original candidate order.
+
+**Stricter metric, for the record.** The rows above are `recall_any@k` — does *any*
+gold session reach the top *k* — the metric agentmemory and MemPalace publish, so the
+comparison is like-for-like. Under the stricter **standard recall@k** (`|gold ∩ top-k| /
+|gold|`, the *fraction* of all gold sessions retrieved), the lexical stack scores
+@1 = 46.9% / @5 = 84.4% / @10 = 91.9%. The whole gap is the 65% of questions with
+multiple gold sessions — any-hit needs one, fractional needs them all, and @1 is capped
+at 1/|gold| there; single-gold question types score identically under both.
+`benchmark/longmemeval.mjs` reports both columns (the rerank row's fractional is not yet
+measured).
+
+**On embeddings, honestly.** With no LLM in the loop, both a dense-embedding
+baseline (MemPalace, ~96.6% @5) and a BM25 + vector + graph hybrid (agentmemory,
+95.2% @5) out-recall our zero-embedding lexical stack (90.6% @5) at the same
+retrieval stage — dense and graph signal genuinely help raw recall, and most of
+our remaining gap is paraphrase (single-session-preference is our lowest category
+at 63%). The rerank row's point is that a *single cheap LLM call closes it*:
+reordering the top-20 lexical candidates reaches 96.8% @5 — matching the dense raw
+number and edging the hybrid's retrieval score — because the lexical candidate set
+is already rich enough (recall@20 = 97.8%) that ranking, not recall, is the
+bottleneck. An embedding-plus-rerank stack still leads when both sides spend an LLM
+call; the takeaway is that claude-mem-lite reaches embedding-competitive precision
+with **no vector model, no knowledge graph, no Python, and no external service**.
+
+Per-category @5 (lexical → +rerank): knowledge-update 98.7 → 100.0 ·
+single-session-user 91.4 → 98.6 · temporal-reasoning 89.5 → 97.7 · multi-session
+95.5 → 97.7 · single-session-assistant 83.9 → 94.6 · single-session-preference
+63.3 → 80.0. Every category improves; none regress.
+
 ## Development
 
 ```bash

package/deep-search.mjs

@@ -34,10 +34,19 @@
 import { searchObservationsHybrid } from './search-engine.mjs';
 import { sanitizeFtsQuery } from './utils.mjs';
 import { RRF_K } from './tfidf.mjs';
+import { llmRerankOrder, defaultRerankLLM } from './rerank.mjs';
 
 // original + up to 3 rewrites (keyword / concept-expansion / HyDE).
 export const MAX_VARIANTS = 4;
 
+// How many RRF-fused candidates the opt-in rerank stage hands to the LLM. The
+// LongMemEval rerank benchmark (benchmark/longmemeval-rerank.mjs) measured the
+// lexical candidate set as rich enough at 20 (recall@20 = 97.8%) that reranking
+// the top-20 captures nearly all of that ceiling (96.8%@5); matching it here keeps
+// the shipped behaviour aligned with the measured number. Module-internal — callers
+// override per-call via deps.rerankTopK; export it if a config surface ever needs it.
+const RERANK_TOPK = 20;
+
 // ─── Auto-escalation (opt-in adaptive deep search) ──────────────────────────
 // Result-count floor below which a normal search is "weak" enough to auto-escalate
 // to deepSearch. Calibrated against the deep-search benchmark fixtures; 3 is the
@@ -371,7 +380,33 @@ function defaultSearchFn(db, query, params) {
 }
 
 /**
- * Opt-in deep search: rewrite → per-variant hybrid search → RRF fusion.
+ * Build the candidate text the opt-in rerank stage shows the LLM. Prefers each
+ * observation's full `narrative` (the field the LongMemEval rerank benchmark
+ * scored); falls back to title / subtitle / snippet / lesson when narrative is
+ * unavailable or the db can't be read (injected rows / null db in unit tests).
+ * @param {Database|null} db
+ * @param {Array<object>} rows  fused candidate rows (already sliced to top-K)
+ * @returns {Map<any,string>} id → candidate text
+ */
+function defaultRerankText(db, rows) {
+  const fallback = (r) => [r.title, r.subtitle, r.snippet, r.lesson_learned].filter(Boolean).join(' — ');
+  if (!db) return new Map(rows.map((r) => [r.id, fallback(r)]));
+  try {
+    const ids = rows.map((r) => r.id);
+    const ph = ids.map(() => '?').join(',');
+    const found = new Map(
+      db.prepare(`SELECT id, narrative, title, subtitle FROM observations WHERE id IN (${ph})`)
+        .all(...ids)
+        .map((o) => [o.id, o.narrative || [o.title, o.subtitle].filter(Boolean).join(' — ')]),
+    );
+    return new Map(rows.map((r) => [r.id, found.get(r.id) || fallback(r)]));
+  } catch {
+    return new Map(rows.map((r) => [r.id, fallback(r)]));
+  }
+}
+
+/**
+ * Opt-in deep search: rewrite → per-variant hybrid search → RRF fusion → opt-in rerank.
  * @param {Database} db open better-sqlite3 handle
  * @param {object} params
  * @param {string} params.query  The user query.
@@ -386,11 +421,15 @@ function defaultSearchFn(db, query, params) {
  * @param {(db:Database, query:string, params:object)=>Array} [deps.searchFn]
  * @param {number} [deps.rrfK=RRF_K]
  * @param {boolean} [deps.auto=false]  use the fail-fast/throttled/cached auto provider
- * @returns {Promise<{results: Array, variants: string[]}>}
+ * @param {boolean} [deps.rerank=false]  opt-in: LLM-rerank the fused top-K (never on the auto path)
+ * @param {(prompt:object)=>Promise<any>} [deps.rerankLlm]  rerank provider (default: lazy haiku)
+ * @param {number} [deps.rerankTopK=RERANK_TOPK]  how many fused candidates to rerank
+ * @param {(db:Database, rows:Array)=>Map} [deps.rerankTextFn]  id→text builder for the rerank prompt
+ * @returns {Promise<{results: Array, variants: string[], reranked: boolean}>}
  */
-export async function deepSearch(db, params, { llm, searchFn = defaultSearchFn, rrfK = RRF_K, auto = false } = {}) {
+export async function deepSearch(db, params, { llm, searchFn = defaultSearchFn, rrfK = RRF_K, auto = false, rerank = false, rerankLlm, rerankTopK = RERANK_TOPK, rerankTextFn = defaultRerankText } = {}) {
   const query = String(params?.query ?? '').trim();
-  if (!query) return { results: [], variants: [] };
+  if (!query) return { results: [], variants: [], reranked: false };
 
   // No injected llm: EXPLICIT deep=true uses the patient defaultLLM; the AUTO
   // path uses a fail-fast + throttled provider with no retry and a process-
@@ -418,5 +457,40 @@ export async function deepSearch(db, params, { llm, searchFn = defaultSearchFn,
 
   const fused = rrfFuseN(lists, rrfK);
   const limit = params.limit ?? 10;
-  return { results: fused.slice(0, limit), variants };
+
+  // Opt-in rerank stage (option C): reorder the fused top-K by an LLM relevance
+  // read, using the same core the LongMemEval benchmark measures (rerank.mjs) so
+  // the shipped algorithm == the measured one. Strictly opt-in — the AUTO
+  // escalation path never reranks, so no default search behaviour changes and the
+  // hot path stays a single LLM call. "Never worse than the fused order" by
+  // construction: a failed/unparseable rerank leaves the fused order untouched.
+  // The candidate set fed here is RICHER than the benchmark's single-query top-20
+  // (it is multi-query RRF), so the measured 96.8%@5 is a conservative floor.
+  let ordered = fused;
+  let reranked = false;
+  if (rerank && fused.length > 1) {
+    const k = Math.min(rerankTopK, fused.length);
+    const top = fused.slice(0, k);
+    const text = rerankTextFn(db, top);
+    const cand = top.map((r) => ({ sid: r.id, text: text.get(r.id) || '' }));
+    const { order, parsed } = await llmRerankOrder(query, cand, rerankLlm || defaultRerankLLM);
+    if (parsed) {
+      const byId = new Map(top.map((r) => [r.id, r]));
+      const head = order.map((id) => byId.get(id)).filter(Boolean);
+      // Re-stamp scores so `score` stays monotonic with the rerank order, reusing
+      // the top-K's OWN values ascending (best = most-negative first): the reranked
+      // block keeps the K best scores so it stays ahead of the fused tail, and orders
+      // within itself by rerank rank. This keeps the shared CLI↔MCP `score` ordering
+      // contract (#8217) consistent with the array order, so a consumer that re-sorts
+      // by score reproduces the rerank order instead of restoring the RRF order.
+      // (server.mjs also skips its context re-rank/re-sort when reranked, so the LLM
+      // judgement is the final order — the re-stamp keeps score honest regardless.)
+      const scores = top.map((r) => r.score).sort((a, b) => a - b);
+      head.forEach((r, i) => { r.score = scores[i]; r.rrfScore = -scores[i]; });
+      ordered = [...head, ...fused.slice(k)];
+      reranked = true;
+    }
+  }
+
+  return { results: ordered.slice(0, limit), variants, reranked };
 }

package/mem-cli.mjs

@@ -9,7 +9,7 @@ import { resolveProject } from './project-utils.mjs';
 import { TIER_CASE_SQL, tierSqlParams } from './tier.mjs';
 import { _resetVocabCache } from './tfidf.mjs';
 import { autoBoostIfNeeded, reRankWithContext, markSuperseded } from './server-internals.mjs';
-import { searchObservationsHybrid, countSearchTotal } from './search-engine.mjs';
+import { searchObservationsHybrid, countSearchTotal, attachBodyTokens } from './search-engine.mjs';
 import { deepSearch, resolveDeepMode, shouldEscalateToDeep, autoDeepLlmReady, hasEscalatableCorpus } from './deep-search.mjs';
 import { ensureRegistryDb, upsertResource } from './registry.mjs';
 import { searchResources } from './registry-retriever.mjs';
@@ -52,7 +52,7 @@ async function cmdSearch(db, args, { llm } = {}) {
   const { positional, flags } = parseArgs(args);
   const query = positional.join(' ');
   if (!query) {
-    fail('[mem] Usage: claude-mem-lite search <query> [--type TYPE] [--source SOURCE] [--limit N] [--project P] [--from DATE] [--to DATE] [--importance N] [--branch B] [--offset N] [--sort relevance|time|importance] [--include-noise] [--deep] [--no-deep]');
+    fail('[mem] Usage: claude-mem-lite search <query> [--type TYPE] [--source SOURCE] [--limit N] [--project P] [--from DATE] [--to DATE] [--importance N] [--branch B] [--offset N] [--sort relevance|time|importance] [--include-noise] [--deep] [--no-deep] [--rerank]');
     return;
   }
 
@@ -109,6 +109,15 @@ async function cmdSearch(db, args, { llm } = {}) {
     : ((flags['no-deep'] === true || flags['no-deep'] === 'true') ? false : undefined);
   const deepMode = resolveDeepMode(explicitDeep, { surface: 'cli' });
 
+  // --rerank: opt-in LLM rerank of the fused top-20 (option C, deep-search.mjs).
+  // One extra Haiku call (~1.4s); only meaningful on the explicit --deep path,
+  // never on auto-escalation. Same rerank core the LongMemEval benchmark measures.
+  const rerankFlag = flags.rerank === true || flags.rerank === 'true';
+  const rerank = rerankFlag && deepMode === 'deep';
+  if (rerankFlag && deepMode !== 'deep') {
+    process.stderr.write('[mem] Note: --rerank requires --deep (it reranks deep-search candidates); ignored\n');
+  }
+
   if (source && !['observations', 'sessions', 'prompts'].includes(source)) {
     fail(`[mem] Invalid --source "${source}". Use: observations, sessions, prompts`);
     return;
@@ -160,6 +169,7 @@ async function cmdSearch(db, args, { llm } = {}) {
   let orFallbackFired = false;
 
   let deepVariants = null;
+  let isReranked = false;
   let isDeep = deepMode === 'deep';
 
   // Search observations — shared engine with server.mjs (#8198/#8212 paired-path fix)
@@ -194,13 +204,19 @@ async function cmdSearch(db, args, { llm } = {}) {
         epochTo: dateTo,
         limit: perSourceLimit,
         currentProject: project ? null : inferProject(),
-      }, llm ? { llm } : { auto });
+      }, llm ? { llm, rerank: rerank && !auto } : { auto, rerank: rerank && !auto });
       deepVariants = ds.variants;
+      isReranked = ds.reranked;
       if (deepVariants.length > 1) {
         process.stderr.write(`[mem] Deep search: rewrote into ${deepVariants.length} query variants, RRF-fused\n`);
       } else {
         process.stderr.write('[mem] Deep search: rewrite returned no usable variants; used original query only\n');
       }
+      if (rerank && !auto) {
+        process.stderr.write(ds.reranked
+          ? '[mem] Deep search: LLM-reranked the fused top-20\n'
+          : '[mem] Deep search: rerank produced no usable order; kept fused order\n');
+      }
       return ds.results;
     };
 
@@ -270,7 +286,9 @@ async function cmdSearch(db, args, { llm } = {}) {
   if (obsResults.length > 0) {
     // reRankWithContext/markSuperseded expect source='obs' — alias _source for compatibility
     for (const r of obsResults) r.source = 'obs';
-    reRankWithContext(db, obsResults, project || inferProject());
+    // Explicit LLM rerank order is final — skip file-context re-rank when reranked
+    // (paired-path with mem_search; markSuperseded still runs for stale-tagging).
+    if (!isReranked) reRankWithContext(db, obsResults, project || inferProject());
     markSuperseded(obsResults);
     if (isCrossSource) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
   }
@@ -305,6 +323,9 @@ async function cmdSearch(db, args, { llm } = {}) {
       includeNoise,
     }), results.length);
   const paged = results.slice(offset, offset + limit);
+  // Enrich the final page with the ~Nt fetch-cost hint (paired with MCP mem_search; #8654 both
+  // source keys handled). Batch-fetches heavy obs fields by id — no-op on an empty page.
+  attachBodyTokens(db, paged);
 
   if (paged.length === 0) {
     if (jsonOutput) {
@@ -343,6 +364,7 @@ async function cmdSearch(db, args, { llm } = {}) {
         importance: r.importance ?? null,
         superseded: Boolean(r.superseded),
         files_modified: r.files_modified || null,
+        body_tokens: r.bodyTokens ?? null,
       };
     });
     out(JSON.stringify({
@@ -364,19 +386,22 @@ async function cmdSearch(db, args, { llm } = {}) {
   // Pluralize on total — "Found 1 of 44 result" reads wrong; the population (44) drives
   // grammatical number, not the page slice (1).
   out(`[mem] Found ${countLabel} result${total !== 1 ? 's' : ''} for "${query}"${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt)' : ''}`);
+  // `~Nt` = est. tokens to fetch this row's full body via mem_get (attachBodyTokens, paired with
+  // MCP). Conditional so a row that skipped enrichment renders cleanly, not "~undefinedt".
+  const tok = r => (r.bodyTokens ? ` ~${r.bodyTokens}t` : '');
   for (const r of paged) {
     const timeStr = showTime && r.created_at_epoch ? ` (${relativeTime(r.created_at_epoch)})` : '';
     if (r._source === 'session') {
       const date = fmtDateShort(r.created_at);
-      out(`S#${r.id} 📋 ${date}${timeStr} ${truncate(r.request || r.completed || '(no summary)', 80)}`);
+      out(`S#${r.id} 📋 ${date}${timeStr} ${truncate(r.request || r.completed || '(no summary)', 80)}${tok(r)}`);
     } else if (r._source === 'prompt') {
       const date = fmtDateShort(r.created_at);
-      out(`P#${r.id} 💬 ${date}${timeStr} ${truncate(r.prompt_text || '(empty)', 80)}`);
+      out(`P#${r.id} 💬 ${date}${timeStr} ${truncate(r.prompt_text || '(empty)', 80)}${tok(r)}`);
     } else {
       const date = fmtDateShort(r.created_at);
       const title = truncate(r.title || r.subtitle || '(untitled)', 80);
       const supersededTag = r.superseded ? ' [SUPERSEDED]' : '';
-      out(`#${r.id} ${typeIcon(r.type)} ${date}${timeStr} ${title}${supersededTag}`);
+      out(`#${r.id} ${typeIcon(r.type)} ${date}${timeStr} ${title}${supersededTag}${tok(r)}`);
       if (r.lesson_learned) {
         out(`  -> ${truncate(r.lesson_learned, 80)}`);
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "3.3.1",
+  "version": "3.5.0",
   "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
   "type": "module",
   "packageManager": "npm@10.9.2",
@@ -31,6 +31,7 @@
     "server-internals.mjs",
     "search-engine.mjs",
     "deep-search.mjs",
+    "rerank.mjs",
     "hook.mjs",
     "hook-shared.mjs",
     "hook-llm.mjs",

package/rerank.mjs

@@ -0,0 +1,78 @@
+// Shared LLM-rerank core: reorder a top-K candidate list by an LLM relevance read.
+//
+// Used by BOTH the production deep-search rerank stage (deep-search.mjs) and the
+// LongMemEval rerank benchmark (benchmark/longmemeval-rerank.mjs), so the measured
+// lift number reflects the EXACT algorithm that ships. "Never worse than the input
+// candidate order" by construction: any LLM/parse failure returns the original order.
+//
+// The LLM is dependency-injected by every caller, so this module is unit-tested with
+// deterministic stubs and never statically imports the native-heavy LLM client (the
+// default provider is pulled in lazily on first real call).
+import { parseJsonFromLLM } from './utils.mjs';
+
+// Module-internal: only buildRerankPrompt (below) consumes these. Kept un-exported
+// so the module's public surface is just the three functions callers actually import.
+const RERANK_SYSTEM =
+  'You rerank search results. Given a QUERY and numbered candidate session snippets, ' +
+  'decide which sessions most likely contain the answer to the query. ' +
+  'Return ONLY JSON {"ranked":[<candidate numbers, most relevant first, each number once>]}. No prose, no markdown.';
+
+function buildRerankPrompt(query, snippets) {
+  const lines = snippets.map((s, i) => `${i + 1}. ${String(s).replace(/\s+/g, ' ').slice(0, 400)}`);
+  return {
+    system: RERANK_SYSTEM,
+    user: `QUERY: ${query}\n\nCANDIDATES:\n${lines.join('\n')}\n\nReturn {"ranked":[...]} over 1..${snippets.length}, best first.`,
+  };
+}
+
+// Extract a 1-based ranking array from whatever the LLM returned: a {ranked:[...]}
+// object (stub / clean JSON), a bare array (clean OR prose-wrapped [..]), or a
+// {text} envelope from callLLMWithModel. The bare-array path is what lifts the
+// real parse-rate: claude-haiku often answers "[3,1,5]" instead of {"ranked":..},
+// and parseJsonFromLLM's leading JSON.parse returns that as an array (no .ranked),
+// which the old object-only check silently dropped. null → nothing recoverable.
+export function extractRanked(raw) {
+  if (raw === null || raw === undefined) return null;
+  if (Array.isArray(raw)) return raw;
+  if (typeof raw === 'object' && Array.isArray(raw.ranked)) return raw.ranked;
+  const text = typeof raw === 'string' ? raw : typeof raw.text === 'string' ? raw.text : '';
+  if (!text) return null;
+  const obj = parseJsonFromLLM(text);
+  if (Array.isArray(obj)) return obj; // bare array [3,1,5]
+  if (obj && Array.isArray(obj.ranked)) return obj.ranked; // {"ranked":[...]}
+  const m = text.match(/\[\s*\d+(?:\s*,\s*\d+)*\s*\]/); // prose-wrapped [..]
+  if (m) {
+    try { const a = JSON.parse(m[0]); if (Array.isArray(a)) return a; } catch { /* fall through */ }
+  }
+  return null;
+}
+
+// Reorder candidate session ids per the LLM's chosen 1-based order; any failure →
+// original order ("never worse than baseline"). { order: sid[], parsed: bool }.
+export async function llmRerankOrder(query, cand /* [{sid,text}] */, llm) {
+  const prompt = buildRerankPrompt(query, cand.map((c) => c.text));
+  let raw;
+  try { raw = await llm(prompt); } catch { raw = null; }
+  const order = extractRanked(raw);
+  if (!order) return { order: cand.map((c) => c.sid), parsed: false };
+  const seen = new Set();
+  const out = [];
+  for (const n of order) {
+    const idx = Number(n) - 1;
+    if (Number.isInteger(idx) && idx >= 0 && idx < cand.length && !seen.has(idx)) {
+      seen.add(idx);
+      out.push(cand[idx].sid);
+    }
+  }
+  cand.forEach((c, i) => { if (!seen.has(i)) out.push(c.sid); }); // append omitted, original order
+  return { order: out, parsed: true };
+}
+
+// Default provider — lazy import so stub-injected callers never load the client.
+// Uses callLLMWithModel (returns {text}) rather than callModelJSONAsync (which
+// JSON-parses internally and nulls on any non-{...} output) so extractRanked can
+// recover bare-array answers the strict JSON parse drops.
+export async function defaultRerankLLM(prompt) {
+  const { callLLMWithModel } = await import('./haiku-client.mjs');
+  return callLLMWithModel(prompt, 'haiku', { timeout: 20000, maxTokens: 300 });
+}

package/search-engine.mjs

@@ -9,7 +9,7 @@ import {
   OBS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE,
   DEFAULT_DECAY_HALF_LIFE_MS,
   notLowSignalTitleClause, LOW_SIGNAL_TITLE,
-  relaxFtsQueryToOr, debugLog, debugCatch,
+  relaxFtsQueryToOr, debugLog, debugCatch, estimateTokens,
 } from './utils.mjs';
 import { getVocabulary, computeVector, vectorSearch, rrfMerge } from './tfidf.mjs';
 import { extractPRFTerms, expandQueryByConcepts } from './server-internals.mjs';
@@ -190,6 +190,45 @@ export function ftsRowToResult(r, { scoreMultiplier, snippet } = {}) {
   };
 }
 
+// Per-result estimate of the token cost to fetch the FULL body via mem_get, surfaced as the
+// `~Nt` hint in search output so the agent can budget the 3-layer protocol (search → timeline →
+// get) before paying to expand any ID. Adopted from thedotmack/claude-mem's token-cost column
+// (reference_claude_mem_comparison) — the one genuinely portable idea from that analysis.
+//
+// Layer-1 search deliberately omits narrative/facts (that's what keeps the index light), so the
+// heavy obs fields are batch-fetched by id HERE rather than carried on every result. The source
+// key is read as `source || _source` because the two render paths disagree (#8654): MCP sets
+// `source`+`text`, CLI sets `_source`+`prompt_text`. estimateTokens floors at 1, so a missing row
+// or empty body yields 1 — never 0/NaN.
+export function attachBodyTokens(db, results) {
+  if (!Array.isArray(results) || results.length === 0) return results;
+  const obsIds = results
+    .filter(r => (r.source || r._source) === 'obs' && Number.isInteger(r.id))
+    .map(r => r.id);
+  const bodyById = new Map();
+  if (obsIds.length > 0) {
+    try {
+      const ph = obsIds.map(() => '?').join(',');
+      const rows = db.prepare(`SELECT id, narrative, facts, text FROM observations WHERE id IN (${ph})`).all(...obsIds);
+      for (const row of rows) bodyById.set(row.id, row);
+    } catch (e) { debugCatch(e, 'attachBodyTokens'); }
+  }
+  for (const r of results) {
+    const src = r.source || r._source;
+    let parts;
+    if (src === 'obs') {
+      const row = bodyById.get(r.id) || {};
+      parts = [r.title, r.subtitle, r.lesson_learned, row.narrative, row.facts, row.text];
+    } else if (src === 'session') {
+      parts = [r.request, r.completed, r.working_on];
+    } else {
+      parts = [r.text, r.prompt_text];
+    }
+    r.bodyTokens = estimateTokens(parts.filter(Boolean).join(' '));
+  }
+  return results;
+}
+
 function expandObsByConceptCo(db, ctx, now, existingIds, results, includeNoise = false) {
   const { ftsQuery, args, epochFrom, epochTo, limit } = ctx;
   if (results.length >= Math.ceil(limit / 2)) return;

package/server.mjs

@@ -9,7 +9,7 @@ import { truncate, typeIcon, inferProject, scrubSecrets, fmtDate, debugLog, debu
 import { resolveProject as _resolveProjectShared } from './project-utils.mjs';
 import { ensureDb, DB_PATH, DB_DIR, REGISTRY_DB_PATH } from './schema.mjs';
 import { reRankWithContext, markSuperseded, autoBoostIfNeeded, runIdleCleanup, buildServerInstructions } from './server-internals.mjs';
-import { searchObservationsHybrid, countSearchTotal } from './search-engine.mjs';
+import { searchObservationsHybrid, countSearchTotal, attachBodyTokens } from './search-engine.mjs';
 import { deepSearch, resolveDeepMode, shouldEscalateToDeep, autoDeepLlmReady, hasEscalatableCorpus } from './deep-search.mjs';
 import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
 import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentTimeline, fetchTimelineWindow } from './lib/timeline-core.mjs';
@@ -294,21 +294,24 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
   const fallbackHint = orFallbackFired && !args.or ? ' (relaxed AND→OR)' : '';
   lines.push(`Found ${countLabel} result(s)${qLabel}${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt)' : ''}\n`);
 
+  // `~Nt` = estimated tokens to fetch this row's full body via mem_get (attachBodyTokens).
+  // Conditional so a result that skipped enrichment renders cleanly, not "~undefinedt".
+  const tok = r => (r.bodyTokens ? ` ~${r.bodyTokens}t` : '');
   for (const r of paginatedResults) {
     if (r.source === 'obs') {
       const supersededTag = r.superseded ? ' [SUPERSEDED]' : '';
-      lines.push(`#${r.id} ${typeIcon(r.type)} [${r.type}] ${truncate(r.title || r.subtitle || '(untitled)')} | ${r.project} | ${fmtDate(r.date)}${supersededTag}`);
+      lines.push(`#${r.id} ${typeIcon(r.type)} [${r.type}] ${truncate(r.title || r.subtitle || '(untitled)')} | ${r.project} | ${fmtDate(r.date)}${supersededTag}${tok(r)}`);
       if (r.snippet && r.snippet.length > 10 && r.snippet !== r.title) {
         lines.push(`     ${truncate(r.snippet, 100)}`);
       }
     } else if (r.source === 'session') {
-      lines.push(`S#${r.id} 📋 ${truncate(r.request || r.completed || '(no summary)')} | ${r.project} | ${fmtDate(r.date)}`);
+      lines.push(`S#${r.id} 📋 ${truncate(r.request || r.completed || '(no summary)')} | ${r.project} | ${fmtDate(r.date)}${tok(r)}`);
     } else if (r.source === 'prompt') {
-      lines.push(`P#${r.id} 💬 ${truncate(r.text)} | ${fmtDate(r.date)}`);
+      lines.push(`P#${r.id} 💬 ${truncate(r.text)} | ${fmtDate(r.date)}${tok(r)}`);
     }
   }
 
-  lines.push(`\nWorkflow: mem_timeline(anchor=ID) for context | mem_get(ids=[...]) for full details`);
+  lines.push(`\nWorkflow: mem_timeline(anchor=ID) for context | mem_get(ids=[...]) for full details  ·  ~Nt = est. tokens to fetch full detail`);
   return { content: [{ type: 'text', text: lines.join('\n') }] };
 }
 
@@ -320,11 +323,11 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
 // NOTE: resolveProject() inside runSearchPipeline closes over the module-level `db`,
 // not the injected one. Tests that pass a project: arg via this seam will trigger
 // resolveProject() against the real (module) DB, not the test DB.
-export async function handleSearchForTest(db, args, { llm } = {}) {
-  return runSearchPipeline(db, args, { llm });
+export async function handleSearchForTest(db, args, { llm, rerankLlm } = {}) {
+  return runSearchPipeline(db, args, { llm, rerankLlm });
 }
 
-async function runSearchPipeline(db, args, { llm } = {}) {
+async function runSearchPipeline(db, args, { llm, rerankLlm } = {}) {
     if (args.project) args = { ...args, project: resolveProject(args.project) };
     const limit = args.limit ?? 20;
     const offset = args.offset ?? 0;
@@ -349,6 +352,9 @@ async function runSearchPipeline(db, args, { llm } = {}) {
     // Resolve tri-state deep mode. MCP defaults to 'auto' (escalate on weak results)
     // unless explicitly overridden via args.deep or CLAUDE_MEM_AUTO_DEEP env flag.
     const deepMode = resolveDeepMode(args.deep, { surface: 'mcp' });
+    // Opt-in LLM rerank (D#43): explicit-deep only — never on AUTO escalation — so
+    // no default search behaviour changes. Parity with CLI `search --deep --rerank`.
+    const rerank = args.rerank === true && deepMode === 'deep';
 
     // Early return when query was provided but sanitized to nothing (all FTS5
     // keywords/special chars). Skipped for deep/auto — deep's LLM rewrite may
@@ -365,13 +371,14 @@ async function runSearchPipeline(db, args, { llm } = {}) {
     const ctx = { db, ftsQuery, searchType: effectiveType, args, epochFrom, epochTo, perSourceLimit, perSourceOffset, currentProject, limit };
     const results = [];
     let deepVariants = null;
+    let deepReranked = false;
     let isDeep = deepMode === 'deep';
     let escalated = false;
     let escalatedObsCount = 0;
 
     // Helper: run deepSearch and load results into the shared `results` array.
     const runDeepInto = async ({ auto = false } = {}) => {
-      const { results: deepRows, variants } = await deepSearch(db, {
+      const { results: deepRows, variants, reranked } = await deepSearch(db, {
         query: args.query,
         project: args.project || null,
         type: args.obs_type || null,
@@ -381,11 +388,12 @@ async function runSearchPipeline(db, args, { llm } = {}) {
         epochFrom, epochTo,
         limit: perSourceLimit,
         currentProject,
-      }, llm ? { llm } : { auto });
+      }, llm ? { llm, rerank: rerank && !auto, rerankLlm } : { auto, rerank: rerank && !auto, rerankLlm });
       // Safe to reset: sessions/prompts are pushed AFTER the obs block, so nothing is lost here.
       results.length = 0;
       results.push(...deepRows);
       deepVariants = variants;
+      deepReranked = reranked;
     };
 
     if (!effectiveType || effectiveType === 'observations') {
@@ -460,9 +468,13 @@ async function runSearchPipeline(db, args, { llm } = {}) {
     // empty-ftsQuery deep path we tag-but-don't-reorder (keep RRF order).
     if ((ftsQuery || isDeep) && results.some(r => r.source === 'obs')) {
       const obsResults = results.filter(r => r.source === 'obs');
-      if (ftsQuery) reRankWithContext(db, obsResults, currentProject);
+      // When the deep candidates were explicitly LLM-reranked, that order is final:
+      // skip the file-context re-rank + re-sort (they would perturb the rerank order
+      // via score multiplication / score-sort). markSuperseded is pure stale-tagging
+      // and still runs. (D#43 — parity with the CLI deep path, which keeps array order.)
+      if (ftsQuery && !deepReranked) reRankWithContext(db, obsResults, currentProject);
       markSuperseded(obsResults);
-      if (ftsQuery) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+      if (ftsQuery && !deepReranked) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
     }
 
     // Tier post-filter: batch-lookup full rows and classify (shared with CLI).
@@ -499,6 +511,9 @@ async function runSearchPipeline(db, args, { llm } = {}) {
       }), results.length);
     // Always apply pagination — single-source results can exceed SQL LIMIT due to expansion (concept co-occurrence, PRF, vector search)
     const paginatedResults = (offset > 0 || results.length > limit) ? results.slice(offset, offset + limit) : results;
+    // Enrich the FINAL page with a fetch-cost estimate (~Nt) so the agent budgets before mem_get.
+    // Uses the same db threaded through the pipeline (#8743) — batch-fetches heavy obs fields by id.
+    attachBodyTokens(db, paginatedResults);
 
     // Observability: announce auto-escalation on stderr (parity with CLI deep note).
     if (escalated) process.stderr.write(`[mem] auto-escalated to deep search (weak results: ${escalatedObsCount} hits)\n`);
@@ -512,9 +527,14 @@ async function runSearchPipeline(db, args, { llm } = {}) {
         ? `\n\n[deep search: rewrote into ${deepVariants.length} variants — ${deepVariants.slice(1).map(v => JSON.stringify(v)).join(', ')}]`
         : '\n\n[deep search: rewrite produced no usable variants; searched the original query only (== baseline)]';
     }
+    // Discoverability signal for the opt-in rerank (D#43): tell the calling agent the
+    // candidates were LLM-reranked — parity with the CLI stderr note.
+    if (deepReranked && output.content?.[0]?.type === 'text') {
+      output.content[0].text += '\n\n[deep search: LLM-reranked the top candidates by relevance]';
+    }
 
     // Return an object that exposes structured fields for tests + the MCP content blob.
-    return { ...output, results: paginatedResults, total: totalBeforePagination, escalated, variants: deepVariants };
+    return { ...output, results: paginatedResults, total: totalBeforePagination, escalated, variants: deepVariants, reranked: deepReranked };
 }
 
 server.registerTool(

package/source-files.mjs

@@ -6,7 +6,7 @@
 
 export const SOURCE_FILES = [
   // Entry points and top-level modules
-  'cli.mjs', 'cli-path.mjs', 'server.mjs', 'server-internals.mjs', 'search-engine.mjs', 'deep-search.mjs', 'tool-schemas.mjs',
+  'cli.mjs', 'cli-path.mjs', 'server.mjs', 'server-internals.mjs', 'search-engine.mjs', 'deep-search.mjs', 'rerank.mjs', 'tool-schemas.mjs',
   'hook.mjs', 'hook-shared.mjs', 'hook-llm.mjs', 'hook-memory.mjs', 'skip-tools.mjs',
   'hook-semaphore.mjs', 'hook-episode.mjs', 'hook-context.mjs', 'hook-handoff.mjs',
   'hook-update.mjs', 'hook-optimize.mjs', 'hook-precompact.mjs',

package/tool-schemas.mjs

@@ -94,6 +94,7 @@ export const memSearchSchema = {
   include_noise: z.boolean().optional().describe('Include hook-llm fallback titles ("Modified X", "Worked on X", raw error logs) — hidden by default as they have ~3% access rate'),
   or: coerceBool.optional().describe('Force OR semantics between query terms from the start (default: AND with automatic OR-fallback when AND returns 0). Aligns with CLI --or.'),
   deep: coerceBool.optional().describe('Tri-state LLM multi-query/HyDE deep search (observations-only). true=force; false=never; omit=AUTO (default ON for mem_search): a normal search that returns weak/few results auto-escalates with ONE Haiku call (query rewritten to keyword/concept/HyDE variants, RRF-fused). Set CLAUDE_MEM_AUTO_DEEP=0 to disable AUTO. Passive recall stays single-query.'),
+  rerank: coerceBool.optional().describe('Opt-in: LLM-rerank the deep-search candidates for ranking precision (one extra Haiku call, ~1.4s). Requires deep=true (no effect on AUTO/normal). Reserve for hard, ranking-sensitive queries where the right memory is likely retrieved but mis-ranked — skip for routine search. Default off.'),
 };
 
 export const memRecentSchema = {